Merge main

This commit is contained in:
Lam Wei Lun
2026-08-21 19:05:15 +08:00
35 changed files with 4585 additions and 570 deletions
+2
View File
@@ -348,6 +348,8 @@ set(lisbslic3r_sources
Polyline.hpp
PresetBundle.cpp
PresetBundle.hpp
PresetCacheFormat.cpp
PresetCacheFormat.hpp
Preset.cpp
Preset.hpp
PrincipalComponents2D.cpp
+3
View File
@@ -28,6 +28,9 @@
#include <cereal/access.hpp>
#include <cereal/types/base_class.hpp>
// The serialize() members below archive ConfigOption hierarchies through
// cereal::base_class, whose registration machinery lives in polymorphic.hpp.
#include <cereal/types/polymorphic.hpp>
namespace Slic3r {
struct FloatOrPercent
+24 -11
View File
@@ -147,6 +147,9 @@ Semver get_version_from_json(std::string file_path)
return Semver();
//throw ConfigurationError(format("Failed loading configuration file \"%1%\": %2%", file_path, err.what()));
}
catch(...) {
return Semver();
}
}
//BBS: add a function to load the key-values from xxx.json
@@ -261,18 +264,28 @@ void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil
}
};
// The four variant sets are immutable after static init and probed for every
// key of every preset loaded; one merged map makes that a single lookup.
// emplace keeps the first insertion, preserving the first-set-wins priority
// of the else-if chain this replaces.
static const std::unordered_map<std::string, int> variant_class = [] {
std::unordered_map<std::string, int> m;
for (const std::string& k : print_options_with_variant) m.emplace(k, 0);
for (const std::string& k : filament_options_with_variant) m.emplace(k, 1);
for (const std::string& k : printer_options_with_variant_1) m.emplace(k, 2);
for (const std::string& k : printer_options_with_variant_2) m.emplace(k, 3);
return m;
}();
for(auto& key :config.keys()){
if(auto iter = print_options_with_variant.find(key); iter != print_options_with_variant.end()){
replace_nil_and_resize(key, process_variant_length);
}
else if(auto iter = filament_options_with_variant.find(key); iter != filament_options_with_variant.end()){
replace_nil_and_resize(key, filament_variant_length);
}
else if(auto iter = printer_options_with_variant_1.find(key); iter != printer_options_with_variant_1.end()){
replace_nil_and_resize(key, machine_variant_length);
}
else if(auto iter = printer_options_with_variant_2.find(key); iter != printer_options_with_variant_2.end()){
replace_nil_and_resize(key, machine_variant_length * 2);
auto iter = variant_class.find(key);
if (iter == variant_class.end())
continue;
switch (iter->second) {
case 0: replace_nil_and_resize(key, process_variant_length); break;
case 1: replace_nil_and_resize(key, filament_variant_length); break;
case 2: replace_nil_and_resize(key, machine_variant_length); break;
case 3: replace_nil_and_resize(key, machine_variant_length * 2); break;
}
}
}
+26 -3
View File
@@ -131,6 +131,10 @@ public:
PrinterVariant() {}
PrinterVariant(const std::string &name) : name(name) {}
std::string name;
// All fields, declaration order — keep in sync; bump CACHE_VERSION on change.
template<class Archive>
void serialize(Archive& ar) { ar(name); } // PrinterVariant
};
struct PrinterModel {
@@ -139,7 +143,7 @@ public:
std::string name;
//BBS: this is internal id for the printer. Currently only used for searching in database
std::string model_id;
PrinterTechnology technology;
PrinterTechnology technology = ptFFF;
std::string family;
std::vector<PrinterVariant> variants;
std::vector<std::string> default_materials;
@@ -162,6 +166,17 @@ public:
}
const PrinterVariant* variant(const std::string &name) const { return const_cast<PrinterModel*>(this)->variant(name); }
// All fields, declaration order — keep in sync; bump CACHE_VERSION on change.
template<class Archive>
void serialize(Archive& ar) // PrinterModel
{
ar(id, name, model_id, technology, family, variants, default_materials,
not_support_bed_types, bed_model, bed_texture, image_bed_type,
bottom_texture_end_name, use_double_extruder_default_texture,
bottom_texture_rect, bottom_texture_rect_longer, middle_texture_rect,
hotend_model);
}
};
std::vector<PrinterModel> models;
@@ -173,6 +188,14 @@ public:
bool valid() const { return ! name.empty() && ! id.empty() && config_version.valid(); }
// All fields, declaration order — keep in sync; bump CACHE_VERSION on change.
template<class Archive>
void serialize(Archive& ar) // VendorProfile
{
ar(name, id, config_version, config_update_url, changelog_url,
models, default_filaments, default_sla_materials);
}
// Load VendorProfile from an ini file.
// If `load_all` is false, only the header with basic info (name, version, URLs) is loaded.
static VendorProfile from_ini(const boost::filesystem::path &path, bool load_all=true);
@@ -427,10 +450,10 @@ public:
Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {}
protected:
Preset() = default;
friend class PresetCollection;
friend class PresetBundle;
Preset() = default;
};
bool is_compatible_with_print (const PresetWithVendorProfile &preset, const PresetWithVendorProfile &active_print, const PresetWithVendorProfile &active_printer);
File diff suppressed because it is too large Load Diff
+70 -3
View File
@@ -2,11 +2,13 @@
#define slic3r_PresetBundle_hpp_
#include "Preset.hpp"
#include "PresetCacheFormat.hpp"
#include "AppConfig.hpp"
#include "PublishSettings.hpp"
#include "enum_bitmask.hpp"
#include <memory>
#include <set>
#include <shared_mutex>
#include <unordered_map>
#include <optional>
@@ -189,6 +191,31 @@ struct PublishedConfig
class PresetBundle
{
public:
// ---- Per-vendor preset cache --------------------------------------------
// One cache file per vendor (plus the Orca filament library), stamped with
// the vendor's own profile version rather than a directory scan. The bytes
// on disk are VendorCacheFile's business (PresetCacheFormat.hpp); what
// lives here is how a cache's contents install into a bundle.
// The cache is not something a caller loads from: a vendor is loaded with
// load_vendor_configs_from_json, which comes from the cache whenever one covers
// it. What is public here is what the cache's own tests drive directly.
// Load a per-vendor cache into this bundle by installing its entries, with
// base_bundle's filament library as the inheritance base. Rejects (returns
// false, with this bundle left clean) unless VendorCacheFile::load accepts
// the file — see its contract for the version and identity checks — and
// every entry installs. Options this build no longer defines are dropped,
// not fatal — the payload names its own keys.
bool load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name,
const Semver& expected_vendor_version, const PresetBundle* base_bundle = nullptr);
// Enable writing a per-vendor cache after a JSON parse (off by default). Cache
// content is pure parse output, so the guard is policy, not correctness: only
// the deliberate generators (load_system_presets_from_json, the cache build
// tool) write files, not every incidental load a dialog performs.
void set_generate_vendor_caches(bool enable) { m_generate_vendor_caches = enable; }
static DynamicPrintConfig construct_full_config(Preset &in_printer_preset,
Preset &in_print_preset,
const DynamicPrintConfig &project_config,
@@ -463,8 +490,12 @@ public:
/*std::pair<PresetsConfigSubstitutions, size_t> load_configbundle(
const std::string &path, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule);*/
//Orca: load config bundle from json, pass the base bundle to support cross vendor inheritance
// Orca: `dir` is where the vendor is looked for — its own directory, whether or
// not the profile JSONs are still there. A whole-vendor load comes from the
// vendor's preset cache whenever one covers the profile on disk, and is parsed
// from the JSONs in `dir` only when none does. Nothing here reads resources.
std::pair<PresetsConfigSubstitutions, size_t> load_vendor_configs_from_json(
const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr);
const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr);
// Export a config bundle file containing all the presets and the names of the active presets.
//void export_configbundle(const std::string &path, bool export_system_settings = false, bool export_physical_printers = false);
@@ -536,11 +567,49 @@ public:
// Orca: for validation only.
bool has_errors(bool check_duplicate_filament_subtypes = false) const;
// Errors the last load recorded. What the cache's error accounting promises —
// a cache-served vendor reports what its parse would — is pinned against this.
int error_count() const { return m_errors; }
// Orca: for validation only. Flag any system preset whose inherits / compatible_printers /
// compatible_prints references a deleted (unknown) or renamed (old) preset name.
bool check_preset_references() const;
// Merge one vendor's presets with the other vendor's presets, report duplicates.
// Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a
// bundle out of several per-vendor caches loaded into separate PresetBundle instances.
std::vector<std::string> merge_presets(PresetBundle &&other);
private:
// Load one vendor from the preset cache installed in `dir`, judged against
// the vendor profile there. False, with this bundle left clean, when there
// is no usable cache and the vendor has to be parsed. This is how
// load_vendor_configs_from_json reads a cache.
bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle);
// Load one source-form preset entry into this bundle: resolve `inherits`,
// flatten, validate and register the preset. Returns the reason loading
// failed, empty on success. See the definition for the sharing contract
// between the JSON parse and the cache load.
// retain_configs, when non-null, names the only presets registered into
// config_maps (a full config copy each). The cache load passes the names its
// entries inherit — the only ones ever looked up again; the JSON parse
// retains all, not knowing what later subfiles inherit.
std::string load_vendor_preset(const CachedPreset& entry,
const std::string& path, const std::string& vendor_name,
const PresetBundle* base_bundle,
LoadConfigBundleAttributes flags,
ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions,
std::map<std::string, DynamicPrintConfig>& config_maps, std::map<std::string, std::string>& filament_id_maps,
PresetCollection* presets_collection, size_t& count, bool is_from_lib,
const std::set<std::string>* retain_configs = nullptr);
// Clear every collection's m_printer_hold_alias, which reset() leaves alone.
void clear_printer_hold_aliases();
// Whether to (re)write a per-vendor cache after a JSON parse.
bool m_generate_vendor_caches { false };
// Orca: validation only - flag any printer with two or more compatible
// filament presets sharing one filament_id (ambiguous AMS subtype match).
bool check_duplicate_filament_subtypes() const;
@@ -548,8 +617,6 @@ private:
//std::pair<PresetsConfigSubstitutions, std::string> load_system_presets(ForwardCompatibilitySubstitutionRule compatibility_rule);
//BBS: add json related logic
std::pair<PresetsConfigSubstitutions, std::string> load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule);
// Merge one vendor's presets with the other vendor's presets, report duplicates.
std::vector<std::string> merge_presets(PresetBundle &&other);
// Update the multicolor information for filaments.
void update_filament_multi_color();
// Update renamed_from and alias maps of system profiles.
+588
View File
@@ -0,0 +1,588 @@
#include "libslic3r/PresetCacheFormat.hpp"
#include <algorithm>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <utility>
#include <boost/crc.hpp>
#include <boost/filesystem.hpp>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/log/trivial.hpp>
#include <boost/nowide/fstream.hpp>
#include <cereal/types/map.hpp>
#include <cereal/types/set.hpp>
#include "libslic3r/Utils.hpp"
namespace Slic3r {
CacheDictionary::CacheDictionary()
{
// ENUM_UNNAMED is index 0 and always the empty name.
m_enum_values.emplace_back();
}
// The ints an enum option holds — one for a coEnum, the whole vector for coEnums.
static std::vector<int> enum_ints(const ConfigOptionDef& def, const ConfigOption* opt)
{
if (def.type == coEnum)
return { opt->getInt() };
return static_cast<const ConfigOptionInts*>(opt)->values;
}
// The name this build gives one of those ints, empty where it has none — a
// nullable option's nil, or a definition carrying no enum_keys_map. Enums are
// written by name so a build that reorders an enum's values still reads it right.
static std::string enum_name_of(const ConfigOptionDef& def, int value)
{
if (def.enum_keys_map != nullptr)
for (const auto& kvp : *def.enum_keys_map)
if (kvp.second == value)
return kvp.first;
return {};
}
void CacheDictionary::collect(const DynamicPrintConfig& config)
{
for (auto it = config.cbegin(); it != config.cend(); ++ it) {
const ConfigOptionDef* def = print_config_def.get(it->first);
if (def == nullptr)
continue; // save_config does not write it either
if (m_key_index.try_emplace(it->first, uint16_t(m_keys.size())).second) {
m_keys.push_back(it->first);
m_types.push_back(uint16_t(def->type));
}
if (def->type != coEnum && def->type != coEnums)
continue;
for (int value : enum_ints(*def, it->second.get())) {
std::string name = enum_name_of(*def, value);
if (! name.empty() && m_enum_index.try_emplace(name, uint16_t(m_enum_values.size())).second)
m_enum_values.push_back(std::move(name));
}
}
}
uint16_t CacheDictionary::key_index(const t_config_option_key& key) const
{
auto it = m_key_index.find(key);
if (it == m_key_index.end())
throw std::runtime_error("preset cache: option " + key + " was never collected into the dictionary");
return it->second;
}
uint16_t CacheDictionary::enum_index(const std::string& name) const
{
if (name.empty())
return ENUM_UNNAMED;
auto it = m_enum_index.find(name);
return it == m_enum_index.end() ? ENUM_UNNAMED : it->second;
}
void CacheDictionary::save(cereal::BinaryOutputArchive& ar) const
{
// Checked here rather than left to the caller: an index that wrapped would
// be written silently, and nothing downstream could tell.
if (m_keys.size() > MAX_ENTRIES || m_enum_values.size() > MAX_ENTRIES)
throw std::runtime_error("preset cache: the option dictionary outgrew the uint16 it is indexed with");
ar(m_keys, m_types, m_enum_values);
}
void CacheDictionary::load(cereal::BinaryInputArchive& ar)
{
ar(m_keys, m_types, m_enum_values);
if (m_keys.size() != m_types.size())
throw std::runtime_error("preset cache: dictionary key and type tables differ in length");
if (m_keys.size() > MAX_ENTRIES || m_enum_values.size() > MAX_ENTRIES)
throw std::runtime_error("preset cache: dictionary is larger than the uint16 it is indexed with");
if (m_enum_values.empty() || ! m_enum_values.front().empty())
throw std::runtime_error("preset cache: dictionary is missing its unnamed-enum slot");
// Resolved once per file: every option read after this is a vector index.
m_defs.resize(m_keys.size());
for (size_t i = 0; i < m_keys.size(); ++ i) {
const ConfigOptionDef* def = print_config_def.get(m_keys[i]);
m_defs[i] = (def != nullptr && uint16_t(def->type) == m_types[i]) ? def : nullptr;
}
}
// ---- one config -----------------------------------------------------------
static void save_enum_option(cereal::BinaryOutputArchive& ar, const ConfigOptionDef& def,
const ConfigOption* opt, const CacheDictionary& dict)
{
const std::vector<int> values = enum_ints(def, opt);
ar(uint32_t(values.size()));
for (int value : values) {
const uint16_t idx = dict.enum_index(enum_name_of(def, value));
ar(idx);
if (idx == CacheDictionary::ENUM_UNNAMED)
ar(int32_t(value));
}
}
// `config` may be null, in which case the option is read and dropped.
static void load_enum_option(cereal::BinaryInputArchive& ar, ConfigOptionType type,
const ConfigOptionDef* def, DynamicPrintConfig* config,
const CacheDictionary& dict)
{
uint32_t cnt = 0;
ar(cnt);
if (type == coEnum && cnt != 1)
throw std::runtime_error("preset cache: a scalar enum carrying more than one value");
// Every element is read whatever happens, so the stream stays in sync and
// whatever follows this option still loads.
bool usable = def != nullptr && config != nullptr;
std::vector<int> values;
values.reserve(cnt);
for (uint32_t i = 0; i < cnt; ++ i) {
uint16_t idx = 0;
ar(idx);
if (! dict.valid_enum_index(idx))
throw std::runtime_error("preset cache: enum value index past the end of the dictionary");
if (idx == CacheDictionary::ENUM_UNNAMED) {
// An int the writer could not name — a nil, or an option whose
// definition carried no enum_keys_map. It travels verbatim.
int32_t raw = 0;
ar(raw);
values.push_back(int(raw));
continue;
}
if (! usable)
continue; // the index above was this element's whole payload
if (def->enum_keys_map == nullptr) {
usable = false; // this build no longer maps this option's names
continue;
}
const auto it = def->enum_keys_map->find(dict.enum_name_at(idx));
if (it == def->enum_keys_map->end()) {
usable = false; // a value this build dropped: the option goes with it
continue;
}
values.push_back(it->second);
}
if (! usable)
return;
if (type == coEnum) {
config->set_key_value(def->opt_key, new ConfigOptionEnumGeneric(def->enum_keys_map, values.front()));
} else {
auto* opt = def->nullable ? static_cast<ConfigOptionInts*>(new ConfigOptionEnumsGenericNullable(def->enum_keys_map))
: static_cast<ConfigOptionInts*>(new ConfigOptionEnumsGeneric(def->enum_keys_map));
opt->values = std::move(values);
config->set_key_value(def->opt_key, opt);
}
}
void save_config(cereal::BinaryOutputArchive& ar, const DynamicPrintConfig& config, const CacheDictionary& dict)
{
struct Written { uint16_t idx; const ConfigOptionDef* def; const ConfigOption* opt; };
std::vector<Written> written;
written.reserve(config.size());
for (auto it = config.cbegin(); it != config.cend(); ++ it)
if (const ConfigOptionDef* def = print_config_def.get(it->first))
written.push_back({ dict.key_index(it->first), def, it->second.get() });
ar(uint32_t(written.size()));
for (const Written& w : written) {
ar(w.idx);
if (w.def->type == coEnum || w.def->type == coEnums)
save_enum_option(ar, *w.def, w.opt, dict);
else
w.def->save_option_to_archive(ar, w.opt);
}
}
// `config` null means: read everything, keep nothing.
static void read_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig* config, const CacheDictionary& dict)
{
uint32_t cnt = 0;
ar(cnt);
if (config != nullptr)
config->clear();
// Reused across the loop: constructing a ConfigOptionDef per dropped option
// would allocate its strings and vectors for nothing.
ConfigOptionDef scratch;
for (uint32_t i = 0; i < cnt; ++ i) {
uint16_t idx = 0;
ar(idx);
if (! dict.valid_key_index(idx))
throw std::runtime_error("preset cache: option index past the end of the dictionary");
const ConfigOptionType type = dict.type_at(idx);
const ConfigOptionDef* def = dict.def_at(idx);
if (type == coEnum || type == coEnums) {
load_enum_option(ar, type, def, config, dict);
} else if (def != nullptr && config != nullptr) {
config->set_key_value(def->opt_key, def->load_option_from_archive(ar));
} else {
// Read by the type the writer recorded, then drop: the same outcome
// a JSON profile gets for an option this build no longer has.
scratch.type = type;
std::unique_ptr<ConfigOption> discard(scratch.load_option_from_archive(ar));
}
}
}
void load_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig& config, const CacheDictionary& dict)
{
read_config(ar, &config, dict);
}
void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict)
{
read_config(ar, nullptr, dict);
}
// ---- The per-vendor cache file (<vendor>.opc) -----------------------------
namespace {
#pragma pack(push, 1)
struct CacheFileHeader {
uint32_t magic;
uint32_t version;
uint64_t data_size;
uint32_t crc32;
};
#pragma pack(pop)
static_assert(sizeof(CacheFileHeader) == 20, "CacheFileHeader must be 20 bytes");
constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ"
// Bump when the wire format changes in a way the payload cannot describe
// itself out of: reordering, removing or retyping a field of a hand-written
// serialize() (VendorProfile and its nested types, CachedPreset via
// save_entries below), or a change to the cache's own layout or the
// meaning of its stamps. Option-schema drift is NOT such a change — the
// dictionary handles it, which is why this no longer moves every release.
constexpr uint32_t CACHE_VERSION = 1;
// A stamp-string read that refuses an absurd length before allocating anything.
// The stamps are read from files named from the outside (peek_version is
// pointed at whatever <vendor>.opc a directory holds), so the length word may
// be arbitrary bytes — and a resize to a garbage 64-bit length does not fail as
// a catchable bad_alloc here, it takes the app down through the out-of-memory
// handler. A vendor name or profile version is a short token; anything longer
// is not a cache this build wrote.
std::string read_bounded_string(cereal::BinaryInputArchive& ar)
{
constexpr uint64_t MAX_STAMP_LEN = 1024;
cereal::size_type len = 0;
ar(cereal::make_size_tag(len));
if (uint64_t(len) > MAX_STAMP_LEN)
throw std::runtime_error("preset cache: string length out of bounds");
std::string s(size_t(len), '\0');
ar(cereal::binary_data(s.data(), size_t(len)));
return s;
}
// The prologue every cache reader starts with: the format version, then the
// vendor's identity. Returns the vendor version stamped on a body this build can
// read, empty on anything else — which is the same answer as "not this vendor".
std::string read_cache_stamps(cereal::BinaryInputArchive& ar, const std::string& expected_vendor_name)
{
// The version is judged before anything variable-length is read: on a body
// that is not a per-vendor cache of this version, the bytes where a string
// length would sit may be arbitrary framing.
uint32_t cache_version = 0;
ar(cache_version);
if (cache_version != CACHE_VERSION)
return {};
const std::string vendor_name = read_bounded_string(ar);
const std::string vendor_version = read_bounded_string(ar);
if (vendor_name != expected_vendor_name)
return {};
return vendor_version;
}
// A cache stays usable as long as it was built from a vendor profile at least
// as new as the one now on disk. Profiles whose version is invalid cannot be
// judged this way and are never served from cache; where no profile sits
// beside the cache at all, nothing can be newer than it — that state is passed
// as Semver::inf(), which no real profile can carry (an invalid version could
// not say it apart from "profile there but unjudgeable", and zero would
// collide with a genuine "0.0.0"). This is the serve rule; the install rule
// (cache_covers in PresetBundle.cpp) deliberately reads an unjudgeable profile
// the other way, so the two are not one function.
bool cache_covers_version(const std::string& cached, const Semver& on_disk)
{
if (on_disk == Semver::inf())
return true; // before parsing `cached`: nothing exists that the stamp must cover
if (! on_disk.valid())
return false;
const auto cached_ver = Semver::parse(cached);
return cached_ver && *cached_ver >= on_disk;
}
// CachedPreset on the wire: all fields, declaration order, in one place.
// `config` writes, reads or skips the config sitting in the middle of that
// order — the three things a reader can want to do with it — so save, load and
// the name peek below cannot drift apart. Keep in sync with the struct in
// PresetCacheFormat.hpp and bump CACHE_VERSION on change. Written here rather
// than as a serialize() member because the config needs the file's dictionary,
// which cereal cannot thread through one.
template<class Archive, class Entry, class ConfigFn>
void visit_entry(Archive& ar, Entry& e, ConfigFn&& config)
{
ar(e.name, e.sub_path);
config();
ar(e.inherits, e.description, e.instantiation, e.setting_id, e.filament_id, e.renamed_from);
}
// The count comes from a file that has already passed magic and CRC, but a
// reserve is a promise to allocate: cap it and let push_back grow the rest.
constexpr uint32_t MAX_RESERVED_ENTRIES = 4096;
void save_entries(cereal::BinaryOutputArchive& ar,
const std::vector<CachedPreset>& entries,
const CacheDictionary& dict)
{
ar(uint32_t(entries.size()));
for (const CachedPreset& e : entries)
visit_entry(ar, e, [&] { save_config(ar, e.config_src, dict); });
}
void load_entries(cereal::BinaryInputArchive& ar,
std::vector<CachedPreset>& entries,
const CacheDictionary& dict)
{
uint32_t cnt = 0;
ar(cnt);
entries.clear();
entries.reserve(std::min(cnt, MAX_RESERVED_ENTRIES));
for (uint32_t i = 0; i < cnt; ++ i) {
CachedPreset e;
visit_entry(ar, e, [&] { load_config(ar, e.config_src, dict); });
entries.push_back(std::move(e));
}
}
// Read a raw cache body: verify magic, size, CRC.
bool read_cache_blob(const std::string& path, std::string& out_blob)
{
try {
boost::nowide::ifstream ifs(path, std::ios::binary);
if (!ifs.is_open())
return false;
CacheFileHeader fhdr;
if (!ifs.read(reinterpret_cast<char*>(&fhdr), sizeof(fhdr)))
return false;
if (fhdr.magic != CACHE_MAGIC)
return false;
// data_size is 8 bytes from a file nothing has authenticated yet, and
// it is about to size an allocation. The body is the whole of the file
// behind the header — anything else is not a cache this build wrote.
ifs.seekg(0, std::ios::end);
const std::streamoff file_size = ifs.tellg();
if (file_size < std::streamoff(sizeof(fhdr)) ||
fhdr.data_size == 0 ||
fhdr.data_size != uint64_t(file_size) - sizeof(fhdr))
return false;
ifs.seekg(sizeof(fhdr), std::ios::beg);
out_blob.assign(fhdr.data_size, '\0');
if (!ifs.read(&out_blob[0], static_cast<std::streamsize>(fhdr.data_size)))
return false;
boost::crc_32_type crc;
crc.process_bytes(out_blob.data(), out_blob.size());
if (crc.checksum() != fhdr.crc32) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: CRC mismatch: " << path;
return false;
}
return true;
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: read failed (" << path << "): " << e.what();
return false;
}
}
// Write a cache body behind the standard 20-byte file header. False when the
// file could not be opened or written whole.
bool write_cache_blob(const std::string& path, const std::string& blob)
{
boost::crc_32_type crc;
crc.process_bytes(blob.data(), blob.size());
// Written beside the target and moved into place, as AppConfig::save does:
// a cache is truncated and rewritten in full, so a write that dies partway
// would otherwise leave a header claiming more body than the file holds.
// The PID suffix also keeps two instances writing the same vendor from
// interleaving.
const std::string tmp_path = path + "." + std::to_string(get_current_pid()) + ".tmp";
try {
boost::filesystem::create_directories(boost::filesystem::path(path).parent_path());
{
boost::nowide::ofstream ofs(tmp_path, std::ios::binary | std::ios::trunc);
if (!ofs.is_open()) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: cannot open for writing: " << tmp_path;
return false;
}
CacheFileHeader fhdr;
fhdr.magic = CACHE_MAGIC;
fhdr.version = CACHE_VERSION;
fhdr.data_size = static_cast<uint64_t>(blob.size());
fhdr.crc32 = crc.checksum();
ofs.write(reinterpret_cast<const char*>(&fhdr), sizeof(fhdr));
ofs.write(blob.data(), static_cast<std::streamsize>(blob.size()));
ofs.close(); // flush; close() raises failbit on error
if (! ofs.good()) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: write failed (" << tmp_path << ")";
boost::system::error_code ec;
boost::filesystem::remove(tmp_path, ec);
return false;
}
}
if (const std::error_code ec = rename_file(tmp_path, path)) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: could not move " << tmp_path << " into place: " << ec.message();
boost::system::error_code rm;
boost::filesystem::remove(tmp_path, rm);
return false;
}
return true;
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: write failed (" << path << "): " << e.what();
boost::system::error_code ec;
boost::filesystem::remove(tmp_path, ec);
return false;
}
}
} // anonymous namespace
// static
bool VendorCacheFile::save(const std::string& path, const std::string& vendor_name,
const std::string& vendor_version, const VendorCacheData& data)
{
try {
// Collected before anything is written: the dictionary sits ahead of the
// entries so a reader resolves it once and then indexes.
CacheDictionary dict;
for (const std::vector<CachedPreset>* entries : { &data.process_entries, &data.filament_entries, &data.machine_entries })
for (const CachedPreset& e : *entries)
dict.collect(e.config_src);
std::ostringstream body(std::ios::binary);
{
cereal::BinaryOutputArchive ar(body);
ar(CACHE_VERSION);
ar(vendor_name, vendor_version);
dict.save(ar);
ar(data.vendors);
save_entries(ar, data.process_entries, dict);
save_entries(ar, data.filament_entries, dict);
save_entries(ar, data.machine_entries, dict);
ar(data.parse_errors);
}
return write_cache_blob(path, body.str());
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: failed to save vendor cache " << path << ": " << e.what();
return false;
}
}
// static
bool VendorCacheFile::load(const std::string& path, const std::string& expected_vendor_name,
const Semver& expected_vendor_version, VendorCacheData& data)
{
std::string blob;
if (! read_cache_blob(path, blob))
return false;
try {
// Read in place: an istringstream would copy the blob once more just to
// stream over it.
boost::iostreams::stream<boost::iostreams::array_source> body(blob.data(), blob.size());
cereal::BinaryInputArchive ar(body);
const std::string vendor_version = read_cache_stamps(ar, expected_vendor_name);
if (vendor_version.empty() || ! cache_covers_version(vendor_version, expected_vendor_version))
return false;
CacheDictionary dict;
dict.load(ar);
ar(data.vendors);
load_entries(ar, data.process_entries, dict);
load_entries(ar, data.filament_entries, dict);
load_entries(ar, data.machine_entries, dict);
ar(data.parse_errors);
if (data.vendors.find(expected_vendor_name) == data.vendors.end())
throw std::runtime_error("vendor cache does not carry its own vendor profile");
return true;
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: rejecting vendor cache " << path << ": " << e.what();
return false;
}
}
// static
std::string VendorCacheFile::peek_version(const std::string& path, const std::string& expected_vendor_name)
{
try {
boost::nowide::ifstream ifs(path, std::ios::binary);
CacheFileHeader fhdr;
if (! ifs.read(reinterpret_cast<char*>(&fhdr), sizeof(fhdr)) || fhdr.magic != CACHE_MAGIC)
return {};
// Only the head of the body is read, and its CRC left unverified: the
// stamps sit at the front, and this answers "what version is this?"
// without paying for tens of megabytes. Callers that need to know the
// file is whole use usable_version instead.
std::string head(static_cast<size_t>(std::min<uint64_t>(fhdr.data_size, 1024)), '\0');
if (! ifs.read(&head[0], static_cast<std::streamsize>(head.size())))
return {};
std::istringstream body(head, std::ios::binary);
cereal::BinaryInputArchive ar(body);
return read_cache_stamps(ar, expected_vendor_name);
} catch (const std::exception&) {
return {};
}
}
// static
Semver VendorCacheFile::usable_version(const std::string& path, const std::string& expected_vendor_name)
{
std::string blob;
if (! read_cache_blob(path, blob))
return Semver::invalid();
try {
boost::iostreams::stream<boost::iostreams::array_source> body(blob.data(), blob.size());
cereal::BinaryInputArchive ar(body);
const auto ver = Semver::parse(read_cache_stamps(ar, expected_vendor_name));
return ver ? *ver : Semver::invalid();
} catch (const std::exception&) {
return Semver::invalid();
}
}
// static
bool VendorCacheFile::carries_preset(const std::string& path, const std::string& vendor_name,
Preset::Type type, const std::string& preset_name)
{
std::string blob;
if (! read_cache_blob(path, blob))
return false;
try {
boost::iostreams::stream<boost::iostreams::array_source> body(blob.data(), blob.size());
cereal::BinaryInputArchive ar(body);
if (read_cache_stamps(ar, vendor_name).empty())
return false;
CacheDictionary dict;
dict.load(ar);
VendorMap vendors;
ar(vendors);
// Reused: every entry overwrites it, and only its name is ever looked at.
CachedPreset entry;
// Written in this order by save. The list that could carry the preset
// is the last one worth reading.
for (Preset::Type kind : { Preset::TYPE_PRINT, Preset::TYPE_FILAMENT, Preset::TYPE_PRINTER }) {
uint32_t cnt = 0;
ar(cnt);
for (uint32_t i = 0; i < cnt; ++ i) {
visit_entry(ar, entry, [&] { skip_config(ar, dict); });
if (kind == type && entry.name == preset_name)
return true;
}
if (kind == type)
return false;
}
return false;
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: could not read preset names from " << path << ": " << e.what();
return false;
}
}
} // namespace Slic3r
+192
View File
@@ -0,0 +1,192 @@
#ifndef slic3r_PresetCacheFormat_hpp_
#define slic3r_PresetCacheFormat_hpp_
#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>
#include <cereal/archives/binary.hpp>
#include <cereal/types/string.hpp>
#include <cereal/types/vector.hpp>
#include "libslic3r/Config.hpp"
#include "libslic3r/Preset.hpp"
#include "libslic3r/PrintConfig.hpp"
#include "libslic3r/Semver.hpp"
namespace Slic3r {
// How the preset cache writes a DynamicPrintConfig.
//
// Not through the global cereal hooks in PrintConfig.hpp: those key an option by
// its serialization_key_ordinal, which ConfigDef::add assigns by declaration
// order at static-init time. Inserting one option into the middle of
// PrintConfig.cpp shifts every later ordinal, and the lookup on the way back in
// then SUCCEEDS on the wrong option — where the two share a type, and hundreds
// of coFloat/coBool/coInt options do, the bytes deserialize cleanly into the
// wrong key. Silently wrong print settings, no error. Those hooks are also the
// undo/redo wire format, where the process cannot change underneath them, so
// they stay as they are and the cache keys by name instead.
//
// Names are not repeated per preset. Each cache file carries one dictionary of
// the distinct opt_keys it uses, the type each was written as, and the distinct
// enum value names; an option on the wire is then a uint16 index into it plus
// its value. The dictionary is resolved to this build's option definitions once
// per file, after which reading an option is a vector index.
class CacheDictionary
{
public:
CacheDictionary();
// Index reserved in the enum table for an int the writing build could not
// name — a nullable option's nil, or a definition carrying no
// enum_keys_map. The raw int32 follows it on the wire and is loaded
// verbatim, so those values survive too.
static constexpr uint16_t ENUM_UNNAMED = 0;
// ---- writing ----
// Record every key and enum value `config` uses. Call for every config that
// will be written, before writing the dictionary.
void collect(const DynamicPrintConfig& config);
uint16_t key_index(const t_config_option_key& key) const;
// ENUM_UNNAMED for an empty name or one that was never collected.
uint16_t enum_index(const std::string& name) const;
// ---- reading ----
// The definition an index resolves to in THIS build, or nullptr where the
// key is unknown here or is now defined with a different type. A nullptr
// entry's value is still read — using type_at(idx), the type the writer
// recorded — and then dropped, which is what a JSON profile gets for an
// option this build no longer has.
const ConfigOptionDef* def_at(uint16_t idx) const { return m_defs[idx]; }
ConfigOptionType type_at(uint16_t idx) const { return ConfigOptionType(m_types[idx]); }
const std::string& enum_name_at(uint16_t idx) const { return m_enum_values[idx]; }
// m_defs, not m_keys: only load() sizes it, so this is false for every index
// on a dictionary that was collected rather than read.
bool valid_key_index(uint16_t idx) const { return size_t(idx) < m_defs.size(); }
bool valid_enum_index(uint16_t idx) const { return size_t(idx) < m_enum_values.size(); }
// The layout these two agree on is covered by CACHE_VERSION (PresetCacheFormat.cpp);
// bump it when they change.
// Throws when either table outgrew the uint16 the wire format indexes it
// with. Both are bounded by the option count (912 at the time of writing), so
// that is a build-time failure in CI, not a runtime one.
void save(cereal::BinaryOutputArchive& ar) const;
// Throws on a dictionary that cannot be indexed as written.
void load(cereal::BinaryInputArchive& ar);
private:
// Indices are uint16, so a table may hold at most this many entries.
static constexpr size_t MAX_ENTRIES = 0xFFFF;
std::vector<std::string> m_keys;
// ConfigOptionType, as written. Sixteen bits, not eight: coVectorType is
// 0x4000, so every vector type — coFloats, coEnums, coStrings — is above
// 255, and a byte would fold each one onto its scalar counterpart.
std::vector<uint16_t> m_types;
std::vector<std::string> m_enum_values; // [ENUM_UNNAMED] is always empty
// Writing.
std::unordered_map<std::string, uint16_t> m_key_index;
std::unordered_map<std::string, uint16_t> m_enum_index;
// Reading, resolved once by load().
std::vector<const ConfigOptionDef*> m_defs;
};
// One config, keyed through `dict`. Options print_config_def does not know are
// not written: nothing could give them a type on the way back in.
void save_config(cereal::BinaryOutputArchive& ar, const DynamicPrintConfig& config, const CacheDictionary& dict);
// Throws only on a payload that cannot be indexed; an option this build cannot
// place is dropped, not fatal.
void load_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig& config, const CacheDictionary& dict);
// Consume one config without building it, for a reader that only wants what
// comes after.
void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict);
// One preset as its JSON subfile states it: the config diff, the name of the
// preset it inherits, and the parse metadata — everything the parse phase of
// load_vendor_configs_from_json extracts and nothing it derives. Inheritance
// is resolved when the entry is installed, against whatever filament library
// is loaded then, so a cache carries no other vendor's values and no other
// vendor's update can make it stale.
// Written and read by visit_entry in PresetCacheFormat.cpp, which lists every
// field below in this order — once, for the save, the load and the name peek alike.
struct CachedPreset
{
std::string name;
std::string sub_path; // path under the vendor's directory
DynamicPrintConfig config_src; // the preset's own diff, nothing inherited
std::string inherits;
std::string description;
std::string instantiation; // "true"/"false" as stated; anything else was already counted as a parse error
std::string setting_id;
std::string filament_id;
std::vector<std::string> renamed_from;
};
// What one per-vendor cache file carries besides its stamps: the vendor profile
// map, the presets in source form, and how many errors their parse counted.
struct VendorCacheData
{
VendorMap vendors;
std::vector<CachedPreset> process_entries;
std::vector<CachedPreset> filament_entries;
std::vector<CachedPreset> machine_entries;
uint64_t parse_errors = 0;
};
// A per-vendor preset cache file (<vendor>.opc): a 20-byte header (magic, format
// version, body size, CRC) framing one cereal body — stamps (format version,
// vendor name, vendor profile version), the option dictionary, then the
// VendorCacheData. Everything about those bytes lives here; when a vendor is
// served from its cache, and how entries install into a bundle, is
// PresetBundle's business.
class VendorCacheFile
{
public:
// Save one vendor (vendor_name at vendor_version). False when the file
// could not be written whole.
static bool save(const std::string& path, const std::string& vendor_name,
const std::string& vendor_version, const VendorCacheData& data);
// Read a whole cache into `data`. False — with `data` in an unspecified
// state — unless the file is a cache this build wrote, its CRC holds, it
// names this vendor, it was built from a vendor profile at least as new as
// `expected_vendor_version`, and it carries its own vendor profile. An
// invalid expected version (a profile whose version
// cannot be judged) is never served from cache; Semver::inf() (no profile
// beside the cache at all) accepts whatever is cached.
static bool load(const std::string& path, const std::string& expected_vendor_name,
const Semver& expected_vendor_version, VendorCacheData& data);
// Read the profile version a cache was stamped with, without deserializing
// its presets. Empty if the file is unreadable, not a cache this build
// understands, or not this vendor's. This is how an installed vendor's
// version is known when only its cache is installed.
static std::string peek_version(const std::string& path, const std::string& expected_vendor_name);
// The profile version an installed cache can actually be served at, or an
// invalid Semver when the file is not a cache this build can read. Unlike
// peek_version this verifies the body's CRC, at the cost of reading the
// whole file: where the cache is the vendor's whole installation, "a file
// is there" is not enough to call it installed, and a vendor wrongly
// believed installed is never repaired.
static Semver usable_version(const std::string& path, const std::string& expected_vendor_name);
// Whether a cache carries a preset of `type` under `preset_name`, without
// installing any of them. False when the file is not a cache this build can
// read. The three kinds are written in one stream, so reaching the machines
// means reading past the processes and filaments — their configs are consumed
// and dropped rather than built. This is how a build that ships caches instead
// of preset JSONs answers "which vendor carries this preset?".
static bool carries_preset(const std::string& path, const std::string& vendor_name,
Preset::Type type, const std::string& preset_name);
};
} // namespace Slic3r
#endif // slic3r_PresetCacheFormat_hpp_
+2 -1
View File
@@ -2488,7 +2488,8 @@ namespace cereal {
archive(serialization_key_ordinal);
assert(serialization_key_ordinal > 0);
auto it = Slic3r::print_config_def.by_serialization_key_ordinal.find(serialization_key_ordinal);
assert(it != Slic3r::print_config_def.by_serialization_key_ordinal.end());
if (it == Slic3r::print_config_def.by_serialization_key_ordinal.end())
throw std::runtime_error("VendorCache: unknown serialization_key_ordinal " + std::to_string(serialization_key_ordinal) + " - cache is stale");
config.set_key_value(it->second->opt_key, it->second->load_option_from_archive(archive));
}
}
+13
View File
@@ -190,6 +190,19 @@ public:
os << self.to_string();
return os;
}
// cereal: round-trip through the standard 3-part string (major.minor.patch).
// to_string() uses a BBS 4-part format that semver_parse() cannot read back.
template<class Archive>
std::string save_minimal(const Archive&) const { return to_string_sf(); }
template<class Archive>
void load_minimal(const Archive&, const std::string& s) {
auto v = Semver::parse(s);
if (! v)
throw std::runtime_error("Semver: cannot parse serialized version: " + s);
*this = std::move(*v);
}
private:
semver_t ver;
+38 -5
View File
@@ -3,6 +3,7 @@
#include <iomanip>
#include <locale>
#include <set>
#include <utility>
#include <functional>
#include <type_traits>
@@ -18,6 +19,7 @@
#include <openssl/md5.h>
#include "libslic3r.h"
#include "Semver.hpp"
//define CLI errors
@@ -722,11 +724,42 @@ void copy_directory_recursively(const boost::filesystem::path& source,
std::function<bool(const std::string)> filter = nullptr,
bool merge_mode = false);
// Install vendor bundles from resources directory to data directory
// bundle_names: vector of vendor bundle names (without .json extension)
// resource_subdir: subdirectory under resources_dir() (default: "profiles")
// data_subdir: subdirectory under data_dir() (default: "system")
// Returns: true if all bundles installed successfully, false otherwise
// ---- Vendor installation on disk ------------------------------------------
// How a vendor bundle is installed from resources into data_dir()/system: as
// its profile and preset JSONs or, in a build that ships preset caches, as its
// .opc preset cache alone. Loading what is installed is PresetBundle's business;
// the cache file format itself is VendorCacheFile's (PresetCacheFormat.hpp).
// True if `vendor` is installed in data_dir()/system. A build that ships preset
// caches installs the cache alone, so it — not the profile — marks a vendor
// installed; a cache this build cannot read marks nothing.
bool is_vendor_installed(const std::string& vendor);
// The version the installed vendor would be loaded at: its cache's stamp while
// that covers the profile beside it, the profile's own version once it does not.
// Invalid Semver if neither form is installed.
Semver installed_vendor_version(const std::string& vendor);
// Remove every form `vendor` can be installed as from data_dir()/system: its
// profile, its preset cache, and its preset directory.
void remove_installed_vendor(const std::string& vendor);
// The vendors `dir` holds, sorted: one is named by its profile or, in a build that
// ships preset caches instead of the raw profile JSONs, by its cache alone.
std::set<std::string> vendor_names_in(const boost::filesystem::path& dir);
// The version a build ships `vendor` at: whichever of its preset cache and its
// profile is newer, that being the one installing lays down. Invalid Semver if the
// build ships neither.
Semver resource_vendor_version(const std::string& vendor);
// Install vendors from the resources directory into the data directory, each as
// its preset cache or as its profile and preset JSONs — whichever of the two the
// build ships at the newer version. Anything the previous install of that vendor
// left behind goes, so only the form just installed is there to be loaded.
// bundle_names: vendor names, without extension.
// Every bundle that can be installed is, whatever the others do. Returns false
// if any named bundle could not be installed.
bool install_vendor_bundles_from_resources(const std::vector<std::string>& bundle_names,
const std::string& resource_subdir = "profiles",
const std::string& data_subdir = "system");
+141 -13
View File
@@ -17,6 +17,10 @@
#include "Platform.hpp"
#include "Time.hpp"
#include "libslic3r.h"
// For the vendor-installation helpers: the vendor profile version
// (get_version_from_json) and the preset cache stamp (VendorCacheFile).
#include "Preset.hpp"
#include "PresetCacheFormat.hpp"
#ifdef __APPLE__
#include "MacUtils.hpp"
@@ -1724,6 +1728,85 @@ void copy_directory_recursively(const boost::filesystem::path& source,
return;
}
// ---- Vendor installation on disk ------------------------------------------
// Whether a cache stamped `cache_ver` still speaks for a vendor whose profile on
// disk claims `profile_ver`: it does unless the profile has moved ahead of it. A
// profile that is missing or carries no judgeable version cannot be ahead of
// anything. The one rule behind both "which form gets installed" and "which form
// is installed"; they must not drift apart. Deliberately NOT the serve rule
// (VendorCacheFile::load), which refuses an unjudgeable profile instead.
static bool cache_covers(const Semver& cache_ver, const Semver& profile_ver)
{
return cache_ver.valid() && (! profile_ver.valid() || cache_ver >= profile_ver);
}
bool is_vendor_installed(const std::string& vendor)
{
const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR;
// A cache is the whole of a cache-only installation, so a file this build
// cannot serve the vendor from is not an installation. Left counted as one,
// the updater would never lay a working copy down.
return boost::filesystem::exists(dir / (vendor + ".json"))
|| VendorCacheFile::usable_version((dir / (vendor + ".opc")).string(), vendor).valid();
}
Semver installed_vendor_version(const std::string& vendor)
{
const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR;
const boost::filesystem::path json = dir / (vendor + ".json");
// Guarded: get_version_from_json logs an error and throws-and-catches its way
// to an invalid version on a file that is not there, and a cache-only vendor
// never has one.
const Semver from_json = boost::filesystem::exists(json) ? get_version_from_json(json.string()) : Semver();
const Semver from_cache = VendorCacheFile::usable_version((dir / (vendor + ".opc")).string(), vendor);
// Whichever form a load would serve.
return cache_covers(from_cache, from_json) ? from_cache : from_json;
}
void remove_installed_vendor(const std::string& vendor)
{
const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR;
boost::filesystem::remove(dir / (vendor + ".json"));
boost::filesystem::remove(dir / (vendor + ".opc"));
if (boost::filesystem::exists(dir / vendor))
boost::filesystem::remove_all(dir / vendor);
}
std::set<std::string> vendor_names_in(const boost::filesystem::path& dir)
{
std::set<std::string> names;
for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) {
const auto& path = dir_entry.path();
if (Slic3r::is_json_file(path.string()) || path.extension() == ".opc")
names.insert(path.stem().string());
}
return names;
}
// A vendor's preset cache is the whole of its installation: it carries the presets,
// the vendor profile and the version they were built at, so where one ships nothing
// else needs copying. Unless the profile beside it claims a newer version — a cache
// generated before that profile was bumped is out of date, and a cache that cannot
// be read is no installation at all — and the vendor is installed the way it was
// before caches existed, as its profile and the preset JSONs it points at. Returns
// the version the cache is stamped with, invalid when it is not the form to install.
static Semver installable_cache_version(const boost::filesystem::path& dir, const std::string& vendor)
{
const auto cache_ver = Semver::parse(VendorCacheFile::peek_version((dir / (vendor + ".opc")).string(), vendor));
if (! cache_ver)
return Semver::invalid();
const Semver profile_ver = get_version_from_json((dir / (vendor + ".json")).string());
return cache_covers(*cache_ver, profile_ver) ? *cache_ver : Semver::invalid();
}
Semver resource_vendor_version(const std::string& vendor)
{
const boost::filesystem::path dir = boost::filesystem::path(resources_dir()) / "profiles";
const Semver ver = installable_cache_version(dir, vendor);
return ver.valid() ? ver : get_version_from_json((dir / (vendor + ".json")).string());
}
bool install_vendor_bundles_from_resources(
const std::vector<std::string>& bundle_names,
const std::string& resource_subdir,
@@ -1736,37 +1819,82 @@ bool install_vendor_bundles_from_resources(
BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources...";
// One vendor that cannot be installed is one vendor missing, not a reason to
// leave the rest uninstalled. The caller is told, and every bundle that can
// be laid down is.
bool all_installed = true;
for (const auto &bundle : bundle_names) {
try {
if (bundle.empty()) {
BOOST_LOG_TRIVIAL(warning) << "Refusing to install a bundle with no name";
all_installed = false;
continue;
}
// Install the JSON file
auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json");
auto path_in_vendors = (vendor_path / bundle).replace_extension(".json");
auto cache_in_rsrc = (rsrc_path / bundle).replace_extension(".opc");
auto cache_in_vendors = (vendor_path / bundle).replace_extension(".opc");
if (!fs::exists(path_in_rsrc)) {
// Either form of the vendor will do: a build may ship it as a cache alone.
if (!fs::exists(path_in_rsrc) && !fs::exists(cache_in_rsrc)) {
BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle;
return false;
all_installed = false;
continue;
}
// Create target directory if needed
if (!fs::exists(vendor_path))
fs::create_directories(vendor_path);
// Copy JSON file
std::string error_message;
CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false);
if (cfr != CopyFileResult::SUCCESS) {
BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message;
return false;
bool installed_cache = false;
if (installable_cache_version(rsrc_path, bundle).valid()) {
installed_cache = copy_file(cache_in_rsrc.string(), cache_in_vendors.string(), error_message, false) == CopyFileResult::SUCCESS;
if (! installed_cache) {
BOOST_LOG_TRIVIAL(warning) << "Failed to copy " << bundle << ".opc: " << error_message;
} else if (! VendorCacheFile::usable_version(cache_in_vendors.string(), bundle).valid()) {
// The copy is what will be loaded, so it — not the kilobyte
// peek that chose this form — decides whether the profile
// beside it can go.
BOOST_LOG_TRIVIAL(warning) << "Installed cache for " << bundle << " cannot be read; installing its profile instead";
boost::system::error_code ec;
fs::remove(cache_in_vendors, ec);
installed_cache = false;
}
}
if (! installed_cache) {
CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false);
if (cfr != CopyFileResult::SUCCESS) {
BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message;
all_installed = false;
continue;
}
// Only now: an earlier install's cache would shadow this profile,
// but removing it before the profile lands would leave neither.
boost::system::error_code ec;
fs::remove(cache_in_vendors, ec);
} else {
// Left in place, an earlier install's profile would shadow the cache.
boost::system::error_code ec;
fs::remove(path_in_vendors, ec);
if (ec)
BOOST_LOG_TRIVIAL(warning) << "Could not remove the superseded profile " << path_in_vendors.string() << ": " << ec.message();
}
// Copy the vendor directory (if it exists)
auto dir_in_rsrc = rsrc_path / bundle;
auto dir_in_vendors = vendor_path / bundle;
if (fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) {
// Remove existing directory
if (fs::exists(dir_in_vendors))
fs::remove_all(dir_in_vendors);
// Whatever is installed came from an earlier version of this vendor and
// would be parsed in place of the one being installed now.
if (fs::exists(dir_in_vendors))
fs::remove_all(dir_in_vendors);
if (! installed_cache && fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) {
fs::create_directories(dir_in_vendors);
// Copy with file filter (same as PresetUpdater::install_bundles_rsrc)
@@ -1787,11 +1915,11 @@ bool install_vendor_bundles_from_resources(
} catch (const std::exception& e) {
BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what();
return false;
all_installed = false;
}
}
return true;
return all_installed;
}
void save_string_file(const boost::filesystem::path& p, const std::string& str)