diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md index fc19fc81f5..c774910637 100644 --- a/docs/HLSD/preset-cache.md +++ b/docs/HLSD/preset-cache.md @@ -162,13 +162,15 @@ cache nothing can invalidate is worse than no cache. Vendors load in a fixed order, because filament inheritance crosses exactly one boundary: any vendor's filament may inherit from the shared Orca filament library, -and nothing else reaches across vendors — an `include` is always vendor-local. The -library therefore goes first, alone; every other vendor follows in parallel, resolving -against it; and the results are merged in a stable order: +and nothing else reaches across vendors — an `include` is always vendor-local. Only +installing a vendor's presets crosses it; reading the vendor, from its cache or its +JSONs, needs nothing from the library. So every other vendor is read while the library +loads, each is installed against it as soon as both are done, and the results are +merged in a stable order: ```mermaid flowchart LR - lib["1 · OrcaFilamentLibrary
loaded first, synchronously"] --> par["2 · every other vendor in parallel,
each into its own bundle, filaments
resolving against the loaded library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] + lib["1 · OrcaFilamentLibrary loaded;
meanwhile every other vendor read
from its cache or its JSONs"] --> par["2 · every other vendor installed
in parallel, each into its own bundle,
filaments resolving against the library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] ``` `PresetBundle::load_vendors` runs these steps for startup and for the setup wizard, @@ -275,12 +277,13 @@ Any change to the set — a vendor added, removed or updated, or its cache-only `.opc` replaced by a newer one — changes the stamps and retires the whole file; the wizard then rebuilds the bundle with `PresetBundle::load_vendors`, the load startup uses (per-vendor caches serving where they cover), and writes the catalog -back. When a vendor fails to load, that open falls back to the wizard's own scan of -the vendor JSONs, as when no bundle can be built, and writes nothing. Selections, -region and per-open decorations are applied downstream of the cache either way, so a -served catalog is indistinguishable from a rebuilt one. Nothing ships this file and -the updater never touches it; it is a locally written artifact, re-derived whenever -stale, written through a temp file and rename so half a cache is never readable. +back. When a vendor fails to load, the filament library included, that open falls +back to the wizard's own scan of the vendor JSONs, as when no bundle can be built, +and writes nothing. Selections, region and per-open decorations are applied +downstream of the cache either way, so a served catalog is indistinguishable from a +rebuilt one. Nothing ships this file and the updater never touches it; it is a +locally written artifact, re-derived whenever stale, written through a temp file and +rename so half a cache is never readable. The cache lives under `/cache/`, not beside the vendors: everything that scans `/system/` treats any `.opc` there as a vendor, so a non-vendor diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index 4d71f38bc3..b8035618ad 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include // Mark string for localization and translate. @@ -2522,18 +2523,17 @@ std::pair PresetBundle::load_system_pre } std::pair PresetBundle::load_vendors(const std::vector& vendors, - ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, const std::atomic* cancel) + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, const std::atomic* cancel, + std::vector* failed) { - const auto load_t0 = std::chrono::steady_clock::now(); - auto canceled = [cancel] { return cancel != nullptr && cancel->load(); }; + const auto load_t0 = std::chrono::steady_clock::now(); + auto canceled = [cancel] { return cancel != nullptr && cancel->load(); }; PresetsConfigSubstitutions substitutions; std::string errors_cummulative; bool first = true; - // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded - // first because other vendors' filaments may inherit from it via the - // `base_bundle` lookup in install_vendor. The remaining vendors are - // independent (no cross-vendor inheritance) and can be loaded in parallel. + // Other vendors inherit from nothing but the library, and only installing them + // looks it up, so each is read while it loads and installed once both are done. const VendorSource* orca_lib = nullptr; std::vector other_vendors; other_vendors.reserve(vendors.size()); @@ -2544,7 +2544,87 @@ std::pair PresetBundle::load_vendors(co other_vendors.push_back(&vendor); } - // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. + // One of the other vendors, loaded into a PresetBundle of its own. + struct VendorLoad + { + const VendorSource* source { nullptr }; + std::unique_ptr bundle; + VendorRead read; + PresetsConfigSubstitutions substitutions; + std::string error; + // Counts the vendor's read and the library load; the second to finish installs it. + std::atomic ready { 0 }; + }; + std::vector loads(other_vendors.size()); + for (size_t i = 0; i < other_vendors.size(); ++i) + loads[i].source = other_vendors[i]; + + // Started slowest first, since the load ends when the slowest vendor does. + std::vector by_cost(loads.size()); + std::iota(by_cost.begin(), by_cost.end(), size_t(0)); + // A parse from JSON costs far more than any cache load, so parses go first, and + // within each group the bigger file, the cache or the .json that lists what + // to parse. A cache older than its profile is ordered as a parse. + const bool reads_caches = allow_cache && ! validation_mode; + std::vector> vendor_costs(loads.size()); + for (size_t i = 0; i < loads.size(); ++i) { + const VendorSource& vendor = *loads[i].source; + const boost::filesystem::path cache = vendor.dir / (vendor.name + ".opc"); + const boost::filesystem::path profile = vendor.dir / (vendor.name + ".json"); + boost::system::error_code ec, profile_ec; + const uintmax_t cache_size = reads_caches ? boost::filesystem::file_size(cache, ec) : 0; + if (reads_caches && ! ec) { + const std::time_t profile_time = boost::filesystem::last_write_time(profile, profile_ec); + if (profile_ec || profile_time <= boost::filesystem::last_write_time(cache, ec)) { + vendor_costs[i] = { false, cache_size }; + continue; + } + } + const uintmax_t profile_size = boost::filesystem::file_size(profile, profile_ec); + vendor_costs[i] = { true, profile_ec ? 0 : profile_size }; + } + std::stable_sort(by_cost.begin(), by_cost.end(), + [&](size_t a, size_t b) { return vendor_costs[a] > vendor_costs[b]; }); + + // A vendor that failed to read, or that is still to install when `cancel` is + // set, is left out. + auto install = [&](VendorLoad& load) { + if (load.bundle && ! canceled()) { + try { + load.substitutions = load.bundle->install_vendor_read(std::move(load.read), this).first; + } catch (const std::runtime_error &err) { + load.error = err.what(); + load.bundle.reset(); + } + } else + load.bundle.reset(); + load.read = VendorRead(); + }; + tbb::task_group group; + group.run([&] { + // An auto_partitioner splits the range only while a worker is asking for work, + // which at this size left every vendor on the calling thread. + tbb::parallel_for(tbb::blocked_range(0, loads.size(), 1), + [&](const tbb::blocked_range& range) { + for (size_t k = range.begin(); k < range.end(); ++k) { + VendorLoad& load = loads[by_cost[k]]; + if (! canceled()) { + try { + auto bundle = std::make_unique(); + bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); + load.read = bundle->read_vendor(load.source->dir.string(), load.source->name, + PresetBundle::LoadSystem, compatibility_rule, allow_cache); + load.bundle = std::move(bundle); + } catch (const std::runtime_error &err) { + load.error = err.what(); + } + } + if (++ load.ready == 2) + install(load); + } + }, tbb::simple_partitioner()); + }); if (orca_lib != nullptr && ! canceled()) { try { // Match a fresh launch before parsing: hold aliases and the error @@ -2560,77 +2640,33 @@ std::pair PresetBundle::load_vendors(co throw err; errors_cummulative += err.what(); errors_cummulative += "\n"; + if (failed != nullptr) + failed->push_back(orca_lib->name); } } + for (size_t k : by_cost) + if (++ loads[k].ready == 2) + group.run([&install, &load = loads[k]] { install(load); }); + group.wait(); - // Step 2: Load remaining vendors in parallel. Each gets its own - // PresetBundle and uses `this` (which contains ORCA_FILAMENT_LIBRARY) - // as the base_bundle for cross-bundle inheritance lookups. - std::vector> parallel_bundles(other_vendors.size()); - std::vector parallel_substitutions(other_vendors.size()); - std::vector parallel_errors(other_vendors.size()); - - // Started slowest first, since this step ends when the slowest vendor does. - std::vector by_cost(other_vendors.size()); - std::iota(by_cost.begin(), by_cost.end(), size_t(0)); - // A vendor with no cache is parsed from its JSONs, which costs far more than - // any cache load, so those come first, and within each group the bigger file - // goes first, the cache or the .json that names every preset to parse. - std::vector> vendor_costs(other_vendors.size()); - for (size_t i = 0; i < other_vendors.size(); ++i) { - const VendorSource& vendor = *other_vendors[i]; - boost::system::error_code ec; - const uintmax_t cache_size = boost::filesystem::file_size(vendor.dir / (vendor.name + ".opc"), ec); - if (! ec) { - vendor_costs[i] = { false, cache_size }; - continue; - } - const uintmax_t index_size = boost::filesystem::file_size(vendor.dir / (vendor.name + ".json"), ec); - vendor_costs[i] = { true, ec ? 0 : index_size }; - } - std::stable_sort(by_cost.begin(), by_cost.end(), - [&](size_t a, size_t b) { return vendor_costs[a] > vendor_costs[b]; }); - - // An auto_partitioner splits the range only while a worker is asking for work, - // which at this size left every vendor on the calling thread. - tbb::parallel_for(tbb::blocked_range(0, other_vendors.size(), 1), - [&](const tbb::blocked_range& range) { - for (size_t k = range.begin(); k < range.end(); ++k) { - const size_t i = by_cost[k]; - if (canceled()) - continue; - const VendorSource& vendor = *other_vendors[i]; - auto bundle = std::make_unique(); - bundle->set_is_validation_mode(validation_mode); - bundle->set_generate_vendor_caches(m_generate_vendor_caches); - try { - auto result = bundle->load_vendor_configs_from_json( - vendor.dir.string(), vendor.name, PresetBundle::LoadSystem, compatibility_rule, this, allow_cache); - parallel_substitutions[i] = std::move(result.first); - parallel_bundles[i] = std::move(bundle); - } catch (const std::runtime_error &err) { - parallel_errors[i] = err.what(); - } - } - }, tbb::simple_partitioner()); - - // Step 3: Sequentially merge the parallel-loaded bundles into `this`. - // The merge order is the original vendor order so any duplicate-warning - // output stays stable across runs. - for (size_t i = 0; i < other_vendors.size(); ++i) { - if (!parallel_errors[i].empty()) { + // Merged in the original vendor order, so any duplicate-warning output stays + // stable across runs. + for (VendorLoad& load : loads) { + if (! load.error.empty()) { if (validation_mode) - throw std::runtime_error(parallel_errors[i]); - errors_cummulative += parallel_errors[i]; + throw std::runtime_error(load.error); + errors_cummulative += load.error; errors_cummulative += "\n"; + if (failed != nullptr) + failed->push_back(load.source->name); continue; } - if (!parallel_bundles[i]) + if (! load.bundle) continue; - const std::string& vendor_name = other_vendors[i]->name; - append(substitutions, std::move(parallel_substitutions[i])); - std::vector duplicates = this->merge_presets(std::move(*parallel_bundles[i])); + const std::string& vendor_name = load.source->name; + append(substitutions, std::move(load.substitutions)); + std::vector duplicates = this->merge_presets(std::move(*load.bundle)); first = false; if (!duplicates.empty()) { errors_cummulative += "Found duplicated settings in vendor " + vendor_name + "'s json file lists: "; @@ -6994,12 +7030,14 @@ std::pair PresetBundle::load_vendor_configs_ const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle, bool allow_cache) { - // Enable substitutions for user config bundle, throw an exception when loading a system profile. - ConfigSubstitutionContext substitution_context { compatibility_rule }; - PresetsConfigSubstitutions substitutions; - // Errors already on this bundle when the load began; the cache stamp below - // counts only what this parse adds. - const int errors_at_entry = m_errors; + return this->install_vendor_read(this->read_vendor(dir, vendor_name, flags, compatibility_rule, allow_cache), base_bundle); +} + +PresetBundle::VendorRead PresetBundle::read_vendor(const std::string& dir, const std::string& vendor_name, + LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache) +{ + VendorRead read { dir, vendor_name, flags, compatibility_rule }; + read.errors_at_entry = m_errors; //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; @@ -7009,16 +7047,34 @@ std::pair PresetBundle::load_vendor_configs_ // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only // scans want a slice of one. Validation reads the JSONs whatever is cached. - const boost::filesystem::path dir_path(dir); - const bool cacheable = allow_cache && flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); - if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name, base_bundle)) { - size_t presets_loaded = 0; - for (const PresetCollection* coll : std::initializer_list{ - &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) - presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; - return std::make_pair(std::move(substitutions), presets_loaded); + read.cacheable = allow_cache && flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (read.cacheable && ! validation_mode) { + // A vendor is loaded from where it is installed and nowhere else; resources + // reaches the app by being installed into `dir` first. The cache there is + // judged against the profile beside it — or, where the cache is the whole + // of the installation, against nothing, since nothing on disk can then be + // newer than it. That state is Semver::inf(), which no real profile carries. + const boost::filesystem::path dir_path(dir); + const boost::filesystem::path profile = dir_path / (vendor_name + ".json"); + const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) : Semver::inf(); + read.cache_path = (dir_path / (vendor_name + ".opc")).string(); + read.from_cache = VendorCacheFile::load(read.cache_path, vendor_name, version, read.data); + if (read.from_cache) + return read; } + this->parse_vendor_json(read); + return read; +} + +void PresetBundle::parse_vendor_json(VendorRead& read) +{ + // Starts over from what read_vendor was asked for, since a cache that could + // not be installed leaves its own state behind. + read = VendorRead { std::move(read.dir), std::move(read.vendor_name), read.flags, read.compatibility_rule, + read.errors_at_entry, read.cacheable }; + const std::string& dir = read.dir; + const std::string& vendor_name = read.vendor_name; + LoadConfigBundleAttributes flags = read.flags; // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); @@ -7239,11 +7295,12 @@ std::pair PresetBundle::load_vendor_configs_ BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", loaded vendor profile, name %1%, id %2%, version %3%")%vendor_profile.name%vendor_profile.id%vendor_profile.config_version.to_string(); - if (flags.has(LoadConfigBundleAttribute::LoadVendorOnly)) - return std::make_pair(PresetsConfigSubstitutions{}, 0); + if (flags.has(LoadConfigBundleAttribute::LoadVendorOnly)) { + read.vendor_only = true; + return; + } // 3) paste the process/filament/print configs - size_t presets_loaded = 0; // Parse one subfile into a source-form entry — everything the JSON states, // nothing resolved. Installing the entries (install_vendor) is the @@ -7332,26 +7389,23 @@ std::pair PresetBundle::load_vendor_configs_ // and stamped with the version that profile claims — a profile without one // cannot be judged for staleness later, and a cache nothing can invalidate is // worse than none. - const bool will_cache = cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); - VendorCacheData cache_data; + read.will_cache = read.cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); + read.version = vendor_profile.config_version.to_string(); + // Enable substitutions for user config bundle, throw an exception when loading a system profile. + ConfigSubstitutionContext substitution_context { read.compatibility_rule }; // Parsed up to the first sub-file that fails, and the ones before it are // installed before that failure is raised, since the filament library and a // filament-only scan keep what a throwing load installed. - VendorParse parsed; - std::string reason; - std::string failed_subfile; - const char* failed_kind = nullptr; - std::vector failed_errors; auto parse_subfiles = [&](const std::vector>& subfiles, std::vector& entries, std::vector& entries_parsed, const char* kind) { for (const auto& subfile : subfiles) { CachedPreset entry; std::vector errors, warnings; - reason = parse_subfile(substitution_context, subfile, entry, errors, warnings); - if (! reason.empty()) { - failed_subfile = subfile.second; - failed_kind = kind; - failed_errors = std::move(errors); + read.reason = parse_subfile(substitution_context, subfile, entry, errors, warnings); + if (! read.reason.empty()) { + read.failed_subfile = subfile.second; + read.failed_kind = kind; + read.failed_errors = std::move(errors); return false; } entries.emplace_back(std::move(entry)); @@ -7359,40 +7413,60 @@ std::pair PresetBundle::load_vendor_configs_ } return true; }; - { - // One setter for every sub-file, so the ones load_from_json makes have - // nothing to set. - CNumericLocalesSetter locales_setter; - if (parse_subfiles(process_subfiles, cache_data.process_entries, parsed.process_entries, "process") && - parse_subfiles(filament_subfiles, cache_data.filament_entries, parsed.filament_entries, "filament")) - parse_subfiles(machine_subfiles, cache_data.machine_entries, parsed.machine_entries, "printer"); + // One setter for every sub-file, so the ones load_from_json makes have + // nothing to set. + CNumericLocalesSetter locales_setter; + if (parse_subfiles(process_subfiles, read.data.process_entries, read.parsed.process_entries, "process") && + parse_subfiles(filament_subfiles, read.data.filament_entries, read.parsed.filament_entries, "filament")) + parse_subfiles(machine_subfiles, read.data.machine_entries, read.parsed.machine_entries, "printer"); +} + +std::pair PresetBundle::install_vendor_read(VendorRead&& read, const PresetBundle* base_bundle) +{ + const std::string& dir = read.dir; + const std::string& vendor_name = read.vendor_name; + PresetsConfigSubstitutions substitutions; + if (read.from_cache) { + if (this->install_vendor_cache(read.cache_path, vendor_name, std::move(read.data), base_bundle)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + this->parse_vendor_json(read); } + if (read.vendor_only) + return std::make_pair(PresetsConfigSubstitutions{}, 0); + int parse_errors = 0; - for (const std::vector* list : { &parsed.process_entries, &parsed.filament_entries, &parsed.machine_entries }) + for (const std::vector* list : { &read.parsed.process_entries, &read.parsed.filament_entries, &read.parsed.machine_entries }) for (const EntryParse& entry_parsed : *list) parse_errors += int(entry_parsed.errors.size()); - const int errors_before_install = m_errors; - presets_loaded = install_vendor(dir, vendor_name, base_bundle, flags, cache_data, &parsed, reason.empty(), substitutions); + const int errors_before_install = m_errors; + const size_t presets_loaded = install_vendor(dir, vendor_name, base_bundle, read.flags, read.data, &read.parsed, + read.reason.empty(), substitutions); // Errors added by install are counted apart: a cache load runs install again, // so the parse_errors stamped into the cache must hold only what a cache load // will not recount. const int install_errors = m_errors - errors_before_install - parse_errors; - if (! reason.empty()) { - log_errors(failed_errors); + if (! read.reason.empty()) { + log_errors(read.failed_errors); ++m_errors; //parse error - std::string subfile_path = dir + "/" + vendor_name + "/" + failed_subfile; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % failed_kind % subfile_path; + std::string subfile_path = dir + "/" + vendor_name + "/" + read.failed_subfile; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % read.failed_kind % subfile_path; throw failed_loading_error(subfile_path, dir); } - if (will_cache) { + if (read.will_cache) { // Clamped: the count is a difference of three tallies, and a stamp that // wrapped would be added to every future load of this vendor. - cache_data.parse_errors = uint64_t(std::max(0, m_errors - errors_at_entry - install_errors)); - cache_data.vendors = this->vendors; - if (! VendorCacheFile::save((dir_path / (vendor_name + ".opc")).string(), vendor_name, - vendor_profile.config_version.to_string(), cache_data)) + read.data.parse_errors = uint64_t(std::max(0, m_errors - read.errors_at_entry - install_errors)); + read.data.vendors = this->vendors; + if (! VendorCacheFile::save((boost::filesystem::path(dir) / (vendor_name + ".opc")).string(), vendor_name, + read.version, read.data)) BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; } @@ -8107,33 +8181,26 @@ bool BundleMetadata::save_to_json(const std::string& path) const // ---- Per-vendor preset cache: install into this bundle ------------------- // The file format itself lives in PresetCacheFormat.cpp (VendorCacheFile). -bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle) -{ - // A vendor is loaded from where it is installed and nowhere else; resources - // reaches the app by being installed into `dir` first. The cache there is - // judged against the profile beside it — or, where the cache is the whole - // of the installation, against nothing, since nothing on disk can then be - // newer than it. That state is Semver::inf(), which no real profile carries. - const boost::filesystem::path profile = dir / (vendor_name + ".json"); - const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) - : Semver::inf(); - return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, base_bundle); -} - bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, const Semver& expected_vendor_version, const PresetBundle* base_bundle) { - // What this bundle had counted before the cache was tried. The caller - // measures its own parse against this same baseline, so a rejection must - // put it back rather than reset it to zero. - const int errors_at_entry = this->m_errors; // Read and validated before this bundle is touched: a rejected file leaves // no state to roll back. VendorCacheData data; if (! VendorCacheFile::load(cache_path, expected_vendor_name, expected_vendor_version, data)) return false; + // VendorCacheFile::load checked the names match. + return this->install_vendor_cache(cache_path, expected_vendor_name, std::move(data), base_bundle); +} + +bool PresetBundle::install_vendor_cache(const std::string& cache_path, const std::string& vendor_name, VendorCacheData&& data, + const PresetBundle* base_bundle) +{ + // What this bundle had counted before the cache was tried. The caller + // measures its own parse against this same baseline, so a rejection must + // put it back rather than reset it to zero. + const int errors_at_entry = this->m_errors; try { - const std::string& vendor_name = expected_vendor_name; // VendorCacheFile::load checked they match this->vendors = std::move(data.vendors); // What the parse counted before install took over; install recounts its @@ -8146,6 +8213,10 @@ bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::s LoadConfigBundleAttribute::LoadSystem, data, nullptr, true, substitutions); return true; } catch (const std::exception& e) { + // A cancellation of the caller's task group stops the install without + // anything being wrong with the cache. + if (tbb::is_current_task_group_canceling()) + throw; BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); // Restore a clean state so the caller can fall back to the JSON parse. this->reset(false); diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 1f73d04787..403cb89e6c 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -630,25 +630,20 @@ public: boost::filesystem::path dir; }; - // Load `vendors` into this bundle, the Orca filament library first and alone, then - // every other vendor in parallel into a bundle of its own with this one to - // inherit from, merged in the order given. A vendor that cannot be loaded is left - // out and its error added to the returned text, or thrown in validation mode. - // Once `cancel` is set, no further vendor starts loading. + // Load `vendors` into this bundle, the Orca filament library directly and every + // other vendor in parallel into a bundle of its own that inherits from it, merged + // in the order given. A vendor that cannot be loaded has its error added to the + // returned text, or thrown in validation mode, and its name to `failed`; it is + // left out, except for the library, which keeps what it installed before the + // failure. Once `cancel` is set, no further vendor starts loading. std::pair load_vendors(const std::vector& vendors, ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, - const std::atomic* cancel = nullptr); + const std::atomic* cancel = nullptr, std::vector* failed = nullptr); private: // Merge one vendor's presets with the other vendor's presets, report duplicates. std::vector merge_presets(PresetBundle &&other); - // Load one vendor from the preset cache installed in `dir`, judged against - // the vendor profile there. False, with this bundle left clean, when there - // is no usable cache and the vendor has to be parsed. This is how - // load_vendor_configs_from_json reads a cache. - bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle); - // What parsing one entry's JSON sub-file reported. Its errors and warnings are // logged when the entry installs, so they come out in listing order with the // entry's install errors, as parsing and installing one entry at a time leaves them. @@ -668,6 +663,56 @@ private: std::vector machine_entries; }; + // One vendor read from its cache or its JSONs by read_vendor, for + // install_vendor_read to install. + struct VendorRead + { + std::string dir; + std::string vendor_name; + LoadConfigBundleAttributes flags; + ForwardCompatibilitySubstitutionRule compatibility_rule; + // The errors this bundle had counted before the read, which the cache stamp leaves out. + int errors_at_entry { 0 }; + // A whole-vendor load, which can read a cache and write one. + bool cacheable { false }; + // Read from the cache at cache_path, which install can still reject. + bool from_cache { false }; + std::string cache_path; + // Only the vendor profile was asked for. + bool vendor_only { false }; + VendorCacheData data; + // What each entry's JSON parse reported, and whether and with which version + // the cache is written once the entries install. + VendorParse parsed; + bool will_cache { false }; + std::string version; + // The sub-file the parse stopped at, its kind, why, and the errors it reported. + std::string reason; + std::string failed_subfile; + const char* failed_kind { nullptr }; + std::vector failed_errors; + }; + + // Read a vendor into this bundle's vendor profiles and `data`, from its cache + // when one covers it, else from its JSONs; nothing is installed. Throws + // ConfigurationError when the vendor's own JSON cannot be parsed. + VendorRead read_vendor(const std::string& dir, const std::string& vendor_name, LoadConfigBundleAttributes flags, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache); + + // Parse the vendor's JSONs into `read`, up to the first sub-file that fails. + void parse_vendor_json(VendorRead& read); + + // Install what read_vendor read, against base_bundle's filament library. A cache + // that cannot be installed is replaced by a parse of the JSONs. Throws + // ConfigurationError at the first entry that cannot be installed, or after + // installing the entries before a sub-file that could not be parsed. + std::pair install_vendor_read(VendorRead&& read, const PresetBundle* base_bundle); + + // Install a cache's entries. False, with this bundle left clean, when one of them + // cannot be installed. + bool install_vendor_cache(const std::string& cache_path, const std::string& vendor_name, VendorCacheData&& data, + const PresetBundle* base_bundle); + // Log and count errors reported by a resolve or a parse. void log_errors(const std::vector& errors); diff --git a/src/libslic3r/PresetCacheFormat.hpp b/src/libslic3r/PresetCacheFormat.hpp index 2c90c07882..52c40bea21 100644 --- a/src/libslic3r/PresetCacheFormat.hpp +++ b/src/libslic3r/PresetCacheFormat.hpp @@ -109,7 +109,7 @@ void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict); // One preset as its JSON subfile states it: the config diff, the names of the // preset it inherits and the presets it includes, and the parse metadata — -// everything the parse phase of load_vendor_configs_from_json extracts and +// everything PresetBundle::parse_vendor_json extracts and // nothing it derives. Inheritance and includes are resolved when the entry is // installed, against whatever filament library is loaded then, so a cache // carries no other vendor's values and no other vendor's update can make it diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index d4fc30f9f2..b6ba0449bd 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include "MainFrame.hpp" @@ -1463,17 +1462,17 @@ bool GuideFrame::BuildProfileDataFromVendors() // Each vendor comes from its preset cache where one covers it, which is // what makes this worth doing instead of the scan below. - PresetBundle bundle; + PresetBundle bundle; + std::vector failed; const std::string errors = bundle.load_vendors(ordered, ForwardCompatibilitySubstitutionRule::EnableSilent, - /*allow_cache=*/true, m_cancel_token.get()).second; + /*allow_cache=*/true, m_cancel_token.get(), &failed).second; if (*m_cancel_token || bundle.vendors.empty()) return false; if (! errors.empty()) BOOST_LOG_TRIVIAL(warning) << "GuideFrame: loading the vendors reported: " << errors; // A vendor that failed to load sends this open to the scan below, which lists // what it can read of every vendor. - if (! std::all_of(ordered.begin(), ordered.end(), - [&bundle](const PresetBundle::VendorSource& v) { return bundle.vendors.count(v.name) != 0; })) + if (! failed.empty()) return false; if (! BuildProfileJson(bundle, /*require_all_resource_vendors=*/false)) return false; diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp index fcc3c57ba2..4d42a5b0d4 100644 --- a/tests/libslic3r/test_vendor_cache.cpp +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -14,6 +14,10 @@ #include #include +#include +#include +#include + #include "libslic3r/PresetBundle.hpp" #include "libslic3r/PresetCacheFormat.hpp" #include "libslic3r/Preset.hpp" @@ -829,6 +833,52 @@ TEST_CASE("a vendor that fails to load is left out, reported, and the others sti InstallDirs dirs; write_vendor_tree(dirs.system, "Acme", "1.0.0"); write_process_vendor(dirs.system, "Broken", { { "Broken A", "{not-json" } }); + PresetBundle bundle; + std::vector failed; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { "Broken", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false, nullptr, &failed).second; + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Broken") == 0); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(errors.find("Broken") != std::string::npos); + CHECK(failed == std::vector{ "Broken" }); +} + +TEST_CASE("a filament library that fails partway is reported, and vendors inheriting from it are left out", "[VendorCache]") +{ + InstallDirs dirs; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(dirs.system, "1.0.0", "20"); + std::ofstream((dirs.system / (lib + ".json")).string()) + << R"({"version":"1.0.0","name":")" << lib << R"(","filament_list":[)" + << R"({"name":"Generic PLA","sub_path":"filament/generic_pla.json"},)" + << R"({"name":"Generic PETG","sub_path":"filament/generic_petg.json"}]})"; + std::ofstream((dirs.system / lib / "filament" / "generic_petg.json").string()) << "{not-json"; + write_vendor_with_lib_filament(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.system, "Zeta", "1.0.0"); + + PresetBundle bundle; + std::vector failed; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { lib, dirs.system }, { "Zeta", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false, nullptr, &failed).second; + + CHECK(failed == std::vector{ lib, "Acme" }); + CHECK(errors.find("generic_petg.json") != std::string::npos); + CHECK(bundle.vendors.count(lib) == 1); + CHECK(bundle.vendors.count("Acme") == 0); + CHECK(bundle.filaments.find_preset("Acme PLA @0.4", false) == nullptr); + CHECK(bundle.vendors.count("Zeta") == 1); + CHECK(bundle.prints.find_preset("0.20mm Standard @Zeta", false) != nullptr); +} + +TEST_CASE("a vendor whose profile cannot be read is left out, reported, and the others still load", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + std::ofstream((dirs.system / "Broken.json").string()) << "{not-json"; PresetBundle bundle; const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { "Broken", dirs.system } }, ForwardCompatibilitySubstitutionRule::EnableSilent, @@ -837,7 +887,7 @@ TEST_CASE("a vendor that fails to load is left out, reported, and the others sti CHECK(bundle.vendors.count("Acme") == 1); CHECK(bundle.vendors.count("Broken") == 0); CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); - CHECK(errors.find("Broken") != std::string::npos); + CHECK(errors.find("Broken.json") != std::string::npos); } TEST_CASE("a canceled vendor load starts no vendor", "[VendorCache]") @@ -854,6 +904,34 @@ TEST_CASE("a canceled vendor load starts no vendor", "[VendorCache]") CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) == nullptr); } +TEST_CASE("a vendor read while the filament library loads resolves against it, from JSON and from its cache", "[VendorCache]") +{ + InstallDirs dirs; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(dirs.system, "1.0.0", "20"); + write_vendor_with_lib_filament(dirs.system, "Acme", "1.0.0"); + auto load = [&] { + auto bundle = std::make_unique(); + bundle->set_generate_vendor_caches(true); + bundle->load_vendors({ { "Acme", dirs.system }, { lib, dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/true); + return bundle; + }; + + const auto from_json = load(); + REQUIRE(fs::exists(dirs.system / "Acme.opc")); + fs::remove_all(dirs.system / "Acme"); + const auto from_cache = load(); + + for (const PresetBundle* bundle : { from_json.get(), from_cache.get() }) { + const Preset* pla = bundle->filaments.find_preset("Acme PLA @0.4", false); + REQUIRE(pla != nullptr); + CHECK(pla->filament_id == "GFL99"); + CHECK_THAT(pla->config.option("filament_cost")->values.front(), WithinAbs(20., 1e-9)); + CHECK(bundle->error_count() == 0); + } +} + TEST_CASE("repeated cache loads of one vendor produce the same presets", "[VendorCache]") { InstallDirs dirs; @@ -1126,6 +1204,27 @@ TEST_CASE("a vendor installed as its cache alone still loads after a library upd CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); } +TEST_CASE("a cache install stopped by a canceled task group throws instead of rejecting the cache", "[VendorCache]") +{ + TempDir tmp; + const std::string cache = (tmp.path / "Acme.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Acme"), "Acme", "1.0.0", {filament_entry("Acme PLA @0.4")})); + + PresetBundle bundle; + bool threw = false; + tbb::task_group_context context; + tbb::parallel_for(tbb::blocked_range(0, 1), [&](const tbb::blocked_range&) { + context.cancel_group_execution(); + try { + bundle.load_vendor_cache(cache, "Acme", Semver(1, 0, 0)); + } catch (const std::runtime_error&) { + threw = true; + } + }, context); + + CHECK(threw); +} + TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSONs", "[VendorCache]") { TempDir tmp; @@ -1153,6 +1252,12 @@ TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSON user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); CHECK(presets_loaded == 1); CHECK(out.vendors.at("Acme").name == "Acme"); // the profile's name, not the cache's + + // Through load_vendors, which reads the cache before installing it. + PresetBundle several; + several.load_vendors({ { "Acme", user } }, ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/true); + CHECK(several.vendors.at("Acme").name == "Acme"); + CHECK(several.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); } TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]")