diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md index 8d0b1349ef..fc19fc81f5 100644 --- a/docs/HLSD/preset-cache.md +++ b/docs/HLSD/preset-cache.md @@ -28,8 +28,8 @@ Per-vendor granularity is what makes the system practical: - A vendor whose profile is bumped invalidates only its own cache. The other 60-odd vendors keep theirs — even when the bumped vendor is the shared Orca filament library everyone else inherits from. -- The setup wizard, which loads vendors one at a time, gets the same speedup as - startup without a second code path. +- The setup wizard loads its vendors through the same routine as startup, so it + gets the same speedup without a second code path. - A vendor with no cache, or a broken one, costs only that vendor a parse. A cache holds *system* presets only. User presets, project settings and modified @@ -171,6 +171,9 @@ flowchart LR lib["1 · OrcaFilamentLibrary
loaded first, synchronously"] --> par["2 · every other vendor in parallel,
each into its own bundle, filaments
resolving against the loaded library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] ``` +`PresetBundle::load_vendors` runs these steps for startup and for the setup wizard, +which hand it the vendors to load and the directory each is installed in. + Whether a vendor comes from its cache or from a parse changes nothing in that order — both produce the same bundle, so cached and parsed vendors mix freely in one startup. @@ -265,17 +268,19 @@ the wizard caches the *derived JSON*, not another form of the inputs: open, the wizard computes the current stamps (one version peek per vendor) and, when they match, serves the catalog from the file — no bundle built, no preset installed. Caching bundle inputs instead was tried and measured: rebuilding the bundle from -per-vendor caches costs ~2 s of preset installation whatever feeds it, so only -skipping the rebuild entirely wins. +per-vendor caches costs over a second of preset installation whatever feeds it, so +only skipping the rebuild entirely wins. Any change to the set — a vendor added, removed or updated, or its cache-only `.opc` replaced by a newer one — changes the stamps and retires the whole file; -the wizard then rebuilds the bundle vendor by vendor (per-vendor caches serving where -they cover) and writes the catalog back. Selections, region and per-open decorations -are applied downstream of the cache either way, so a served catalog is -indistinguishable from a rebuilt one. Nothing ships this file and the updater never -touches it; it is a locally written artifact, re-derived whenever stale, written -through a temp file and rename so half a cache is never readable. +the wizard then rebuilds the bundle with `PresetBundle::load_vendors`, the load +startup uses (per-vendor caches serving where they cover), and writes the catalog +back. When a vendor fails to load, that open falls back to the wizard's own scan of +the vendor JSONs, as when no bundle can be built, and writes nothing. Selections, +region and per-open decorations are applied downstream of the cache either way, so a +served catalog is indistinguishable from a rebuilt one. Nothing ships this file and +the updater never touches it; it is a locally written artifact, re-derived whenever +stale, written through a temp file and rename so half a cache is never readable. The cache lives under `/cache/`, not beside the vendors: everything that scans `/system/` treats any `.opc` there as a vendor, so a non-vendor diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index a6112dbf48..dfb6820dd6 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -2502,35 +2502,50 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); - const auto load_t0 = std::chrono::steady_clock::now(); - // The vendors below are loaded whole and against each other — the filament // library first, then every other vendor with it as the base — so each parse // is complete enough to be worth caching. m_generate_vendor_caches = allow_cache && (m_generate_vendor_caches || !validation_mode); + // Sorted, so any duplicate-preset warning comes out in the same order on every run. + std::vector vendors; + for (const std::string& name : vendor_names_in(dir)) + if (name == ORCA_FILAMENT_LIBRARY || !(validation_mode && !vendor_to_validate.empty() && name != vendor_to_validate)) + vendors.push_back({ name, dir }); + auto result = this->load_vendors(vendors, compatibility_rule, allow_cache); + + this->update_system_maps(); + + //BBS: add config related logs + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%result.second; + return result; +} + +std::pair PresetBundle::load_vendors(const std::vector& vendors, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, const std::atomic* cancel) +{ + const auto load_t0 = std::chrono::steady_clock::now(); + auto canceled = [cancel] { return cancel != nullptr && cancel->load(); }; + PresetsConfigSubstitutions substitutions; std::string errors_cummulative; bool first = true; - // Sorted, so any duplicate-preset warning below comes out in the same order on - // every run. - const std::set vendor_names = vendor_names_in(dir); // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded // first because other vendors' filaments may inherit from it via the - // `base_bundle` lookup in parse_subfile. The remaining vendors are + // `base_bundle` lookup in install_vendor. The remaining vendors are // independent (no cross-vendor inheritance) and can be loaded in parallel. - std::string orca_lib_vendor; - std::vector other_vendors; - other_vendors.reserve(vendor_names.size()); - for (auto& vn : vendor_names) { - if (vn == ORCA_FILAMENT_LIBRARY) - orca_lib_vendor = vn; - else if (!(validation_mode && !vendor_to_validate.empty() && vn != vendor_to_validate)) - other_vendors.push_back(vn); + const VendorSource* orca_lib = nullptr; + std::vector other_vendors; + other_vendors.reserve(vendors.size()); + for (const VendorSource& vendor : vendors) { + if (vendor.name == ORCA_FILAMENT_LIBRARY) + orca_lib = &vendor; + else + other_vendors.push_back(&vendor); } // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (! orca_lib_vendor.empty()) { + if (orca_lib != nullptr && ! canceled()) { try { // Match a fresh launch before parsing: hold aliases and the error // counter survive reset(), and would otherwise carry prior-cycle @@ -2538,7 +2553,7 @@ std::pair PresetBundle::load_system_pre this->clear_printer_hold_aliases(); this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json( - dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule, nullptr, allow_cache).first); + orca_lib->dir.string(), orca_lib->name, PresetBundle::LoadSystem, compatibility_rule, nullptr, allow_cache).first); first = false; } catch (const std::runtime_error &err) { if (validation_mode) @@ -2563,13 +2578,14 @@ std::pair PresetBundle::load_system_pre // goes first, the cache or the .json that names every preset to parse. std::vector> vendor_costs(other_vendors.size()); for (size_t i = 0; i < other_vendors.size(); ++i) { + const VendorSource& vendor = *other_vendors[i]; boost::system::error_code ec; - const uintmax_t cache_size = boost::filesystem::file_size(dir / (other_vendors[i] + ".opc"), ec); + const uintmax_t cache_size = boost::filesystem::file_size(vendor.dir / (vendor.name + ".opc"), ec); if (! ec) { vendor_costs[i] = { false, cache_size }; continue; } - const uintmax_t index_size = boost::filesystem::file_size(dir / (other_vendors[i] + ".json"), ec); + const uintmax_t index_size = boost::filesystem::file_size(vendor.dir / (vendor.name + ".json"), ec); vendor_costs[i] = { true, ec ? 0 : index_size }; } std::stable_sort(by_cost.begin(), by_cost.end(), @@ -2581,12 +2597,15 @@ std::pair PresetBundle::load_system_pre [&](const tbb::blocked_range& range) { for (size_t k = range.begin(); k < range.end(); ++k) { const size_t i = by_cost[k]; + if (canceled()) + continue; + const VendorSource& vendor = *other_vendors[i]; auto bundle = std::make_unique(); bundle->set_is_validation_mode(validation_mode); bundle->set_generate_vendor_caches(m_generate_vendor_caches); try { auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this, allow_cache); + vendor.dir.string(), vendor.name, PresetBundle::LoadSystem, compatibility_rule, this, allow_cache); parallel_substitutions[i] = std::move(result.first); parallel_bundles[i] = std::move(bundle); } catch (const std::runtime_error &err) { @@ -2609,7 +2628,7 @@ std::pair PresetBundle::load_system_pre if (!parallel_bundles[i]) continue; - const std::string& vendor_name = other_vendors[i]; + const std::string& vendor_name = other_vendors[i]->name; append(substitutions, std::move(parallel_substitutions[i])); std::vector duplicates = this->merge_presets(std::move(*parallel_bundles[i])); first = false; @@ -2630,14 +2649,9 @@ std::pair PresetBundle::load_system_pre this->reset(false); } - this->update_system_maps(); - const auto load_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - load_t0).count(); - BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; - - //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendors.size() << " vendor(s) loaded in " << load_ms << " ms"; return std::make_pair(std::move(substitutions), errors_cummulative); } diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 8f3ca8115c..d84361bb4b 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -622,12 +623,26 @@ public: // default_filament_profile must resolve to a system filament. bool check_printer_default_materials() const; - // Merge one vendor's presets with the other vendor's presets, report duplicates. - // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a - // bundle out of several per-vendor caches loaded into separate PresetBundle instances. - std::vector merge_presets(PresetBundle &&other); + // One vendor to load, and the directory it is installed in. + struct VendorSource + { + std::string name; + boost::filesystem::path dir; + }; + + // Load `vendors` into this bundle, the Orca filament library first and alone, then + // every other vendor in parallel into a bundle of its own with this one to + // inherit from, merged in the order given. A vendor that cannot be loaded is left + // out and its error added to the returned text, or thrown in validation mode. + // Once `cancel` is set, no further vendor starts loading. + std::pair load_vendors(const std::vector& vendors, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, + const std::atomic* cancel = nullptr); private: + // Merge one vendor's presets with the other vendor's presets, report duplicates. + std::vector merge_presets(PresetBundle &&other); + // Load one vendor from the preset cache installed in `dir`, judged against // the vendor profile there. False, with this bundle left clean, when there // is no usable cache and the vendor has to be parsed. This is how diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index 9cea8bec28..d4fc30f9f2 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "MainFrame.hpp" @@ -1405,23 +1406,26 @@ bool GuideFrame::BuildProfileDataFromVendors() // is served from the shipped profiles. Each is stamped by name and // version alone: a profile change requires a version bump, so those two // determine content wherever the vendor's copy sits. - struct VendorSource { std::string name; boost::filesystem::path dir; std::string version; }; - std::vector ordered; - auto add_vendor = [&ordered](const std::string& name, const boost::filesystem::path& dir) { + std::vector ordered; + json stamps = json::array(); + auto add_vendor = [&ordered, &stamps](const std::string& name, const boost::filesystem::path& dir) { // The version a load from `dir` would serve: the profile's where one // exists (a cache is only served while it covers the profile beside // it), the cache's own stamp where the cache is the whole vendor. // A profile without a version (blacklist.json) carries no presets // and is passed over. const boost::filesystem::path profile = dir / (name + ".json"); + std::string version; if (boost::filesystem::exists(profile)) { const Semver v = get_version_from_json(profile.string()); - if (v.valid()) - ordered.push_back({name, dir, v.to_string()}); + if (! v.valid()) + return; + version = v.to_string(); } else { - ordered.push_back({name, dir, - VendorCacheFile::peek_version((dir / (name + ".opc")).string(), name)}); + version = VendorCacheFile::peek_version((dir / (name + ".opc")).string(), name); } + ordered.push_back({name, dir}); + stamps.push_back({name, version}); }; const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); if (auto it = vendor_sources.find(filament_library); it != vendor_sources.end()) @@ -1431,9 +1435,6 @@ bool GuideFrame::BuildProfileDataFromVendors() add_vendor(name, dir); if (ordered.empty()) return false; - json stamps = json::array(); - for (const VendorSource& v : ordered) - stamps.push_back({v.name, v.version}); // What this function derives is a pure function of that stamped set, so // the derived JSON is cached whole: a fresh cache makes an open one @@ -1461,26 +1462,18 @@ bool GuideFrame::BuildProfileDataFromVendors() } // Each vendor comes from its preset cache where one covers it, which is - // what makes this worth doing instead of the scan below; loading into a - // bundle per vendor keeps the install order the startup path has. + // what makes this worth doing instead of the scan below. PresetBundle bundle; - auto load_vendor = [](PresetBundle& into, const std::string& vendor, - const boost::filesystem::path& dir, const PresetBundle* base) { - into.load_vendor_configs_from_json(dir.string(), vendor, PresetBundle::LoadSystem, - ForwardCompatibilitySubstitutionRule::EnableSilent, base); - }; - for (const VendorSource& v : ordered) { - if (*m_cancel_token) - return false; // as in the scan below: a vendor without a cache is parsed, and that takes time - if (v.name == filament_library) { - load_vendor(bundle, v.name, v.dir, nullptr); - } else { - PresetBundle tmp; - load_vendor(tmp, v.name, v.dir, &bundle); - bundle.merge_presets(std::move(tmp)); - } - } - if (bundle.vendors.empty()) + const std::string errors = bundle.load_vendors(ordered, ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/true, m_cancel_token.get()).second; + if (*m_cancel_token || bundle.vendors.empty()) + return false; + if (! errors.empty()) + BOOST_LOG_TRIVIAL(warning) << "GuideFrame: loading the vendors reported: " << errors; + // A vendor that failed to load sends this open to the scan below, which lists + // what it can read of every vendor. + if (! std::all_of(ordered.begin(), ordered.end(), + [&bundle](const PresetBundle::VendorSource& v) { return bundle.vendors.count(v.name) != 0; })) return false; if (! BuildProfileJson(bundle, /*require_all_resource_vendors=*/false)) return false; diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp index 54242d06fd..fcc3c57ba2 100644 --- a/tests/libslic3r/test_vendor_cache.cpp +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -808,6 +809,51 @@ TEST_CASE("a key misplaced into a vendor preset is reported and removed", "[Vend CHECK(preset->config.opt_int("wall_loops") == 5); } +TEST_CASE("each vendor loads from the directory it is listed with", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.profiles, "Zeta", "2.0.0"); + PresetBundle bundle; + bundle.load_vendors({ { "Acme", dirs.system }, { "Zeta", dirs.profiles } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/false); + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Zeta") == 1); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(bundle.prints.find_preset("0.20mm Standard @Zeta", false) != nullptr); +} + +TEST_CASE("a vendor that fails to load is left out, reported, and the others still load", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_process_vendor(dirs.system, "Broken", { { "Broken A", "{not-json" } }); + PresetBundle bundle; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { "Broken", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false).second; + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Broken") == 0); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(errors.find("Broken") != std::string::npos); +} + +TEST_CASE("a canceled vendor load starts no vendor", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.system, "Zeta", "1.0.0"); + const std::atomic cancel { true }; + PresetBundle bundle; + bundle.load_vendors({ { "Acme", dirs.system }, { "Zeta", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/false, &cancel); + + CHECK(bundle.vendors.empty()); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) == nullptr); +} + TEST_CASE("repeated cache loads of one vendor produce the same presets", "[VendorCache]") { InstallDirs dirs;