From f5679ad3433e7021b16f86d96394807d3e2a8cea Mon Sep 17 00:00:00 2001 From: Kris Austin Date: Tue, 29 Sep 2026 16:13:44 -0500 Subject: [PATCH] perf: load presets in parallel, cutting preset load time by over 60% (#15943) Co-authored-by: Rodrigo Faselli <162915171+RF47@users.noreply.github.com> --- docs/HLSD/preset-cache.md | 64 +- src/libslic3r/CMakeLists.txt | 1 + src/libslic3r/Config.cpp | 39 +- src/libslic3r/LocalesUtils.cpp | 16 + src/libslic3r/LocalesUtils.hpp | 5 + src/libslic3r/ParallelResolve.hpp | 65 ++ src/libslic3r/Preset.cpp | 391 +++++--- src/libslic3r/Preset.hpp | 54 +- src/libslic3r/PresetBundle.cpp | 947 +++++++++++------- src/libslic3r/PresetBundle.hpp | 199 +++- src/libslic3r/PresetCacheFormat.hpp | 2 +- src/slic3r/GUI/WebGuideDialog.cpp | 52 +- tests/libslic3r/CMakeLists.txt | 2 + tests/libslic3r/test_locales_utils.cpp | 69 ++ tests/libslic3r/test_parallel_resolve.cpp | 122 +++ .../libslic3r/test_preset_bundle_loading.cpp | 166 +++ tests/libslic3r/test_vendor_cache.cpp | 481 +++++++++ 17 files changed, 2038 insertions(+), 637 deletions(-) create mode 100644 src/libslic3r/ParallelResolve.hpp create mode 100644 tests/libslic3r/test_locales_utils.cpp create mode 100644 tests/libslic3r/test_parallel_resolve.cpp diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md index 666e452f1e..aa39d522a0 100644 --- a/docs/HLSD/preset-cache.md +++ b/docs/HLSD/preset-cache.md @@ -28,8 +28,8 @@ Per-vendor granularity is what makes the system practical: - A vendor whose profile is bumped invalidates only its own cache. The other 60-odd vendors keep theirs — even when the bumped vendor is the shared Orca filament library everyone else inherits from. -- The setup wizard, which loads vendors one at a time, gets the same speedup as - startup without a second code path. +- The setup wizard loads its vendors through the same routine as startup, so it + gets the same speedup without a second code path. - A vendor with no cache, or a broken one, costs only that vendor a parse. A cache holds *system* presets only. User presets, project settings and modified @@ -162,15 +162,20 @@ cache nothing can invalidate is worse than no cache. Vendors load in a fixed order, because filament inheritance crosses exactly one boundary: any vendor's filament may inherit from the shared Orca filament library, -and nothing else reaches across vendors — an `include` is always vendor-local. The -library therefore goes first, alone; every other vendor follows in parallel, resolving -against it; and the results are merged in a stable order: +and nothing else reaches across vendors — an `include` is always vendor-local. Only +installing a vendor's presets crosses it; reading the vendor, from its cache or its +JSONs, needs nothing from the library. So every other vendor is read while the library +loads, each is installed against it as soon as both are done, and the results are +merged in a stable order: ```mermaid flowchart LR - lib["1 · OrcaFilamentLibrary
loaded first, synchronously"] --> par["2 · every other vendor in parallel,
each into its own bundle, filaments
resolving against the loaded library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] + lib["1 · OrcaFilamentLibrary loaded;
meanwhile every other vendor read
from its cache or its JSONs"] --> par["2 · every other vendor installed
in parallel, each into its own bundle,
filaments resolving against the library"] --> merge["3 · bundles merged into one,
in one pass per collection,
in stable vendor order"] ``` +`PresetBundle::load_vendors` runs these steps for startup and for the setup wizard, +which hand it the vendors to load and the directory each is installed in. + Whether a vendor comes from its cache or from a parse changes nothing in that order — both produce the same bundle, so cached and parsed vendors mix freely in one startup. @@ -211,11 +216,12 @@ shipped cache answered first, so the profile in `/system/` was never p and its cache was never written back. Serving from a cache is not a memory-image restore. The entries are deserialized and -then installed one by one — inheritance resolved against the presets installed before -them and the currently loaded filament library, includes layered in, configs flattened -onto the collection defaults, validated and registered — by the same function the JSON -path calls straight after parsing a sub-file. An `include` layers what the included -base states, between the parent and the preset's own keys: the base's diff against the +then installed by `install_vendor`, the routine the JSON path hands the vendor's entries +to once it has parsed the sub-files: inheritance resolved against the presets installed +before them and the currently loaded filament library, includes layered in, configs +flattened onto the collection defaults, validated and registered. An `include` layers +what the included base states, between the parent and the preset's own keys: the base's +diff against the default, taken when the base itself was installed and before the per-variant padding `inherits` sees, so only what a template sets reaches the presets including it. The two paths share everything below the parse, which is what makes a cache-loaded bundle @@ -223,6 +229,16 @@ indistinguishable from a JSON-loaded one by construction rather than by test cov Installation also rebuilds each preset's file path from the local data directory, so a shipped cache never carries the generating machine's paths. +Installing an entry is split in two. `resolve_vendor_preset` flattens it, reading only +what is registered under the names it inherits and includes, and `commit_vendor_preset` +registers it, the only step that writes anything shared. Entries resolve across threads +in runs and commit in the order the vendor lists them. A run ends before an entry that +inherits or includes one already in it, since that one's commit registers what the +entry resolves against, so no entry in a run reads what another in it registers. An +entry's parse messages are held until it commits. The bundle, the log's parse and +install messages and the error count therefore come out as parsing and installing one +entry at a time would leave them, whatever the listing order. + App upgrades work because a cache normally survives one. Only a deliberate `CACHE_VERSION` bump makes an installed cache unreadable, and that is handled at install time rather than at load: a vendor whose cache this build cannot read counts @@ -254,17 +270,20 @@ the wizard caches the *derived JSON*, not another form of the inputs: open, the wizard computes the current stamps (one version peek per vendor) and, when they match, serves the catalog from the file — no bundle built, no preset installed. Caching bundle inputs instead was tried and measured: rebuilding the bundle from -per-vendor caches costs ~2 s of preset installation whatever feeds it, so only -skipping the rebuild entirely wins. +per-vendor caches costs over a second of preset installation whatever feeds it, so +only skipping the rebuild entirely wins. Any change to the set — a vendor added, removed or updated, or its cache-only `.opc` replaced by a newer one — changes the stamps and retires the whole file; -the wizard then rebuilds the bundle vendor by vendor (per-vendor caches serving where -they cover) and writes the catalog back. Selections, region and per-open decorations -are applied downstream of the cache either way, so a served catalog is -indistinguishable from a rebuilt one. Nothing ships this file and the updater never -touches it; it is a locally written artifact, re-derived whenever stale, written -through a temp file and rename so half a cache is never readable. +the wizard then rebuilds the bundle with `PresetBundle::load_vendors`, the load +startup uses (per-vendor caches serving where they cover), and writes the catalog +back. When a vendor fails to load, the filament library included, that open falls +back to the wizard's own scan of the vendor JSONs, as when no bundle can be built, +and writes nothing. Selections, region and per-open decorations are applied +downstream of the cache either way, so a served catalog is indistinguishable from a +rebuilt one. Nothing ships this file and the updater never touches it; it is a +locally written artifact, re-derived whenever stale, written through a temp file and +rename so half a cache is never readable. The cache lives under `/cache/`, not beside the vendors: everything that scans `/system/` treats any `.opc` there as a vendor, so a non-vendor @@ -378,6 +397,13 @@ enumerates only `*.json` will find no vendors at all in a packaged build. the `CachedPreset` field list — written and read by `visit_entry` in `PresetCacheFormat.cpp`, one list for the save, the load and the name peek alike — or the cache's own layout or stamps, requires bumping `CACHE_VERSION` by hand. +- **Adding a kind of reference between presets**, as `inherits` and `include` are: + parse the names into `CachedPreset` (a field change, so `CACHE_VERSION` is bumped), + have `install_vendor_entries` end a run before an entry that names one already in it + and retain what the names point at, look them up only in `resolve_vendor_preset`, and + register what they point at only in `commit_vendor_preset`. The listing-order test in + `test_vendor_cache.cpp` fails for a kind the runs do not check once its fixture uses + it. - **The dictionary indexes with a `uint16`**, so `print_config_def` may hold at most 65535 options and one cache at most 65535 distinct enum value names. `CacheDictionary::save` throws past that, which surfaces when CI generates the diff --git a/src/libslic3r/CMakeLists.txt b/src/libslic3r/CMakeLists.txt index eede13737c..246bb46937 100644 --- a/src/libslic3r/CMakeLists.txt +++ b/src/libslic3r/CMakeLists.txt @@ -355,6 +355,7 @@ set(lisbslic3r_sources Optimize/Optimizer.hpp Orient.cpp Orient.hpp + ParallelResolve.hpp ParameterUtils.cpp ParameterUtils.hpp pchheader.cpp diff --git a/src/libslic3r/Config.cpp b/src/libslic3r/Config.cpp index 42d6970ad1..427f6e66c9 100644 --- a/src/libslic3r/Config.cpp +++ b/src/libslic3r/Config.cpp @@ -849,6 +849,19 @@ ConfigSubstitutions ConfigBase::load_from_json(const std::string &file, ForwardC return std::move(substitutions_ctxt.substitutions); } +// Case-insensitive compare of a JSON key against a fixed ASCII one, without +// boost::iequals, whose std::locale() takes a lock the whole process shares in the +// MSVC runtime. +static bool ascii_iequals(const std::string &key, const char *literal) +{ + auto lower = [](char c) { return (c >= 'A' && c <= 'Z') ? char(c - 'A' + 'a') : c; }; + size_t i = 0; + for (; i < key.size() && literal[i] != '\0'; ++ i) + if (lower(key[i]) != lower(literal[i])) + return false; + return i == key.size() && literal[i] == '\0'; +} + int ConfigBase::load_from_json(const std::string &file, ConfigSubstitutionContext& substitution_context, bool load_inherits_to_config, std::map& key_values, std::string& reason) { json j; @@ -916,44 +929,44 @@ int ConfigBase::load_from_json(const std::string &file, ConfigSubstitutionContex } //parse the json elements for (auto it = j.begin(); it != j.end(); it++) { - if (boost::iequals(it.key(),BBL_JSON_KEY_VERSION)) { + if (ascii_iequals(it.key(), BBL_JSON_KEY_VERSION)) { key_values.emplace(BBL_JSON_KEY_VERSION, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_IS_CUSTOM)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_IS_CUSTOM)) { //skip it } - else if (boost::iequals(it.key(), BBL_JSON_KEY_NAME)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_NAME)) { key_values.emplace(BBL_JSON_KEY_NAME, it.value()); if (it.value() == "project_settings") is_project_settings = true; } - else if (boost::iequals(it.key(), BBL_JSON_KEY_URL)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_URL)) { key_values.emplace(BBL_JSON_KEY_URL, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_TYPE)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_TYPE)) { key_values.emplace(BBL_JSON_KEY_TYPE, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_SETTING_ID)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_SETTING_ID)) { key_values.emplace(BBL_JSON_KEY_SETTING_ID, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_FILAMENT_ID)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_FILAMENT_ID)) { key_values.emplace(BBL_JSON_KEY_FILAMENT_ID, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_FROM)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_FROM)) { key_values.emplace(BBL_JSON_KEY_FROM, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_DESCRIPTION)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_DESCRIPTION)) { key_values.emplace(BBL_JSON_KEY_DESCRIPTION, it.value()); } - else if (boost::iequals(it.key(), BBL_JSON_KEY_INSTANTIATION)) { + else if (ascii_iequals(it.key(), BBL_JSON_KEY_INSTANTIATION)) { key_values.emplace(BBL_JSON_KEY_INSTANTIATION, it.value()); } - else if (!load_inherits_to_config && boost::iequals(it.key(), BBL_JSON_KEY_INHERITS)) { + else if (!load_inherits_to_config && ascii_iequals(it.key(), BBL_JSON_KEY_INHERITS)) { key_values.emplace(BBL_JSON_KEY_INHERITS, it.value()); } - else if (!load_inherits_to_config && boost::iequals(it.key(), BBL_JSON_KEY_INCLUDES)) { + else if (!load_inherits_to_config && ascii_iequals(it.key(), BBL_JSON_KEY_INCLUDES)) { key_values.emplace(BBL_JSON_KEY_INCLUDES, it.value().dump()); - } else if (boost::iequals(it.key(), ORCA_JSON_KEY_RENAMED_FROM)) { + } else if (ascii_iequals(it.key(), ORCA_JSON_KEY_RENAMED_FROM)) { key_values.emplace(ORCA_JSON_KEY_RENAMED_FROM, it.value()); } else { t_config_option_key opt_key = it.key(); diff --git a/src/libslic3r/LocalesUtils.cpp b/src/libslic3r/LocalesUtils.cpp index e727b29b09..9c81511dc5 100644 --- a/src/libslic3r/LocalesUtils.cpp +++ b/src/libslic3r/LocalesUtils.cpp @@ -13,8 +13,19 @@ namespace Slic3r { +// How many setters this thread holds, so the ones nested in another can skip +// setlocale, which takes a lock the whole process shares on Windows. +static thread_local int s_numeric_locale_depth = 0; + CNumericLocalesSetter::CNumericLocalesSetter() { + // Nested in another setter on this thread, whose "C" the separator check + // confirms is still set. + if (s_numeric_locale_depth > 0 && is_decimal_separator_point()) { + m_nested = true; + ++ s_numeric_locale_depth; + return; + } #ifdef _WIN32 _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); m_orig_numeric_locale = std::setlocale(LC_NUMERIC, nullptr); @@ -29,12 +40,17 @@ CNumericLocalesSetter::CNumericLocalesSetter() m_new_locale = newlocale(LC_NUMERIC_MASK, "C", m_new_locale); uselocale(m_new_locale); #endif + // Counted last, since the destructor does not run for a constructor that throws. + ++ s_numeric_locale_depth; } CNumericLocalesSetter::~CNumericLocalesSetter() { + -- s_numeric_locale_depth; + if (m_nested) + return; #ifdef _WIN32 std::setlocale(LC_NUMERIC, m_orig_numeric_locale.data()); #else diff --git a/src/libslic3r/LocalesUtils.hpp b/src/libslic3r/LocalesUtils.hpp index f63c3572f7..d7e59f5ded 100644 --- a/src/libslic3r/LocalesUtils.hpp +++ b/src/libslic3r/LocalesUtils.hpp @@ -19,8 +19,13 @@ class CNumericLocalesSetter { public: CNumericLocalesSetter(); ~CNumericLocalesSetter(); + // A copy would restore the locale twice, and count down once more than up. + CNumericLocalesSetter(const CNumericLocalesSetter&) = delete; + CNumericLocalesSetter& operator=(const CNumericLocalesSetter&) = delete; private: + // Inside another setter on this thread, which does the setting and restoring. + bool m_nested { false }; #ifdef _WIN32 std::string m_orig_numeric_locale; #else diff --git a/src/libslic3r/ParallelResolve.hpp b/src/libslic3r/ParallelResolve.hpp new file mode 100644 index 0000000000..e913be728a --- /dev/null +++ b/src/libslic3r/ParallelResolve.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "Exception.hpp" + +namespace Slic3r { + +// A batch of resolved presets, each a whole config, is what resolve_then_commit adds +// to peak memory, so it stays far below a vendor's preset count and above any core count. +inline constexpr size_t resolve_batch_size = 64; + +// Resolve `count` items that do not depend on each other and install them one at +// a time. +// +// `resolve(i)` runs on any thread and must read only, since the items are +// resolved side by side. `commit(i, resolved)` is called for every item in index +// order, on the calling thread, and is where shared state is written. +// +// The items are worked through in batches, so what is resolved and held at once +// does not grow with `count`. An exception from either callable propagates after +// the batches before it have been committed. A cancellation of the caller's task +// group, which stops a batch partway without an exception, throws RuntimeError +// before that batch commits. +// +// `ChunkSetup`, when given, is constructed once for each piece of a batch TBB hands +// out, for per-thread state a resolve would otherwise set up per item, such as the +// C numeric locale, whose setting takes a lock the whole process shares. +template +void resolve_then_commit(size_t count, Resolve resolve, Commit commit) +{ + using Resolved = std::invoke_result_t; + + std::vector resolved(std::min(count, resolve_batch_size)); + for (size_t first = 0; first < count; first += resolve_batch_size) { + const size_t last = std::min(first + resolve_batch_size, count); + // Isolated, so a thread waiting on the batch runs none of the caller's other + // tasks before finishing it. + tbb::this_task_arena::isolate([&] { + tbb::parallel_for(tbb::blocked_range(first, last), + [&](const tbb::blocked_range& range) { + ChunkSetup setup; + (void) setup; + for (size_t i = range.begin(); i < range.end(); ++ i) + resolved[i - first] = resolve(i); + }); + }); + if (tbb::is_current_task_group_canceling()) + throw RuntimeError("resolve_then_commit: canceled before the batch was resolved"); + for (size_t i = first; i < last; ++ i) + commit(i, std::move(resolved[i - first])); + } +} + +} // namespace Slic3r diff --git a/src/libslic3r/Preset.cpp b/src/libslic3r/Preset.cpp index 9af8969490..e8437f7c0b 100644 --- a/src/libslic3r/Preset.cpp +++ b/src/libslic3r/Preset.cpp @@ -5,6 +5,8 @@ #include "Preset.hpp" #include "PresetBundle.hpp" #include "AppConfig.hpp" +#include "LocalesUtils.hpp" +#include "ParallelResolve.hpp" #ifdef _MSC_VER #define WIN32_LEAN_AND_MEAN @@ -506,10 +508,11 @@ void Preset::normalize(DynamicPrintConfig &config) handle_legacy_sla(config); } -std::string Preset::remove_invalid_keys(DynamicPrintConfig &config, const DynamicPrintConfig &default_config) +std::string Preset::remove_invalid_keys(DynamicPrintConfig &config, const DynamicPrintConfig &default_config, + const DynamicPrintConfig *added) { std::string incorrect_keys; - for (const std::string &key : config.keys()) + for (const std::string &key : (added != nullptr ? *added : config).keys()) if (! default_config.has(key)) { if (incorrect_keys.empty()) incorrect_keys = key; @@ -621,7 +624,6 @@ void Preset::load_info(const std::string& file) } else if (v.first.compare("base_id") == 0) { this->base_id = v.second.get_value(); - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " load info from: " << file << " and base_id: " << this->base_id; if (this->base_id.compare("null") == 0) this->base_id.clear(); } @@ -1669,6 +1671,169 @@ std::string PresetCollection::canonical_preset_name(const std::string &name, con return get_preset_canonical_name(parsed.bare, origin); } +PresetCollection::UserPresetLoad PresetCollection::resolve_user_preset( + const boost::filesystem::path &file, const std::string &canonical_name, + const PresetOrigin &load_origin, ForwardCompatibilitySubstitutionRule substitution_rule, + const std::string &extruder_id_name, const std::string &extruder_variant_name, + std::set *key_set1, std::set *key_set2) const +{ + UserPresetLoad out; + out.preset = Preset(m_type, canonical_name, false); + Preset &preset = out.preset; + preset.bundle_id = load_origin.bundle_id; + preset.file = file.string(); + // Load the preset file, apply preset values on top of defaults. + try { + fs::path idx_path(preset.file); + idx_path.replace_extension(".info"); + if (fs::exists(idx_path)) { + out.info_file = idx_path.string(); + preset.load_info(out.info_file); + } + DynamicPrintConfig config; + //BBS: change to json format + //ConfigSubstitutions config_substitutions = config.load_from_ini(preset.file, substitution_rule); + std::map key_values; + std::string reason; + ConfigSubstitutions config_substitutions = config.load_from_json(preset.file, substitution_rule, key_values, reason); + if (! config_substitutions.empty()) + out.substitutions.push_back({ preset.name, m_type, PresetConfigSubstitutions::Source::UserFile, preset.file, std::move(config_substitutions) }); + if (!reason.empty()) { + out.discard_file = true; + out.errors.push_back((boost::format("parse config %1% failed") % preset.file).str()); + return out; + } + + std::string version_str = key_values[BBL_JSON_KEY_VERSION]; + boost::optional version = Semver::parse(version_str); + if (!version) return out; + preset.version = *version; + + if (key_values.find(BBL_JSON_KEY_FILAMENT_ID) != key_values.end()) + preset.filament_id = key_values[BBL_JSON_KEY_FILAMENT_ID]; + if (key_values.find(BBL_JSON_KEY_DESCRIPTION) != key_values.end()) + preset.description = key_values[BBL_JSON_KEY_DESCRIPTION]; + if (key_values.find(BBL_JSON_KEY_INSTANTIATION) != key_values.end()) + preset.is_visible = key_values[BBL_JSON_KEY_INSTANTIATION] != "false"; + + //Orca: find and use the inherit config as the base + const Preset* inherit_preset = nullptr; + ConfigOption* inherits_config = config.option(BBL_JSON_KEY_INHERITS); + + // check inherits_config + if (inherits_config) { + ConfigOptionString * option_str = dynamic_cast (inherits_config); + std::string inherits_value = option_str->value; + // Orca: try to find if the parent preset has been renamed + inherit_preset = this->find_preset2(inherits_value); + Preset::normalize_inherits(config, inherit_preset); + } + const Preset& default_preset = this->default_preset_for(config); + if (inherit_preset) { + preset.config = inherit_preset->config; + preset.filament_id = inherit_preset->filament_id; + extend_default_config_length(config, false, {}); + preset.config.update_diff_values_to_child_config(config, extruder_id_name, extruder_variant_name, *key_set1, *key_set2); + } + else { + auto inherits_config2 = dynamic_cast(inherits_config); + if ((inherits_config2 && !inherits_config2->value.empty())) { + out.errors.push_back((boost::format("can not find parent %1% for config %2%!") % inherits_config2->value % preset.file).str()); + return out; + } + // We support custom root preset now + // Find a default preset for the config. The PrintPresetCollection provides different default preset based on the "printer_technology" field. + preset.config = default_preset.config; + preset.config.apply(std::move(config)); + extend_default_config_length(preset.config, true, default_preset.config); + } + + Preset::normalize(preset.config); + // Report configuration fields, which are misplaced into a wrong group. + std::string incorrect_keys = Preset::remove_invalid_keys(preset.config, default_preset.config); + if (!incorrect_keys.empty()) + out.errors.push_back("Error in a preset file: The preset \"" + preset.file + + "\" contains the following incorrect keys: " + incorrect_keys + ", which were removed"); + + if (preset.type == Preset::TYPE_FILAMENT && preset.is_user() && preset.inherits().empty()) { + auto compatible_printers = dynamic_cast(preset.config.option("compatible_printers", true)); + if (compatible_printers && compatible_printers->values.empty()) { + size_t at_pos = canonical_name.find('@'); + if (at_pos != std::string::npos && at_pos + 1 < canonical_name.length()) { + compatible_printers->values.push_back(canonical_name.substr(at_pos + 1)); + out.save_compatible_printers = true; + } + } + } + + preset.loaded = true; + out.complete = true; + } catch (const std::ifstream::failure &err) { + out.discard_file = true; + out.errors.push_back((boost::format("The user-config cannot be loaded: %1%. Reason: %2%") % preset.file % err.what()).str()); + //throw Slic3r::RuntimeError(std::string("The selected preset cannot be loaded: ") + preset.file + "\n\tReason: " + err.what()); + } catch (const std::runtime_error &err) { + out.discard_file = true; + out.errors.push_back((boost::format("Failed loading the user-config file: %1%. Reason: %2%") % preset.file % err.what()).str()); + //throw Slic3r::RuntimeError(std::string("Failed loading the preset file: ") + preset.file + "\n\tReason: " + err.what()); + } + out.install = true; + return out; +} + +void PresetCollection::commit_user_preset(UserPresetLoad &&loaded, std::deque &presets_loaded, + PresetsConfigSubstitutions &substitutions, + const std::function &preset_loaded_fn, + bool read_only) +{ + Preset &preset = loaded.preset; + if (! loaded.info_file.empty()) + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " load info from: " << loaded.info_file << " and base_id: " << preset.base_id; + append(substitutions, std::move(loaded.substitutions)); + for (const std::string &error : loaded.errors) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << error; + } + if (loaded.discard_file && !read_only) { + fs::path file_path(loaded.preset.file); + if (fs::exists(file_path)) + fs::remove(file_path); + file_path.replace_extension(".info"); + if (fs::exists(file_path)) + fs::remove(file_path); + } + if (!loaded.install) + return; + + if (loaded.complete) { + if (loaded.save_compatible_printers) { + // A filesystem error from the rewrite is counted, and the preset still loads. + try { + if (!read_only) + preset.save(nullptr); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " added compatible_printers for preset: " << preset.name; + } catch (const std::runtime_error &err) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << " can not write compatible_printers back to " << preset.file << ": " << err.what(); + } + } + //BBS: add some workaround for previous incorrect settings + if ((!preset.setting_id.empty())&&(preset.setting_id == preset.base_id)) + preset.setting_id.clear(); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " load preset: " << preset.name << " and filament_id: " << preset.filament_id << " and base_id: " << preset.base_id; + //BBS: add config related logs + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", preset type %1%, name %2%, path %3%, is_system %4%, is_default %5% is_visible %6%")%Preset::get_type_string(m_type) %preset.name %preset.file %preset.is_system %preset.is_default %preset.is_visible; + // add alias for custom filament preset + set_custom_preset_alias(preset); + } + + if (preset_loaded_fn != nullptr) + preset_loaded_fn(preset); + + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << __LINE__ << " load config successful and preset name is:" << preset.name; + presets_loaded.emplace_back(std::move(preset)); +} + // Load all presets found in dir_path. // Throws an exception on error. void PresetCollection::load_presets( @@ -1706,6 +1871,8 @@ void PresetCollection::load_presets( std::set *key_set1 = nullptr, *key_set2 = nullptr; Preset::get_extruder_names_and_keysets(m_type, extruder_id_name, extruder_variant_name, &key_set1, &key_set2); + struct UserPresetFile { fs::path path; std::string canonical_name; }; + std::vector files; //BBS: change to json format for (auto &dir_entry : boost::filesystem::directory_iterator(dir)) { @@ -1721,149 +1888,26 @@ void PresetCollection::load_presets( BOOST_LOG_TRIVIAL(warning) << "Preset already present, not loading: " << canonical_name; continue; } + files.push_back({ dir_entry.path(), std::move(canonical_name) }); + } + } + + resolve_then_commit(files.size(), + [&](size_t i) { + return this->resolve_user_preset(files[i].path, files[i].canonical_name, resolved_origin, substitution_rule, + extruder_id_name, extruder_variant_name, key_set1, key_set2); + }, + [&](size_t, UserPresetLoad &&loaded) { + // Committing can remove an unreadable preset's file, and a filesystem error + // there is reported without stopping the rest of the directory. try { - Preset preset(m_type, canonical_name, false); - preset.bundle_id = resolved_origin.bundle_id; - preset.file = dir_entry.path().string(); - // Load the preset file, apply preset values on top of defaults. - try { - fs::path idx_path(preset.file); - idx_path.replace_extension(".info"); - if (fs::exists(idx_path)) { - preset.load_info(idx_path.string()); - } - DynamicPrintConfig config; - //BBS: change to json format - //ConfigSubstitutions config_substitutions = config.load_from_ini(preset.file, substitution_rule); - std::map key_values; - std::string reason; - ConfigSubstitutions config_substitutions = config.load_from_json(preset.file, substitution_rule, key_values, reason); - if (! config_substitutions.empty()) - substitutions.push_back({ preset.name, m_type, PresetConfigSubstitutions::Source::UserFile, preset.file, std::move(config_substitutions) }); - if (!reason.empty()) { - fs::path file_path(preset.file); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - file_path.replace_extension(".info"); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - BOOST_LOG_TRIVIAL(error) << boost::format("parse config %1% failed")%preset.file; - ++m_errors; - continue; - } - - std::string version_str = key_values[BBL_JSON_KEY_VERSION]; - boost::optional version = Semver::parse(version_str); - if (!version) continue; - preset.version = *version; - - if (key_values.find(BBL_JSON_KEY_FILAMENT_ID) != key_values.end()) - preset.filament_id = key_values[BBL_JSON_KEY_FILAMENT_ID]; - if (key_values.find(BBL_JSON_KEY_DESCRIPTION) != key_values.end()) - preset.description = key_values[BBL_JSON_KEY_DESCRIPTION]; - if (key_values.find(BBL_JSON_KEY_INSTANTIATION) != key_values.end()) - preset.is_visible = key_values[BBL_JSON_KEY_INSTANTIATION] != "false"; - - //Orca: find and use the inherit config as the base - Preset* inherit_preset = nullptr; - ConfigOption* inherits_config = config.option(BBL_JSON_KEY_INHERITS); - - // check inherits_config - if (inherits_config) { - ConfigOptionString * option_str = dynamic_cast (inherits_config); - std::string inherits_value = option_str->value; - // Orca: try to find if the parent preset has been renamed - inherit_preset = this->find_preset2(inherits_value); - Preset::normalize_inherits(config, inherit_preset); - } else { - ; - } - const Preset& default_preset = this->default_preset_for(config); - if (inherit_preset) { - preset.config = inherit_preset->config; - preset.filament_id = inherit_preset->filament_id; - extend_default_config_length(config, false, {}); - preset.config.update_diff_values_to_child_config(config, extruder_id_name, extruder_variant_name, *key_set1, *key_set2); - } - else { - auto inherits_config2 = dynamic_cast(inherits_config); - if ((inherits_config2 && !inherits_config2->value.empty())) { - BOOST_LOG_TRIVIAL(error) << boost::format("can not find parent %1% for config %2%!")%inherits_config2->value %preset.file; - ++m_errors; - continue; - } - // We support custom root preset now - // Find a default preset for the config. The PrintPresetCollection provides different default preset based on the "printer_technology" field. - preset.config = default_preset.config; - preset.config.apply(std::move(config)); - extend_default_config_length(preset.config, true, default_preset.config); - } - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " load preset: " << name << " and filament_id: " << preset.filament_id << " and base_id: " << preset.base_id; - - Preset::normalize(preset.config); - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(preset.config, default_preset.config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) - << "Error in a preset file: The preset \"" << preset.file - << "\" contains the following incorrect keys: " << incorrect_keys << ", which were removed"; - } - - if (preset.type == Preset::TYPE_FILAMENT && preset.is_user() && preset.inherits().empty()) { - auto compatible_printers = dynamic_cast(preset.config.option("compatible_printers", true)); - if (compatible_printers && compatible_printers->values.empty()) { - size_t at_pos = name.find('@'); - if (at_pos != std::string::npos && at_pos + 1 < name.length()) { - compatible_printers->values.push_back(name.substr(at_pos + 1)); - if (!read_only) - preset.save(nullptr); - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " added compatible_printers for preset: " << name; - } - } - } - - preset.loaded = true; - //BBS: add some workaround for previous incorrect settings - if ((!preset.setting_id.empty())&&(preset.setting_id == preset.base_id)) - preset.setting_id.clear(); - //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", preset type %1%, name %2%, path %3%, is_system %4%, is_default %5% is_visible %6%")%Preset::get_type_string(m_type) %preset.name %preset.file %preset.is_system %preset.is_default %preset.is_visible; - // add alias for custom filament preset - set_custom_preset_alias(preset); - } catch (const std::ifstream::failure &err) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << boost::format("The user-config cannot be loaded: %1%. Reason: %2%")%preset.file %err.what(); - fs::path file_path(preset.file); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - file_path.replace_extension(".info"); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - //throw Slic3r::RuntimeError(std::string("The selected preset cannot be loaded: ") + preset.file + "\n\tReason: " + err.what()); - } catch (const std::runtime_error &err) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << boost::format("Failed loading the user-config file: %1%. Reason: %2%")%preset.file %err.what(); - //throw Slic3r::RuntimeError(std::string("Failed loading the preset file: ") + preset.file + "\n\tReason: " + err.what()); - fs::path file_path(preset.file); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - file_path.replace_extension(".info"); - if (!read_only && fs::exists(file_path)) - fs::remove(file_path); - } - - if (preset_loaded_fn != nullptr) - preset_loaded_fn(preset); - - presets_loaded.emplace_back(preset); - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << __LINE__ << " load config successful and preset name is:" << preset.name; + this->commit_user_preset(std::move(loaded), presets_loaded, substitutions, preset_loaded_fn, read_only); } catch (const std::runtime_error &err) { errors_cummulative += err.what(); errors_cummulative += "\n"; } - } - } + }); + if (presets_loaded.size() > 0) m_presets.insert(m_presets.end(), std::make_move_iterator(presets_loaded.begin()), std::make_move_iterator(presets_loaded.end())); sort_presets(); @@ -2827,6 +2871,18 @@ std::pair PresetCollection::load_external_preset( return std::make_pair(&preset, false); } +Preset& PresetCollection::append_preset(std::string &&path, const std::string &name, DynamicPrintConfig &&config) +{ + lock(); + Preset &preset = m_presets.emplace_back(m_type, name, false); + preset.file = std::move(path); + preset.config = std::move(config); + preset.loaded = true; + preset.is_dirty = false; + unlock(); + return preset; +} + Preset& PresetCollection::load_preset(const std::string &path, const std::string &name, DynamicPrintConfig &&config, bool select, Semver file_version) { lock(); @@ -3846,28 +3902,53 @@ bool PresetCollection::select_preset_by_name_strict(const std::string &name) return false; } -// Merge one vendor's presets with the other vendor's presets, report duplicates. -std::vector PresetCollection::merge_presets(PresetCollection &&other, const VendorMap &new_vendors) +std::vector> PresetCollection::merge_presets(const std::vector &others, const VendorMap &new_vendors) { - std::vector duplicates; - for (Preset &preset : other.m_presets) { - if (preset.is_default || preset.is_external) - continue; - Preset key(m_type, preset.name); - auto it = (m_type == Preset::TYPE_FILAMENT) - ? std::lower_bound(m_presets.begin() + m_num_default_presets, m_presets.end(), key, filament_preset_less) - : std::lower_bound(m_presets.begin() + m_num_default_presets, m_presets.end(), key); - if (it == m_presets.end() || it->name != preset.name) { + auto less = [this](const Preset &a, const Preset &b) { + return m_type == Preset::TYPE_FILAMENT ? filament_preset_less(a, b) : a < b; + }; + struct Incoming { Preset *preset; size_t source; }; + auto incoming_less = [&less](const Incoming &a, const Incoming &b) { return less(*a.preset, *b.preset); }; + // Each of `others` is sorted, so its presets form one sorted run. + std::vector incoming; + std::vector run_ends { 0 }; + for (size_t source = 0; source < others.size(); ++ source) { + for (Preset &preset : others[source]->m_presets) + if (! preset.is_default && ! preset.is_external) + incoming.push_back({ &preset, source }); + assert(std::is_sorted(incoming.begin() + run_ends.back(), incoming.end(), incoming_less)); + run_ends.push_back(incoming.size()); + } + // Merged pairwise and stably, so equal names stay in the order of `others`. + const size_t runs = others.size(); + for (size_t width = 1; width < runs; width *= 2) + for (size_t i = 0; i + width < runs; i += 2 * width) + std::inplace_merge(incoming.begin() + run_ends[i], incoming.begin() + run_ends[i + width], + incoming.begin() + run_ends[std::min(i + 2 * width, runs)], incoming_less); + + std::vector> duplicates(others.size()); + std::deque merged; + auto own = m_presets.begin() + m_num_default_presets; + std::move(m_presets.begin(), own, std::back_inserter(merged)); + // On equal names this collection's preset is kept, else the earliest of `others`, + // and each repeat is listed under the collection it came from. + for (auto next = incoming.begin(); own != m_presets.end() || next != incoming.end();) { + if (next == incoming.end() || (own != m_presets.end() && ! less(*next->preset, *own))) + merged.emplace_back(std::move(*own ++)); + else { + Preset &preset = *(next ++)->preset; if (preset.vendor != nullptr) { // Re-assign a pointer to the vendor structure in the new PresetBundle. auto it = new_vendors.find(preset.vendor->id); assert(it != new_vendors.end()); preset.vendor = &it->second; } - m_presets.emplace(it, std::move(preset)); - } else - duplicates.emplace_back(std::move(preset.name)); + merged.emplace_back(std::move(preset)); + } + for (; next != incoming.end() && next->preset->name == merged.back().name; ++ next) + duplicates[next->source].emplace_back(next->preset->name); } + m_presets = std::move(merged); return duplicates; } diff --git a/src/libslic3r/Preset.hpp b/src/libslic3r/Preset.hpp index 5b32d90ac6..329e835afe 100644 --- a/src/libslic3r/Preset.hpp +++ b/src/libslic3r/Preset.hpp @@ -445,7 +445,10 @@ public: static std::string remove_suffix_modified(const std::string& name); static void normalize(DynamicPrintConfig &config); // Report configuration fields, which are misplaced into a wrong group, remove them from the config. - static std::string remove_invalid_keys(DynamicPrintConfig &config, const DynamicPrintConfig &default_config); + // `added`, when given, is the diff applied over a copy of default_config, and only + // its keys are checked, since no other key can be missing from default_config. + static std::string remove_invalid_keys(DynamicPrintConfig &config, const DynamicPrintConfig &default_config, + const DynamicPrintConfig *added = nullptr); // BBS: move constructor to public Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {} @@ -801,6 +804,8 @@ public: // Return number of presets including the "- default -" preset. size_t size() const { return m_presets.size(); } bool has_defaults_only() const { return m_presets.size() <= m_num_default_presets; } + // How many presets this collection refused or repaired while loading. + int error_count() const { return m_errors; } // For Print / Filament presets, disable those, which are not compatible with the printer. template @@ -874,8 +879,10 @@ protected: // This is a temporary state, which shall be fixed immediately by the following step. bool select_preset_by_name_strict(const std::string &name); - // Merge one vendor's presets with the other vendor's presets, report duplicates. - std::vector merge_presets(PresetCollection &&other, const VendorMap &new_vendors); + // Move the presets of `others` into this collection in one pass. A name this + // collection or an earlier one of `others` already has is left out, and reported + // in the list of the collection that repeats it. + std::vector> merge_presets(const std::vector &others, const VendorMap &new_vendors); // Update m_map_alias_to_profile_name from loaded system profiles. void update_map_alias_to_profile_name(); @@ -890,6 +897,43 @@ protected: void set_custom_preset_alias(Preset &preset); private: + // One preset file read and flattened against the presets already in this + // collection, before anything the collection shares has been touched. + struct UserPresetLoad + { + Preset preset; + // Joins the collection. A file that threw partway still joins it, without + // the steps that did not run. + bool install { false }; + // The whole of the load ran, so the preset is ready to be aliased. + bool complete { false }; + // A filament preset that named no compatible printer and was given one from + // its name, which commit writes back to its file. + bool save_compatible_printers { false }; + // Unreadable, so commit removes it and its .info file. + bool discard_file { false }; + // The .info file read beside the preset, which commit logs. + std::string info_file; + // Counted and logged by commit, in the order the directory listed the files. + std::vector errors; + PresetsConfigSubstitutions substitutions; + }; + + // Read and flatten one preset file. It reads only, and resolves against the presets + // loaded before this pass, never another file of the same pass, so the files of a + // pass are independent of each other. + UserPresetLoad resolve_user_preset(const boost::filesystem::path &file, const std::string &canonical_name, + const PresetOrigin &load_origin, ForwardCompatibilitySubstitutionRule substitution_rule, + const std::string &extruder_id_name, const std::string &extruder_variant_name, + std::set *key_set1, std::set *key_set2) const; + + // Install one resolved preset. The collection, its alias maps, the error count + // and the preset files on disk are touched here and only here. + void commit_user_preset(UserPresetLoad &&loaded, std::deque &presets_loaded, + PresetsConfigSubstitutions &substitutions, + const std::function &preset_loaded_fn, + bool read_only); + std::string canonical_preset_name(const std::string &name, const PresetOrigin &load_origin = PresetOrigin()) const; // Comparator that sorts "Generic " prefixed presets before others, then alphabetically within each group. @@ -901,6 +945,10 @@ private: return a.name < b.name; } + // Append a preset without keeping the collection sorted, for a caller installing + // many at once; find_preset() is unusable until sort_presets() runs. + Preset& append_preset(std::string &&path, const std::string &name, DynamicPrintConfig &&config); + // Sort presets: filament presets use generic-first ordering, others sort alphabetically. void sort_presets() { if (m_type == Preset::TYPE_FILAMENT) diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index b2cc67f970..c5b78a8e0f 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -5,6 +5,7 @@ #include "PresetBundle.hpp" +#include "ParallelResolve.hpp" #include "PresetCacheFormat.hpp" #include "PrintConfig.hpp" #include "PublishSettings.hpp" @@ -19,6 +20,7 @@ #include #include +#include #include #include #include @@ -37,6 +39,8 @@ #include #include #include +#include +#include // Mark string for localization and translate. #define L(s) Slic3r::I18N::translate(s) @@ -2501,35 +2505,129 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); - const auto load_t0 = std::chrono::steady_clock::now(); - // The vendors below are loaded whole and against each other — the filament // library first, then every other vendor with it as the base — so each parse // is complete enough to be worth caching. m_generate_vendor_caches = allow_cache && (m_generate_vendor_caches || !validation_mode); + // Sorted, so any duplicate-preset warning comes out in the same order on every run. + std::vector vendors; + for (const std::string& name : vendor_names_in(dir)) + if (name == ORCA_FILAMENT_LIBRARY || !(validation_mode && !vendor_to_validate.empty() && name != vendor_to_validate)) + vendors.push_back({ name, dir }); + auto result = this->load_vendors(vendors, compatibility_rule, allow_cache); + + this->update_system_maps(); + + //BBS: add config related logs + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%result.second; + return result; +} + +std::pair PresetBundle::load_vendors(const std::vector& vendors, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, const std::atomic* cancel, + std::vector* failed) +{ + const auto load_t0 = std::chrono::steady_clock::now(); + auto canceled = [cancel] { return cancel != nullptr && cancel->load(); }; + PresetsConfigSubstitutions substitutions; std::string errors_cummulative; bool first = true; - // Sorted, so any duplicate-preset warning below comes out in the same order on - // every run. - const std::set vendor_names = vendor_names_in(dir); - // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded - // first because other vendors' filaments may inherit from it via the - // `base_bundle` lookup in parse_subfile. The remaining vendors are - // independent (no cross-vendor inheritance) and can be loaded in parallel. - std::string orca_lib_vendor; - std::vector other_vendors; - other_vendors.reserve(vendor_names.size()); - for (auto& vn : vendor_names) { - if (vn == ORCA_FILAMENT_LIBRARY) - orca_lib_vendor = vn; - else if (!(validation_mode && !vendor_to_validate.empty() && vn != vendor_to_validate)) - other_vendors.push_back(vn); + // Other vendors inherit from nothing but the library, and only installing them + // looks it up, so each is read while it loads and installed once both are done. + const VendorSource* orca_lib = nullptr; + std::vector other_vendors; + other_vendors.reserve(vendors.size()); + for (const VendorSource& vendor : vendors) { + if (vendor.name == ORCA_FILAMENT_LIBRARY) + orca_lib = &vendor; + else + other_vendors.push_back(&vendor); } - // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (! orca_lib_vendor.empty()) { + // One of the other vendors, loaded into a PresetBundle of its own. + struct VendorLoad + { + const VendorSource* source { nullptr }; + std::unique_ptr bundle; + VendorRead read; + PresetsConfigSubstitutions substitutions; + std::string error; + // Counts the vendor's read and the library load; the second to finish installs it. + std::atomic ready { 0 }; + }; + std::vector loads(other_vendors.size()); + for (size_t i = 0; i < other_vendors.size(); ++i) + loads[i].source = other_vendors[i]; + + // Started slowest first, since the load ends when the slowest vendor does. + std::vector by_cost(loads.size()); + std::iota(by_cost.begin(), by_cost.end(), size_t(0)); + // A parse from JSON costs far more than any cache load, so parses go first, and + // within each group the bigger file, the cache or the .json that lists what + // to parse. A cache older than its profile is ordered as a parse. + const bool reads_caches = allow_cache && ! validation_mode; + std::vector> vendor_costs(loads.size()); + for (size_t i = 0; i < loads.size(); ++i) { + const VendorSource& vendor = *loads[i].source; + const boost::filesystem::path cache = vendor.dir / (vendor.name + ".opc"); + const boost::filesystem::path profile = vendor.dir / (vendor.name + ".json"); + boost::system::error_code ec, profile_ec; + const uintmax_t cache_size = reads_caches ? boost::filesystem::file_size(cache, ec) : 0; + if (reads_caches && ! ec) { + const std::time_t profile_time = boost::filesystem::last_write_time(profile, profile_ec); + if (profile_ec || profile_time <= boost::filesystem::last_write_time(cache, ec)) { + vendor_costs[i] = { false, cache_size }; + continue; + } + } + const uintmax_t profile_size = boost::filesystem::file_size(profile, profile_ec); + vendor_costs[i] = { true, profile_ec ? 0 : profile_size }; + } + std::stable_sort(by_cost.begin(), by_cost.end(), + [&](size_t a, size_t b) { return vendor_costs[a] > vendor_costs[b]; }); + + // A vendor that failed to read, or that is still to install when `cancel` is + // set, is left out. + auto install = [&](VendorLoad& load) { + if (load.bundle && ! canceled()) { + try { + load.substitutions = load.bundle->install_vendor_read(std::move(load.read), this).first; + } catch (const std::runtime_error &err) { + load.error = err.what(); + load.bundle.reset(); + } + } else + load.bundle.reset(); + load.read = VendorRead(); + }; + tbb::task_group group; + group.run([&] { + // An auto_partitioner splits the range only while a worker is asking for work, + // which at this size left every vendor on the calling thread. + tbb::parallel_for(tbb::blocked_range(0, loads.size(), 1), + [&](const tbb::blocked_range& range) { + for (size_t k = range.begin(); k < range.end(); ++k) { + VendorLoad& load = loads[by_cost[k]]; + if (! canceled()) { + try { + auto bundle = std::make_unique(); + bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); + load.read = bundle->read_vendor(load.source->dir.string(), load.source->name, + PresetBundle::LoadSystem, compatibility_rule, allow_cache); + load.bundle = std::move(bundle); + } catch (const std::runtime_error &err) { + load.error = err.what(); + } + } + if (++ load.ready == 2) + install(load); + } + }, tbb::simple_partitioner()); + }); + if (orca_lib != nullptr && ! canceled()) { try { // Match a fresh launch before parsing: hold aliases and the error // counter survive reset(), and would otherwise carry prior-cycle @@ -2537,66 +2635,55 @@ std::pair PresetBundle::load_system_pre this->clear_printer_hold_aliases(); this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json( - dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule, nullptr, allow_cache).first); + orca_lib->dir.string(), orca_lib->name, PresetBundle::LoadSystem, compatibility_rule, nullptr, allow_cache).first); first = false; } catch (const std::runtime_error &err) { if (validation_mode) throw err; errors_cummulative += err.what(); errors_cummulative += "\n"; + if (failed != nullptr) + failed->push_back(orca_lib->name); } } + for (size_t k : by_cost) + if (++ loads[k].ready == 2) + group.run([&install, &load = loads[k]] { install(load); }); + group.wait(); - // Step 2: Load remaining vendors in parallel. Each gets its own - // PresetBundle and uses `this` (which contains ORCA_FILAMENT_LIBRARY) - // as the base_bundle for cross-bundle inheritance lookups. - std::vector> parallel_bundles(other_vendors.size()); - std::vector parallel_substitutions(other_vendors.size()); - std::vector parallel_errors(other_vendors.size()); - - tbb::parallel_for(tbb::blocked_range(0, other_vendors.size()), - [&](const tbb::blocked_range& range) { - for (size_t i = range.begin(); i < range.end(); ++i) { - auto bundle = std::make_unique(); - bundle->set_is_validation_mode(validation_mode); - bundle->set_generate_vendor_caches(m_generate_vendor_caches); - try { - auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this, allow_cache); - parallel_substitutions[i] = std::move(result.first); - parallel_bundles[i] = std::move(bundle); - } catch (const std::runtime_error &err) { - parallel_errors[i] = err.what(); - } - } - }); - - // Step 3: Sequentially merge the parallel-loaded bundles into `this`. - // The merge order is the original vendor order so any duplicate-warning - // output stays stable across runs. - for (size_t i = 0; i < other_vendors.size(); ++i) { - if (!parallel_errors[i].empty()) { + // Merged in the original vendor order, so any duplicate-warning output stays + // stable across runs. + std::vector bundles; + for (VendorLoad& load : loads) + if (load.bundle) + bundles.push_back(load.bundle.get()); + const std::vector> duplicates = this->merge_presets(bundles); + size_t merged = 0; + for (VendorLoad& load : loads) { + if (! load.error.empty()) { if (validation_mode) - throw std::runtime_error(parallel_errors[i]); - errors_cummulative += parallel_errors[i]; + throw std::runtime_error(load.error); + errors_cummulative += load.error; errors_cummulative += "\n"; + if (failed != nullptr) + failed->push_back(load.source->name); continue; } - if (!parallel_bundles[i]) + if (! load.bundle) continue; - const std::string& vendor_name = other_vendors[i]; - append(substitutions, std::move(parallel_substitutions[i])); - std::vector duplicates = this->merge_presets(std::move(*parallel_bundles[i])); + const std::string& vendor_name = load.source->name; + const std::vector& vendor_duplicates = duplicates[merged ++]; + append(substitutions, std::move(load.substitutions)); first = false; - if (!duplicates.empty()) { + if (!vendor_duplicates.empty()) { errors_cummulative += "Found duplicated settings in vendor " + vendor_name + "'s json file lists: "; - for (size_t j = 0; j < duplicates.size(); ++j) { - if (j > 0) + for (size_t k = 0; k < vendor_duplicates.size(); ++k) { + if (k > 0) errors_cummulative += ", "; - errors_cummulative += duplicates[j]; + errors_cummulative += vendor_duplicates[k]; ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Found duplicated preset: " + duplicates[j] + " in vendor: " + vendor_name + ": "; + BOOST_LOG_TRIVIAL(error) << "Found duplicated preset: " + vendor_duplicates[k] + " in vendor: " + vendor_name + ": "; } } } @@ -2606,14 +2693,9 @@ std::pair PresetBundle::load_system_pre this->reset(false); } - this->update_system_maps(); - const auto load_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - load_t0).count(); - BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; - - //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendors.size() << " vendor(s) loaded in " << load_ms << " ms"; return std::make_pair(std::move(substitutions), errors_cummulative); } @@ -2682,7 +2764,7 @@ std::pair PresetBundle::load_system_fil // Report duplicate profiles. PresetBundle other; append(substitutions, other.load_vendor_configs_from_json(dir.string(), vendor_name, PresetBundle::LoadSystem | PresetBundle::LoadFilamentOnly, compatibility_rule).first); - std::vector duplicates = this->merge_presets(std::move(other)); + std::vector duplicates = std::move(this->merge_presets({ &other }).front()); if (!duplicates.empty()) { errors_cummulative += "Found duplicated settings in vendor " + vendor_name + "'s json file lists: "; for (size_t i = 0; i < duplicates.size(); ++i) { @@ -2728,26 +2810,33 @@ VendorProfile PresetBundle::get_custom_vendor_models() const return vendor; } -// Merge one vendor's presets with the other vendor's presets, report duplicates. -std::vector PresetBundle::merge_presets(PresetBundle &&other) +std::vector> PresetBundle::merge_presets(const std::vector &others) { - this->vendors.insert(other.vendors.begin(), other.vendors.end()); - std::vector duplicate_prints = this->prints .merge_presets(std::move(other.prints), this->vendors); - std::vector duplicate_sla_prints = this->sla_prints .merge_presets(std::move(other.sla_prints), this->vendors); - std::vector duplicate_filaments = this->filaments .merge_presets(std::move(other.filaments), this->vendors); - std::vector duplicate_sla_materials = this->sla_materials.merge_presets(std::move(other.sla_materials), this->vendors); - std::vector duplicate_printers = this->printers .merge_presets(std::move(other.printers), this->vendors); - append(this->obsolete_presets.prints, std::move(other.obsolete_presets.prints)); - append(this->obsolete_presets.sla_prints, std::move(other.obsolete_presets.sla_prints)); - append(this->obsolete_presets.filaments, std::move(other.obsolete_presets.filaments)); - append(this->obsolete_presets.sla_materials, std::move(other.obsolete_presets.sla_materials)); - append(this->obsolete_presets.printers, std::move(other.obsolete_presets.printers)); - append(duplicate_prints, std::move(duplicate_sla_prints)); - append(duplicate_prints, std::move(duplicate_filaments)); - append(duplicate_prints, std::move(duplicate_sla_materials)); - append(duplicate_prints, std::move(duplicate_printers)); - m_errors += other.m_errors; - return duplicate_prints; + for (PresetBundle *other : others) + this->vendors.insert(other->vendors.begin(), other->vendors.end()); + std::vector> duplicates(others.size()); + auto merge = [&](auto collection) { + std::vector other_collections; + for (PresetBundle *other : others) + other_collections.push_back(&(other->*collection)); + std::vector> collection_duplicates = (this->*collection).merge_presets(other_collections, this->vendors); + for (size_t i = 0; i < others.size(); ++ i) + append(duplicates[i], std::move(collection_duplicates[i])); + }; + merge(&PresetBundle::prints); + merge(&PresetBundle::sla_prints); + merge(&PresetBundle::filaments); + merge(&PresetBundle::sla_materials); + merge(&PresetBundle::printers); + for (PresetBundle *other : others) { + append(this->obsolete_presets.prints, std::move(other->obsolete_presets.prints)); + append(this->obsolete_presets.sla_prints, std::move(other->obsolete_presets.sla_prints)); + append(this->obsolete_presets.filaments, std::move(other->obsolete_presets.filaments)); + append(this->obsolete_presets.sla_materials, std::move(other->obsolete_presets.sla_materials)); + append(this->obsolete_presets.printers, std::move(other->obsolete_presets.printers)); + m_errors += other->m_errors; + } + return duplicates; } void PresetBundle::update_system_maps() @@ -6598,38 +6687,38 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(": finished"); } -// Orca: load one source-form preset entry — parsed from its JSON subfile just -// now, or deserialized from the vendor's cache; the code is shared so a -// cache-loaded bundle cannot come out different from a JSON-loaded one. -// Resolves `inherits` against the presets loaded before this one -// (config_maps) or against base_bundle's filament library, layers each -// `include` (include_maps) under the preset's own keys, flattens, validates -// and registers the preset. Returns the reason loading failed, empty on -// success. -std::string PresetBundle::load_vendor_preset( - const CachedPreset& entry, - const std::string& path, const std::string& vendor_name, - const PresetBundle* base_bundle, - LoadConfigBundleAttributes flags, - ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, - std::map& config_maps, std::map& include_maps, - std::map& filament_id_maps, - PresetCollection* presets_collection, size_t& count, bool is_from_lib, - const std::set* retain_configs, const std::set* retain_includes) +static ConfigurationError failed_loading_error(const std::string& file, const std::string& dir) { - const VendorProfile* current_vendor_profile = &this->vendors.at(vendor_name); - const std::string subfile = path + "/" + vendor_name + "/" + entry.sub_path; + return ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % file % dir).str()); +} + +void PresetBundle::log_errors(const std::vector& errors) +{ + for (const std::string& error : errors) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << error; + } +} + +PresetBundle::PresetInstall PresetBundle::resolve_vendor_preset(const CachedPreset& entry, const VendorInstall& install) const +{ + const std::string& path = install.path; + const std::string& vendor_name = install.vendor_name; + const PresetBundle* base_bundle = install.base_bundle; + const PresetCollection& presets_collection = *install.presets; + const VendorProfile& current_vendor_profile = *install.vendor_profile; const std::string& preset_name = entry.name; - std::string alias_name, filament_id = entry.filament_id; - std::vector renamed_from = entry.renamed_from; - DynamicPrintConfig config; const DynamicPrintConfig* default_config = nullptr; - std::string reason; + const bool retain = install.is_from_lib || install.inherited.count(preset_name) != 0; + + PresetInstall out; + out.filament_id = entry.filament_id; + out.renamed_from = entry.renamed_from; //check whether it inherits other preset or not if (! entry.inherits.empty()) { - auto it2 = config_maps.find(entry.inherits); - if (it2 != config_maps.end()) + auto it2 = install.config_maps.find(entry.inherits); + if (it2 != install.config_maps.end()) default_config = &(it2->second); if (default_config == nullptr && base_bundle != nullptr) { auto base_it2 = base_bundle->m_config_maps.find(entry.inherits); @@ -6637,115 +6726,98 @@ std::string PresetBundle::load_vendor_preset( default_config = &(base_it2->second); } if (default_config != nullptr) { - if (filament_id.empty() && (presets_collection->type() == Preset::TYPE_FILAMENT)) { - auto filament_id_map_iter = filament_id_maps.find(entry.inherits); - if (filament_id_map_iter != filament_id_maps.end()) { - filament_id = filament_id_map_iter->second; + if (out.filament_id.empty() && (presets_collection.type() == Preset::TYPE_FILAMENT)) { + auto filament_id_map_iter = install.filament_id_maps.find(entry.inherits); + if (filament_id_map_iter != install.filament_id_maps.end()) { + out.filament_id = filament_id_map_iter->second; } - if (filament_id.empty() && base_bundle != nullptr) { + if (out.filament_id.empty() && base_bundle != nullptr) { auto base_filament_id_map_iter = base_bundle->m_filament_id_maps.find(entry.inherits); if (base_filament_id_map_iter != base_bundle->m_filament_id_maps.end()) { - filament_id = base_filament_id_map_iter->second; + out.filament_id = base_filament_id_map_iter->second; } } } } else { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << preset_name; - // throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find inherits: " + entry.inherits; - return reason; + out.errors.push_back(std::string(__FUNCTION__) + ": can not find inherits " + entry.inherits + " for " + preset_name); + out.reason = "Can not find inherits: " + entry.inherits; + return out; } } else - default_config = &presets_collection->default_preset_for(entry.config_src).config; - config = *default_config; + default_config = &presets_collection.default_preset_for(entry.config_src).config; + out.config = *default_config; // Layer each included preset's own keys over the parent, in the order listed; // this preset's own keys go on top. for (const std::string& name : entry.includes) { - auto it = include_maps.find(name); - if (it == include_maps.end()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find include " << name << " for " << preset_name; + auto it = install.include_maps.find(name); + if (it == install.include_maps.end()) { + out.errors.push_back(std::string(__FUNCTION__) + ": can not find include " + name + " for " + preset_name); continue; } - config.apply(it->second); + out.config.apply(it->second); } - config.apply(entry.config_src); + out.config.apply(entry.config_src); // Record what a base states, its diff against the default, for the presets // that include it. It is taken before extend_default_config_length pads every // per-variant key to the base's variant count: the padded defaults would // otherwise override the values each includer inherits. - if (entry.instantiation == "false" && (retain_includes == nullptr || retain_includes->count(preset_name) != 0)) { - DynamicPrintConfig included; - included.apply_only(config, config.diff(presets_collection->default_preset_for(config).config)); - include_maps.emplace(preset_name, std::move(included)); + if (entry.instantiation == "false" && install.included.count(preset_name) != 0) { + out.included.emplace(); + out.included->apply_only(out.config, out.config.diff(presets_collection.default_preset_for(out.config).config)); } - extend_default_config_length(config, true, *default_config); + extend_default_config_length(out.config, true, *default_config); + // Report configuration fields, which are misplaced into a wrong group, before + // Preset::normalize derives keys from them. An include diff holds only keys of the + // collection default, so only the entry's own keys can be missing from default_config. + std::string incorrect_keys = Preset::remove_invalid_keys(out.config, *default_config, &entry.config_src); + if (!incorrect_keys.empty()) + out.errors.push_back(std::string(__FUNCTION__) + ": The config " + path + "/" + vendor_name + "/" + entry.sub_path + + " contains incorrect keys: " + incorrect_keys + ", which were removed"); if (entry.instantiation == "false" && "Template" != vendor_name) { - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } - - if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) - config_maps.emplace(preset_name, std::move(config)); - if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!filament_id.empty())) - filament_id_maps.emplace(preset_name, filament_id); - return reason; + out.config_only = true; + if (retain) + out.retained = std::move(out.config); + return out; } - if (config.has("alias")) - alias_name = (dynamic_cast(config.option("alias")))->value; - Preset::normalize(config); + if (out.config.has("alias")) + out.alias = (dynamic_cast(out.config.option("alias")))->value; - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } + Preset::normalize(out.config); - if (presets_collection->type() == Preset::TYPE_PRINTER) { + if (presets_collection.type() == Preset::TYPE_PRINTER) { // Filter out printer presets, which are not mentioned in the vendor profile. // These presets are considered not installed. - auto printer_model = config.opt_string("printer_model"); + auto printer_model = out.config.opt_string("printer_model"); if (printer_model.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer model, it will be ignored."; - reason = std::string("can not find printer_model"); - return reason; + out.errors.push_back("Error in a Vendor Config Bundle \"" + path + "\": The printer preset \"" + + preset_name + "\" defines no printer model, it will be ignored."); + out.reason = std::string("can not find printer_model"); + return out; } - auto printer_variant = config.opt_string("printer_variant"); + auto printer_variant = out.config.opt_string("printer_variant"); if (printer_variant.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer variant, it will be ignored."; - reason = std::string("can not find printer_variant"); - return reason; + out.errors.push_back("Error in a Vendor Config Bundle \"" + path + "\": The printer preset \"" + + preset_name + "\" defines no printer variant, it will be ignored."); + out.reason = std::string("can not find printer_variant"); + return out; } - auto it_model = std::find_if(current_vendor_profile->models.cbegin(), current_vendor_profile->models.cend(), + auto it_model = std::find_if(current_vendor_profile.models.cbegin(), current_vendor_profile.models.cend(), [&](const VendorProfile::PrinterModel &m) { return m.id == printer_model; } ); - if (it_model == current_vendor_profile->models.end()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer model \"" << printer_model << "\", it will be ignored."; - reason = std::string("can not find printer model in vendor profile"); - return reason; + if (it_model == current_vendor_profile.models.end()) { + out.errors.push_back("Error in a Vendor Config Bundle \"" + path + "\": The printer preset \"" + + preset_name + "\" defines invalid printer model \"" + printer_model + "\", it will be ignored."); + out.reason = std::string("can not find printer model in vendor profile"); + return out; } auto it_variant = it_model->variant(printer_variant); if (it_variant == nullptr) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer variant \"" << printer_variant << "\", it will be ignored."; - reason = std::string("can not find printer_variant in vendor profile"); - return reason; + out.errors.push_back("Error in a Vendor Config Bundle \"" + path + "\": The printer preset \"" + + preset_name + "\" defines invalid printer variant \"" + printer_variant + "\", it will be ignored."); + out.reason = std::string("can not find printer_variant in vendor profile"); + return out; } // An instantiation printer profile's nozzle_diameter must match the numeric (diameter) // prefix of its printer_variant: "0.4" -> {0.4}, "0.8HF" -> {0.8} (a trailing @@ -6756,7 +6828,7 @@ std::string PresetBundle::load_vendor_preset( // validated, not variant uniqueness. Validation-only so the app keeps loading existing // profiles unchanged. if (validation_mode && entry.instantiation == "true") { - const auto *nd = config.option("nozzle_diameter"); + const auto *nd = out.config.option("nozzle_diameter"); std::set nozzles, variant_nozzles; if (nd != nullptr) nozzles.insert(nd->values.begin(), nd->values.end()); @@ -6771,34 +6843,71 @@ std::string PresetBundle::load_vendor_preset( variant_nozzles.insert(d); } if (!variant_ok || variant_nozzles != nozzles) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" has printer_variant \"" << printer_variant << - "\" that does not match its nozzle_diameter \"" << (nd ? nd->serialize() : std::string()) << "\". " + out.errors.push_back("Error in a Vendor Config Bundle \"" + path + "\": The printer preset \"" + + preset_name + "\" has printer_variant \"" + printer_variant + + "\" that does not match its nozzle_diameter \"" + (nd ? nd->serialize() : std::string()) + "\". " "printer_variant must begin with the nozzle diameter, optionally followed by a non-numeric suffix " "(e.g. \"0.4\", \"0.8HF\"); for multi-nozzle printers, join the per-nozzle diameters with \"+\" in " - "nozzle order (e.g. \"0.4+0.6\")."; + "nozzle order (e.g. \"0.4+0.6\")."); } } } - const Preset *preset_existing = presets_collection->find_preset(preset_name, false); - if (preset_existing != nullptr) { + + // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. + if (out.alias.empty()) { + size_t end_pos = preset_name.find_first_of("@"); + if (end_pos != std::string::npos) { + out.alias = preset_name.substr(0, end_pos); + if (out.renamed_from.empty()) + // Add the preset name with the '@' character removed into the "renamed_from" list. + out.renamed_from.emplace_back(out.alias + preset_name.substr(end_pos + 1)); + boost::trim_right(out.alias); + } + } + + out.file_path = (boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR / vendor_name / entry.sub_path).make_preferred().string(); + if (validation_mode) + out.file_path = (boost::filesystem::path(data_dir()) / vendor_name / entry.sub_path).make_preferred().string(); + if (m_preserve_vendor_source_paths) + out.file_path = (boost::filesystem::path(path) / vendor_name / entry.sub_path).make_preferred().string(); + if (retain) + out.retained = out.config; + return out; +} + +std::string PresetBundle::commit_vendor_preset(const CachedPreset& entry, PresetInstall&& resolved, + ConfigSubstitutions&& substitutions, VendorInstall& install) +{ + const std::string& path = install.path; + const std::string& vendor_name = install.vendor_name; + PresetCollection* presets_collection = install.presets; + const VendorProfile* current_vendor_profile = install.vendor_profile; + const std::string& preset_name = entry.name; + + log_errors(resolved.errors); + if (resolved.included) + install.include_maps.emplace(preset_name, std::move(*resolved.included)); + if (! resolved.reason.empty()) + return resolved.reason; + + if (resolved.config_only) { + if (resolved.retained) + install.config_maps.emplace(preset_name, std::move(*resolved.retained)); + if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!resolved.filament_id.empty())) + install.filament_id_maps.emplace(preset_name, resolved.filament_id); + return std::string(); + } + + if (! install.installed_names.insert(preset_name).second) { ++m_errors; BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << preset_name << "\" has already been loaded from another Config Bundle."; - reason = std::string("duplicated defines"); - return reason; + return std::string("duplicated defines"); } - auto file_path = (boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR / vendor_name / entry.sub_path).make_preferred(); - if (validation_mode) - file_path = (boost::filesystem::path(data_dir()) / vendor_name / entry.sub_path).make_preferred(); - if (m_preserve_vendor_source_paths) - file_path = (boost::filesystem::path(path) / vendor_name / entry.sub_path).make_preferred(); - // Load the preset into the list of presets, save it to disk. - Preset &loaded = presets_collection->load_preset(file_path.string(), preset_name, std::move(config), false); - if (flags.has(LoadConfigBundleAttribute::LoadSystem)) { + Preset &loaded = presets_collection->append_preset(std::move(resolved.file_path), preset_name, std::move(resolved.config)); + if (install.flags.has(LoadConfigBundleAttribute::LoadSystem)) { loaded.is_system = true; loaded.vendor = current_vendor_profile; loaded.version = current_vendor_profile->config_version; @@ -6811,51 +6920,126 @@ std::string PresetBundle::load_vendor_preset( if (loaded.setting_id.empty() && entry.instantiation == "true") loaded.setting_id = generate_preset_setting_id( vendor_name, Preset::get_type_string(presets_collection->type()), preset_name); - loaded.filament_id = filament_id; - loaded.m_from_orca_filament_lib = is_from_lib; - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << " " << __LINE__ << ", " << loaded.name << " load filament_id: " << filament_id; + loaded.filament_id = resolved.filament_id; + loaded.m_from_orca_filament_lib = install.is_from_lib; if (presets_collection->type() == Preset::TYPE_FILAMENT) { - if (filament_id.empty() && "Template" != vendor_name) { + if (resolved.filament_id.empty() && "Template" != vendor_name) { ++m_errors; BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": can not find filament_id for " << preset_name; //throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find filament_id for " + preset_name; - return reason; + return "Can not find filament_id for " + preset_name; } else { - filament_id_maps.emplace(preset_name, filament_id); + install.filament_id_maps.emplace(preset_name, resolved.filament_id); } } } - // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. - if (alias_name.empty()) { - size_t end_pos = preset_name.find_first_of("@"); - if (end_pos != std::string::npos) { - alias_name = preset_name.substr(0, end_pos); - if (renamed_from.empty()) - // Add the preset name with the '@' character removed into the "renamed_from" list. - renamed_from.emplace_back(alias_name + preset_name.substr(end_pos + 1)); - boost::trim_right(alias_name); - } - } - if (alias_name.empty()) + if (resolved.alias.empty()) loaded.alias = preset_name; else { - loaded.alias = std::move(alias_name); + loaded.alias = std::move(resolved.alias); filaments.set_printer_hold_alias(loaded.alias, loaded); } - loaded.renamed_from = std::move(renamed_from); - if (! substitution_context.empty()) - substitutions.push_back({ + loaded.renamed_from = std::move(resolved.renamed_from); + if (! substitutions.empty()) + install.substitutions->push_back({ preset_name, presets_collection->type(), PresetConfigSubstitutions::Source::ConfigBundle, - std::string(), std::move(substitution_context.substitutions) }); - if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) - config_maps.emplace(preset_name, loaded.config); - ++count; + std::string(), std::move(substitutions) }); + if (resolved.retained) + install.config_maps.emplace(preset_name, std::move(*resolved.retained)); + ++install.count; //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", got preset %1%, from %2%")%loaded.name %subfile; - return reason; + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << ", got preset " << loaded.name << ", filament_id " << loaded.filament_id + << ", from " << path << "/" << vendor_name << "/" << entry.sub_path; + return std::string(); +} + +void PresetBundle::install_vendor_entries(VendorInstall& install, const std::vector& entries, + std::vector* parsed) +{ + assert(parsed == nullptr || parsed->size() == entries.size()); + for (const CachedPreset& entry : entries) { + if (! entry.inherits.empty()) + install.inherited.insert(entry.inherits); + install.included.insert(entry.includes.begin(), entry.includes.end()); + } + // The names an appended preset may not repeat, since the collection cannot be + // searched until it is sorted again. + for (const Preset& preset : install.presets->m_presets) + install.installed_names.insert(preset.name); + // Held until the collection is sorted again, also when an entry throws, so a reader + // holding the lock never finds it unsorted. + install.presets->lock(); + ScopeGuard sort_on_exit([&install] { + install.presets->sort_presets(); + install.presets->unlock(); + }); + + auto resolve = [&](size_t i) { return resolve_vendor_preset(entries[i], install); }; + auto commit = [&](size_t i, PresetInstall&& resolved) { + ConfigSubstitutions substitutions; + if (parsed != nullptr) { + log_errors((*parsed)[i].errors); + for (const std::string& warning : (*parsed)[i].warnings) + BOOST_LOG_TRIVIAL(error) << warning; + substitutions = std::move((*parsed)[i].substitutions); + } + const std::string reason = commit_vendor_preset(entries[i], std::move(resolved), std::move(substitutions), install); + if (! reason.empty()) { + ++m_errors; + const std::string subfile_path = install.path + "/" + install.vendor_name + "/" + entries[i].sub_path; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not install " << entries[i].name << " from " << subfile_path << ": " << reason; + throw failed_loading_error(subfile_path, install.path); + } + }; + // A run of entries resolves together and commits in listing order, and ends + // before an entry that inherits or includes one already in it. So no entry + // resolves against a registration from its own run, and the result matches + // installing one at a time. + std::unordered_set run_names; + size_t run_begin = 0; + auto install_run = [&](size_t run_end) { + resolve_then_commit(run_end - run_begin, + [&](size_t k) { return resolve(run_begin + k); }, + [&](size_t k, PresetInstall&& resolved) { commit(run_begin + k, std::move(resolved)); }); + run_names.clear(); + run_begin = run_end; + }; + auto in_run = [&](const std::string& name) { return run_names.count(name) != 0; }; + for (size_t i = 0; i < entries.size(); ++ i) { + const CachedPreset& entry = entries[i]; + if (in_run(entry.inherits) || std::any_of(entry.includes.begin(), entry.includes.end(), in_run)) + install_run(i); + if (install.inherited.count(entry.name) != 0 || install.included.count(entry.name) != 0) + run_names.insert(entry.name); + } + install_run(entries.size()); +} + +size_t PresetBundle::install_vendor(const std::string& path, const std::string& vendor_name, const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, const VendorCacheData& entries, + VendorParse* parsed, bool complete, PresetsConfigSubstitutions& substitutions) +{ + const bool is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; + const VendorProfile* vendor_profile = &this->vendors.at(vendor_name); + size_t count = 0; + auto install_collection = [&](const std::vector& list, std::vector* list_parsed, + PresetCollection* presets, bool is_from_lib) { + VendorInstall install { path, vendor_name, vendor_profile, base_bundle, flags, presets, is_from_lib, &substitutions }; + install_vendor_entries(install, list, list_parsed); + count += install.count; + return install; + }; + install_collection(entries.process_entries, parsed ? &parsed->process_entries : nullptr, &this->prints, false); + VendorInstall filaments = install_collection(entries.filament_entries, parsed ? &parsed->filament_entries : nullptr, + &this->filaments, is_orca_lib); + if (is_orca_lib && complete) { + m_config_maps = std::move(filaments.config_maps); + m_filament_id_maps = std::move(filaments.filament_id_maps); + } + install_collection(entries.machine_entries, parsed ? &parsed->machine_entries : nullptr, &this->printers, false); + return count; } //BBS: Load a config bundle file from json @@ -6863,12 +7047,14 @@ std::pair PresetBundle::load_vendor_configs_ const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle, bool allow_cache) { - // Enable substitutions for user config bundle, throw an exception when loading a system profile. - ConfigSubstitutionContext substitution_context { compatibility_rule }; - PresetsConfigSubstitutions substitutions; - // Errors already on this bundle when the load began; the cache stamp below - // counts only what this parse adds. - const int errors_at_entry = m_errors; + return this->install_vendor_read(this->read_vendor(dir, vendor_name, flags, compatibility_rule, allow_cache), base_bundle); +} + +PresetBundle::VendorRead PresetBundle::read_vendor(const std::string& dir, const std::string& vendor_name, + LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache) +{ + VendorRead read { dir, vendor_name, flags, compatibility_rule }; + read.errors_at_entry = m_errors; //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; @@ -6878,16 +7064,34 @@ std::pair PresetBundle::load_vendor_configs_ // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only // scans want a slice of one. Validation reads the JSONs whatever is cached. - const boost::filesystem::path dir_path(dir); - const bool cacheable = allow_cache && flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); - if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name, base_bundle)) { - size_t presets_loaded = 0; - for (const PresetCollection* coll : std::initializer_list{ - &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) - presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; - return std::make_pair(std::move(substitutions), presets_loaded); + read.cacheable = allow_cache && flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (read.cacheable && ! validation_mode) { + // A vendor is loaded from where it is installed and nowhere else; resources + // reaches the app by being installed into `dir` first. The cache there is + // judged against the profile beside it — or, where the cache is the whole + // of the installation, against nothing, since nothing on disk can then be + // newer than it. That state is Semver::inf(), which no real profile carries. + const boost::filesystem::path dir_path(dir); + const boost::filesystem::path profile = dir_path / (vendor_name + ".json"); + const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) : Semver::inf(); + read.cache_path = (dir_path / (vendor_name + ".opc")).string(); + read.from_cache = VendorCacheFile::load(read.cache_path, vendor_name, version, read.data); + if (read.from_cache) + return read; } + this->parse_vendor_json(read); + return read; +} + +void PresetBundle::parse_vendor_json(VendorRead& read) +{ + // Starts over from what read_vendor was asked for, since a cache that could + // not be installed leaves its own state behind. + read = VendorRead { std::move(read.dir), std::move(read.vendor_name), read.flags, read.compatibility_rule, + read.errors_at_entry, read.cacheable }; + const std::string& dir = read.dir; + const std::string& vendor_name = read.vendor_name; + LoadConfigBundleAttributes flags = read.flags; // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); @@ -7108,21 +7312,21 @@ std::pair PresetBundle::load_vendor_configs_ BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", loaded vendor profile, name %1%, id %2%, version %3%")%vendor_profile.name%vendor_profile.id%vendor_profile.config_version.to_string(); - if (flags.has(LoadConfigBundleAttribute::LoadVendorOnly)) - return std::make_pair(PresetsConfigSubstitutions{}, 0); + if (flags.has(LoadConfigBundleAttribute::LoadVendorOnly)) { + read.vendor_only = true; + return; + } // 3) paste the process/filament/print configs - PresetCollection *presets = nullptr; - size_t presets_loaded = 0; // Parse one subfile into a source-form entry — everything the JSON states, - // nothing resolved. Loading the entry (load_vendor_preset) is the - // same code whether the entry was parsed just now or deserialized from the + // nothing resolved. Installing the entries (install_vendor) is the + // same code whether they were parsed just now or deserialized from the // vendor's cache. - auto parse_subfile = [this, dir, vendor_name]( + auto parse_subfile = [&dir, &vendor_name]( ConfigSubstitutionContext& substitution_context, const std::pair& subfile_iter, - CachedPreset& entry) -> std::string { + CachedPreset& entry, std::vector& errors, std::vector& warnings) -> std::string { std::string subfile = dir + "/" + vendor_name + "/" + subfile_iter.second; std::string reason; @@ -7134,8 +7338,7 @@ std::pair PresetBundle::load_vendor_configs_ entry.sub_path = subfile_iter.second; entry.config_src.load_from_json(subfile, substitution_context, false, key_values, reason); if (!reason.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": load config file "< PresetBundle::load_vendor_configs_ entry.name = subfile_iter.first; } if(key_values.find(BBL_JSON_KEY_INSTANTIATION) == key_values.end()) - { - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": Missing instantiation attribute for " << entry.name; - ++m_errors; - } + errors.push_back(std::string(__FUNCTION__) + ": Missing instantiation attribute for " + entry.name); entry.instantiation = key_values[BBL_JSON_KEY_INSTANTIATION]; - if(entry.instantiation != "false" && entry.instantiation != "true"){ - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": Missing instantiation attribute for " << entry.name; - ++m_errors; - } + if(entry.instantiation != "false" && entry.instantiation != "true") + errors.push_back(std::string(__FUNCTION__) + ": Missing instantiation attribute for " + entry.name); auto setting_it = key_values.find(BBL_JSON_KEY_SETTING_ID); if (setting_it != key_values.end()) entry.setting_id = setting_it->second; @@ -7171,8 +7369,7 @@ std::pair PresetBundle::load_vendor_configs_ // An `inherits` key naming nothing can never resolve; fail it // here so install can key off the empty string as "no inherits". if (entry.inherits.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << entry.name; + errors.push_back(std::string(__FUNCTION__) + ": can not find inherits " + entry.inherits + " for " + entry.name); reason = "Can not find inherits: " + entry.inherits; return reason; } @@ -7185,96 +7382,108 @@ std::pair PresetBundle::load_vendor_configs_ for (const auto& name : includes) { if (name.is_string()) entry.includes.push_back(name.get()); - else { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": invalid include " << name.dump() << " for " << entry.name; - } + else + errors.push_back(std::string(__FUNCTION__) + ": invalid include " + name.dump() + " for " + entry.name); } } if (key_values.find(ORCA_JSON_KEY_RENAMED_FROM) != key_values.end()) { - if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], entry.renamed_from)) { - BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << dir << "\": The preset \"" << entry.name - << "\" contains invalid \"renamed_from\" key, which is being ignored."; - } + if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], entry.renamed_from)) + warnings.push_back("Error in a Config \"" + dir + "\": The preset \"" + entry.name + + "\" contains invalid \"renamed_from\" key, which is being ignored."); } } catch(nlohmann::detail::parse_error &err) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": parse "<< subfile <<" got a nlohmann::detail::parse_error, reason = " << err.what(); + errors.push_back(std::string(__FUNCTION__) + ": parse " + subfile + " got a nlohmann::detail::parse_error, reason = " + err.what()); reason = std::string("json parse error") + err.what(); return reason; } return reason; }; - std::map configs, include_maps; - std::map filament_id_maps; // Orca: whether to (re)write the vendor's cache after this parse, leaving it // in step with the profile so the next run reads it instead. It is written // where the vendor was looked for, even when the profile came from resources, // and stamped with the version that profile claims — a profile without one // cannot be judged for staleness later, and a cache nothing can invalidate is // worse than none. - const bool will_cache = cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); - VendorCacheData cache_data; + read.will_cache = read.cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); + read.version = vendor_profile.config_version.to_string(); + // Enable substitutions for user config bundle, throw an exception when loading a system profile. + ConfigSubstitutionContext substitution_context { read.compatibility_rule }; + // Parsed up to the first sub-file that fails, and the ones before it are + // installed before that failure is raised, since the filament library and a + // filament-only scan keep what a throwing load installed. + auto parse_subfiles = [&](const std::vector>& subfiles, + std::vector& entries, std::vector& entries_parsed, const char* kind) { + for (const auto& subfile : subfiles) { + CachedPreset entry; + std::vector errors, warnings; + read.reason = parse_subfile(substitution_context, subfile, entry, errors, warnings); + if (! read.reason.empty()) { + read.failed_subfile = subfile.second; + read.failed_kind = kind; + read.failed_errors = std::move(errors); + return false; + } + entries.emplace_back(std::move(entry)); + entries_parsed.push_back({ std::move(substitution_context.substitutions), std::move(errors), std::move(warnings) }); + } + return true; + }; + // One setter for every sub-file, so the ones load_from_json makes have + // nothing to set. + CNumericLocalesSetter locales_setter; + if (parse_subfiles(process_subfiles, read.data.process_entries, read.parsed.process_entries, "process") && + parse_subfiles(filament_subfiles, read.data.filament_entries, read.parsed.filament_entries, "filament")) + parse_subfiles(machine_subfiles, read.data.machine_entries, read.parsed.machine_entries, "printer"); +} + +std::pair PresetBundle::install_vendor_read(VendorRead&& read, const PresetBundle* base_bundle) +{ + const std::string& dir = read.dir; + const std::string& vendor_name = read.vendor_name; + PresetsConfigSubstitutions substitutions; + if (read.from_cache) { + if (this->install_vendor_cache(read.cache_path, vendor_name, std::move(read.data), base_bundle)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + this->parse_vendor_json(read); + } + if (read.vendor_only) + return std::make_pair(PresetsConfigSubstitutions{}, 0); + + int parse_errors = 0; + for (const std::vector* list : { &read.parsed.process_entries, &read.parsed.filament_entries, &read.parsed.machine_entries }) + for (const EntryParse& entry_parsed : *list) + parse_errors += int(entry_parsed.errors.size()); + const int errors_before_install = m_errors; + const size_t presets_loaded = install_vendor(dir, vendor_name, base_bundle, read.flags, read.data, &read.parsed, + read.reason.empty(), substitutions); // Errors added by install are counted apart: a cache load runs install again, // so the parse_errors stamped into the cache must hold only what a cache load // will not recount. - int install_errors = 0; - auto load_subfiles = [&](std::vector>& subfiles, - std::vector& entries, const char* kind, bool is_from_lib = false) { - configs.clear(); - include_maps.clear(); - filament_id_maps.clear(); - for (auto& subfile : subfiles) { - CachedPreset entry; - std::string reason = parse_subfile(substitution_context, subfile, entry); - if (reason.empty()) { - const int errors_before_install = m_errors; - reason = load_vendor_preset(entry, dir, vendor_name, base_bundle, flags, - substitution_context, substitutions, configs, include_maps, filament_id_maps, - presets, presets_loaded, is_from_lib); - install_errors += m_errors - errors_before_install; - } - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = dir + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % kind % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % dir).str()); - } - if (will_cache) - entries.emplace_back(std::move(entry)); - } - }; - - // The section order below — process, filaments (with the ORCA-lib map copy), - // printers — is mirrored by load_vendor_cache's install loops; keep the two - // in lockstep. - //3.1) paste the process - presets = &this->prints; - load_subfiles(process_subfiles, cache_data.process_entries, "process"); - - //3.2) paste the filaments - presets = &this->filaments; - const auto is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; - load_subfiles(filament_subfiles, cache_data.filament_entries, "filament", is_orca_lib); - if (is_orca_lib) { - m_config_maps = configs; - m_filament_id_maps = filament_id_maps; + const int install_errors = m_errors - errors_before_install - parse_errors; + if (! read.reason.empty()) { + log_errors(read.failed_errors); + ++m_errors; + //parse error + std::string subfile_path = dir + "/" + vendor_name + "/" + read.failed_subfile; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % read.failed_kind % subfile_path; + throw failed_loading_error(subfile_path, dir); } - //3.3) paste the printers - presets = &this->printers; - load_subfiles(machine_subfiles, cache_data.machine_entries, "printer"); - - if (will_cache) { + if (read.will_cache) { // Clamped: the count is a difference of three tallies, and a stamp that // wrapped would be added to every future load of this vendor. - cache_data.parse_errors = uint64_t(std::max(0, m_errors - errors_at_entry - install_errors)); - cache_data.vendors = this->vendors; - if (! VendorCacheFile::save((dir_path / (vendor_name + ".opc")).string(), vendor_name, - vendor_profile.config_version.to_string(), cache_data)) + read.data.parse_errors = uint64_t(std::max(0, m_errors - read.errors_at_entry - install_errors)); + read.data.vendors = this->vendors; + if (! VendorCacheFile::save((boost::filesystem::path(dir) / (vendor_name + ".opc")).string(), vendor_name, + read.version, read.data)) BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; } @@ -7989,82 +8198,42 @@ bool BundleMetadata::save_to_json(const std::string& path) const // ---- Per-vendor preset cache: install into this bundle ------------------- // The file format itself lives in PresetCacheFormat.cpp (VendorCacheFile). -bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle) -{ - // A vendor is loaded from where it is installed and nowhere else; resources - // reaches the app by being installed into `dir` first. The cache there is - // judged against the profile beside it — or, where the cache is the whole - // of the installation, against nothing, since nothing on disk can then be - // newer than it. That state is Semver::inf(), which no real profile carries. - const boost::filesystem::path profile = dir / (vendor_name + ".json"); - const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) - : Semver::inf(); - return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, base_bundle); -} - bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, const Semver& expected_vendor_version, const PresetBundle* base_bundle) { - // What this bundle had counted before the cache was tried. The caller - // measures its own parse against this same baseline, so a rejection must - // put it back rather than reset it to zero. - const int errors_at_entry = this->m_errors; // Read and validated before this bundle is touched: a rejected file leaves // no state to roll back. VendorCacheData data; if (! VendorCacheFile::load(cache_path, expected_vendor_name, expected_vendor_version, data)) return false; + // VendorCacheFile::load checked the names match. + return this->install_vendor_cache(cache_path, expected_vendor_name, std::move(data), base_bundle); +} + +bool PresetBundle::install_vendor_cache(const std::string& cache_path, const std::string& vendor_name, VendorCacheData&& data, + const PresetBundle* base_bundle) +{ + // What this bundle had counted before the cache was tried. The caller + // measures its own parse against this same baseline, so a rejection must + // put it back rather than reset it to zero. + const int errors_at_entry = this->m_errors; try { - const std::string& vendor_name = expected_vendor_name; // VendorCacheFile::load checked they match this->vendors = std::move(data.vendors); // What the parse counted before install took over; install recounts its // own below, so m_errors comes out as a JSON parse would leave it. m_errors += int(data.parse_errors); - // Install the entries exactly as load_vendor_configs_from_json installs - // them straight after parsing — same code, same order. The substitution - // context stays empty (the entries were substituted when they were - // parsed), so no substitutions are reported, as before. - ConfigSubstitutionContext substitution_context { ForwardCompatibilitySubstitutionRule::EnableSilent }; + // Stays empty, since the entries were substituted when they were parsed. PresetsConfigSubstitutions substitutions; - std::map configs, include_maps; - std::map filament_id_maps; - const std::string path = boost::filesystem::path(cache_path).parent_path().string(); - size_t count = 0; - auto install_entries = [&](const std::vector& entries, PresetCollection* presets, bool is_from_lib) { - configs.clear(); - include_maps.clear(); - filament_id_maps.clear(); - // Only configs of presets that other entries inherit or include are - // ever looked up again; registering just those skips one full config - // copy for every leaf preset. The library's filaments are all retained - // — they become the m_config_maps other vendors resolve against. - std::set inherited, included; - for (const CachedPreset& entry : entries) { - if (! entry.inherits.empty()) - inherited.insert(entry.inherits); - included.insert(entry.includes.begin(), entry.includes.end()); - } - const std::set* retain_configs = is_from_lib ? nullptr : &inherited; - for (const CachedPreset& entry : entries) { - const std::string reason = load_vendor_preset(entry, path, vendor_name, - base_bundle, LoadConfigBundleAttribute::LoadSystem, substitution_context, substitutions, - configs, include_maps, filament_id_maps, presets, count, is_from_lib, retain_configs, &included); - if (! reason.empty()) - throw std::runtime_error("entry " + entry.name + " failed to install: " + reason); - } - }; - install_entries(data.process_entries, &this->prints, false); - const bool is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; - install_entries(data.filament_entries, &this->filaments, is_orca_lib); - if (is_orca_lib) { - m_config_maps = configs; - m_filament_id_maps = filament_id_maps; - } - install_entries(data.machine_entries, &this->printers, false); + install_vendor(boost::filesystem::path(cache_path).parent_path().string(), vendor_name, base_bundle, + LoadConfigBundleAttribute::LoadSystem, data, nullptr, true, substitutions); return true; } catch (const std::exception& e) { + // A cancellation of the caller's task group stops the install without + // anything being wrong with the cache. + if (tbb::is_current_task_group_canceling()) + throw; BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); // Restore a clean state so the caller can fall back to the JSON parse. this->reset(false); diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 741f6069a4..0bf1bbd5a5 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -622,36 +623,180 @@ public: // default_filament_profile must resolve to a system filament. bool check_printer_default_materials() const; - // Merge one vendor's presets with the other vendor's presets, report duplicates. - // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a - // bundle out of several per-vendor caches loaded into separate PresetBundle instances. - std::vector merge_presets(PresetBundle &&other); + // One vendor to load, and the directory it is installed in. + struct VendorSource + { + std::string name; + boost::filesystem::path dir; + }; + + // Load `vendors` into this bundle, the Orca filament library directly and every + // other vendor in parallel into a bundle of its own that inherits from it, merged + // in the order given. A vendor that cannot be loaded has its error added to the + // returned text, or thrown in validation mode, and its name to `failed`; it is + // left out, except for the library, which keeps what it installed before the + // failure. Once `cancel` is set, no further vendor starts loading. + std::pair load_vendors(const std::vector& vendors, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache, + const std::atomic* cancel = nullptr, std::vector* failed = nullptr); private: - // Load one vendor from the preset cache installed in `dir`, judged against - // the vendor profile there. False, with this bundle left clean, when there - // is no usable cache and the vendor has to be parsed. This is how - // load_vendor_configs_from_json reads a cache. - bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle); + // Move the presets and vendor profiles of `others` into this bundle, in one pass + // over each collection. A preset whose name this bundle or an earlier one of + // `others` already has is left out and listed under the bundle that repeats it. + std::vector> merge_presets(const std::vector &others); - // Load one source-form preset entry into this bundle: resolve `inherits` - // and `include`, flatten, validate and register the preset. Returns the - // reason loading failed, empty on success. See the definition for the - // sharing contract between the JSON parse and the cache load. - // retain_configs / retain_includes, when non-null, name the only presets - // registered into config_maps / include_maps (a config copy each). The - // cache load passes the names its entries inherit / include — the only - // ones ever looked up again; the JSON parse retains all, not knowing what - // later subfiles name. - std::string load_vendor_preset(const CachedPreset& entry, - const std::string& path, const std::string& vendor_name, - const PresetBundle* base_bundle, - LoadConfigBundleAttributes flags, - ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, - std::map& config_maps, std::map& include_maps, - std::map& filament_id_maps, - PresetCollection* presets_collection, size_t& count, bool is_from_lib, - const std::set* retain_configs = nullptr, const std::set* retain_includes = nullptr); + // What parsing one entry's JSON sub-file reported. Its errors and warnings are + // logged when the entry installs, so they come out in listing order with the + // entry's install errors, as parsing and installing one entry at a time leaves them. + struct EntryParse + { + ConfigSubstitutions substitutions; + // Counted in the bundle's error count. + std::vector errors; + std::vector warnings; + }; + + // An EntryParse for each entry of the VendorCacheData list of the same name. + struct VendorParse + { + std::vector process_entries; + std::vector filament_entries; + std::vector machine_entries; + }; + + // One vendor read from its cache or its JSONs by read_vendor, for + // install_vendor_read to install. + struct VendorRead + { + std::string dir; + std::string vendor_name; + LoadConfigBundleAttributes flags; + ForwardCompatibilitySubstitutionRule compatibility_rule; + // The errors this bundle had counted before the read, which the cache stamp leaves out. + int errors_at_entry { 0 }; + // A whole-vendor load, which can read a cache and write one. + bool cacheable { false }; + // Read from the cache at cache_path, which install can still reject. + bool from_cache { false }; + std::string cache_path; + // Only the vendor profile was asked for. + bool vendor_only { false }; + VendorCacheData data; + // What each entry's JSON parse reported, and whether and with which version + // the cache is written once the entries install. + VendorParse parsed; + bool will_cache { false }; + std::string version; + // The sub-file the parse stopped at, its kind, why, and the errors it reported. + std::string reason; + std::string failed_subfile; + const char* failed_kind { nullptr }; + std::vector failed_errors; + }; + + // Read a vendor into this bundle's vendor profiles and `data`, from its cache + // when one covers it, else from its JSONs; nothing is installed. Throws + // ConfigurationError when the vendor's own JSON cannot be parsed. + VendorRead read_vendor(const std::string& dir, const std::string& vendor_name, LoadConfigBundleAttributes flags, + ForwardCompatibilitySubstitutionRule compatibility_rule, bool allow_cache); + + // Parse the vendor's JSONs into `read`, up to the first sub-file that fails. + void parse_vendor_json(VendorRead& read); + + // Install what read_vendor read, against base_bundle's filament library. A cache + // that cannot be installed is replaced by a parse of the JSONs. Throws + // ConfigurationError at the first entry that cannot be installed, or after + // installing the entries before a sub-file that could not be parsed. + std::pair install_vendor_read(VendorRead&& read, const PresetBundle* base_bundle); + + // Install a cache's entries. False, with this bundle left clean, when one of them + // cannot be installed. + bool install_vendor_cache(const std::string& cache_path, const std::string& vendor_name, VendorCacheData&& data, + const PresetBundle* base_bundle); + + // Log and count errors reported by a resolve or a parse. + void log_errors(const std::vector& errors); + + // The state of installing one collection of one vendor, which + // resolve_vendor_preset reads through a const reference and only + // commit_vendor_preset writes. + struct VendorInstall + { + // The directory holding /, whose sub-paths the entries name. + std::string path; + std::string vendor_name; + const VendorProfile* vendor_profile; + const PresetBundle* base_bundle; + LoadConfigBundleAttributes flags; + PresetCollection* presets; + // The Orca filament library, which keeps every config for other vendors to + // resolve against. + bool is_from_lib; + PresetsConfigSubstitutions* substitutions; + // The names some entry inherits / includes, the only ones whose configs / + // include diffs are looked up again. + std::set inherited; + std::set included; + std::map config_maps; + std::map include_maps; + std::map filament_id_maps; + std::unordered_set installed_names; + size_t count { 0 }; + }; + + // Install a vendor's source-form entries, parsed from its JSON or read from its + // cache: processes, then filaments, then printers. Both loads go through here, + // so a cache-loaded bundle cannot come out different from a JSON-loaded one. + // `parsed` is given for entries parsed just now. `complete` says the entries + // are the vendor's whole lists; only then are the filament library's configs + // and filament ids left in m_config_maps and m_filament_id_maps. Returns the + // number of presets installed, and throws ConfigurationError at the first + // entry that cannot be installed. + size_t install_vendor(const std::string& path, const std::string& vendor_name, const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, const VendorCacheData& entries, + VendorParse* parsed, bool complete, PresetsConfigSubstitutions& substitutions); + + // Install one collection's entries in the order they are listed. + void install_vendor_entries(VendorInstall& install, const std::vector& entries, + std::vector* parsed); + + // One entry flattened against the preset it inherits, before anything this + // bundle shares has been touched. + struct PresetInstall + { + DynamicPrintConfig config; + std::string file_path; + // Empty when the preset is its own alias. + std::string alias; + std::string filament_id; + std::vector renamed_from; + // Reported by commit, so resolving entries together leaves the log and + // the error count as one entry at a time produces them. + std::vector errors; + // What a base states for the presets that include it, when it is retained. + std::optional included; + // The config kept for the entries that inherit this one, or for other + // vendors when this is the filament library. + std::optional retained; + // Not instantiated, so it contributes a config and no preset. + bool config_only { false }; + // Non-empty when the entry is rejected, and says why. + std::string reason; + }; + + // Flatten one entry against the config it inherits, from this collection's + // config_maps or base_bundle's filament library, with the include diffs it + // names layered in. It looks up nothing but the names the entry inherits and + // includes, and writes nothing. + PresetInstall resolve_vendor_preset(const CachedPreset& entry, const VendorInstall& install) const; + + // Install a resolved entry. The collections, the maps in `install` and the + // error count are touched here and only here, one entry at a time. + // Presets are appended, so a repeated name is caught with `installed_names`, + // and the collection is sorted once every entry is in. + std::string commit_vendor_preset(const CachedPreset& entry, PresetInstall&& resolved, + ConfigSubstitutions&& substitutions, VendorInstall& install); // Clear every collection's m_printer_hold_alias, which reset() leaves alone. void clear_printer_hold_aliases(); diff --git a/src/libslic3r/PresetCacheFormat.hpp b/src/libslic3r/PresetCacheFormat.hpp index 2c90c07882..52c40bea21 100644 --- a/src/libslic3r/PresetCacheFormat.hpp +++ b/src/libslic3r/PresetCacheFormat.hpp @@ -109,7 +109,7 @@ void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict); // One preset as its JSON subfile states it: the config diff, the names of the // preset it inherits and the presets it includes, and the parse metadata — -// everything the parse phase of load_vendor_configs_from_json extracts and +// everything PresetBundle::parse_vendor_json extracts and // nothing it derives. Inheritance and includes are resolved when the entry is // installed, against whatever filament library is loaded then, so a cache // carries no other vendor's values and no other vendor's update can make it diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index e9e74b798e..5370509587 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -1405,23 +1405,26 @@ bool GuideFrame::BuildProfileDataFromVendors() // is served from the shipped profiles. Each is stamped by name and // version alone: a profile change requires a version bump, so those two // determine content wherever the vendor's copy sits. - struct VendorSource { std::string name; boost::filesystem::path dir; std::string version; }; - std::vector ordered; - auto add_vendor = [&ordered](const std::string& name, const boost::filesystem::path& dir) { + std::vector ordered; + json stamps = json::array(); + auto add_vendor = [&ordered, &stamps](const std::string& name, const boost::filesystem::path& dir) { // The version a load from `dir` would serve: the profile's where one // exists (a cache is only served while it covers the profile beside // it), the cache's own stamp where the cache is the whole vendor. // A profile without a version (blacklist.json) carries no presets // and is passed over. const boost::filesystem::path profile = dir / (name + ".json"); + std::string version; if (boost::filesystem::exists(profile)) { const Semver v = get_version_from_json(profile.string()); - if (v.valid()) - ordered.push_back({name, dir, v.to_string()}); + if (! v.valid()) + return; + version = v.to_string(); } else { - ordered.push_back({name, dir, - VendorCacheFile::peek_version((dir / (name + ".opc")).string(), name)}); + version = VendorCacheFile::peek_version((dir / (name + ".opc")).string(), name); } + ordered.push_back({name, dir}); + stamps.push_back({name, version}); }; const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); if (auto it = vendor_sources.find(filament_library); it != vendor_sources.end()) @@ -1431,9 +1434,6 @@ bool GuideFrame::BuildProfileDataFromVendors() add_vendor(name, dir); if (ordered.empty()) return false; - json stamps = json::array(); - for (const VendorSource& v : ordered) - stamps.push_back({v.name, v.version}); // What this function derives is a pure function of that stamped set, so // the derived JSON is cached whole: a fresh cache makes an open one @@ -1461,26 +1461,18 @@ bool GuideFrame::BuildProfileDataFromVendors() } // Each vendor comes from its preset cache where one covers it, which is - // what makes this worth doing instead of the scan below; loading into a - // bundle per vendor keeps the install order the startup path has. - PresetBundle bundle; - auto load_vendor = [](PresetBundle& into, const std::string& vendor, - const boost::filesystem::path& dir, const PresetBundle* base) { - into.load_vendor_configs_from_json(dir.string(), vendor, PresetBundle::LoadSystem, - ForwardCompatibilitySubstitutionRule::EnableSilent, base); - }; - for (const VendorSource& v : ordered) { - if (*m_cancel_token) - return false; // as in the scan below: a vendor without a cache is parsed, and that takes time - if (v.name == filament_library) { - load_vendor(bundle, v.name, v.dir, nullptr); - } else { - PresetBundle tmp; - load_vendor(tmp, v.name, v.dir, &bundle); - bundle.merge_presets(std::move(tmp)); - } - } - if (bundle.vendors.empty()) + // what makes this worth doing instead of the scan below. + PresetBundle bundle; + std::vector failed; + const std::string errors = bundle.load_vendors(ordered, ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/true, m_cancel_token.get(), &failed).second; + if (*m_cancel_token || bundle.vendors.empty()) + return false; + if (! errors.empty()) + BOOST_LOG_TRIVIAL(warning) << "GuideFrame: loading the vendors reported: " << errors; + // A vendor that failed to load sends this open to the scan below, which lists + // what it can read of every vendor. + if (! failed.empty()) return false; if (! BuildProfileJson(bundle, /*require_all_resource_vendors=*/false)) return false; diff --git a/tests/libslic3r/CMakeLists.txt b/tests/libslic3r/CMakeLists.txt index 09120cd907..911e23d5c7 100644 --- a/tests/libslic3r/CMakeLists.txt +++ b/tests/libslic3r/CMakeLists.txt @@ -16,8 +16,10 @@ add_executable(${_TEST_NAME}_tests test_clipper_utils.cpp test_config.cpp test_config_variant_expansion.cpp + test_locales_utils.cpp test_drc.cpp test_toolordering_nozzle_group.cpp + test_parallel_resolve.cpp test_preset_bundle_loading.cpp test_preset_setting_id.cpp test_preset_diff.cpp diff --git a/tests/libslic3r/test_locales_utils.cpp b/tests/libslic3r/test_locales_utils.cpp new file mode 100644 index 0000000000..6f1528649d --- /dev/null +++ b/tests/libslic3r/test_locales_utils.cpp @@ -0,0 +1,69 @@ +#include + +#include + +#include "libslic3r/LocalesUtils.hpp" + +using namespace Slic3r; + +namespace { + +// Switches this thread's numeric locale to one whose decimal separator is a comma, +// when the system has one installed. +struct CommaNumericLocale +{ +#ifdef _WIN32 + bool apply() + { + for (const char* name : { "de-DE", "German_Germany.1252" }) + if (std::setlocale(LC_NUMERIC, name) != nullptr) + return true; + return false; + } +#else + locale_t locale { (locale_t) 0 }; + bool apply() + { + for (const char* name : { "de_DE.UTF-8", "de_DE.utf8", "de_DE" }) + if ((locale = newlocale(LC_NUMERIC_MASK, name, (locale_t) 0)) != (locale_t) 0) { + uselocale(locale); + return true; + } + return false; + } + // Freed once the setters around apply() have put the thread's own locale back. + ~CommaNumericLocale() + { + if (locale != (locale_t) 0) + freelocale(locale); + } +#endif +}; + +} // namespace + +TEST_CASE("a setter nested in another leaves the C locale in place for the outer one", "[LocalesUtils]") +{ + CNumericLocalesSetter outer; + { + CNumericLocalesSetter inner; + CHECK(is_decimal_separator_point()); + } + CHECK(is_decimal_separator_point()); +} + +TEST_CASE("a setter nested in another sets C again when the locale changed between them", "[LocalesUtils]") +{ + CommaNumericLocale comma; + { + CNumericLocalesSetter outer; + if (! comma.apply()) + SKIP("no locale with a comma decimal separator is installed"); + REQUIRE_FALSE(is_decimal_separator_point()); + { + CNumericLocalesSetter inner; + CHECK(is_decimal_separator_point()); + } + CHECK_FALSE(is_decimal_separator_point()); + } +} diff --git a/tests/libslic3r/test_parallel_resolve.cpp b/tests/libslic3r/test_parallel_resolve.cpp new file mode 100644 index 0000000000..76811f1475 --- /dev/null +++ b/tests/libslic3r/test_parallel_resolve.cpp @@ -0,0 +1,122 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "libslic3r/ParallelResolve.hpp" + +using namespace Slic3r; + +namespace { + +thread_local int t_live_setups = 0; + +// Counts how many are alive on the constructing thread. +struct CountingSetup +{ + CountingSetup() { ++ t_live_setups; } + ~CountingSetup() { -- t_live_setups; } +}; + +std::vector first_indices(size_t count) +{ + std::vector indices(count); + std::iota(indices.begin(), indices.end(), size_t(0)); + return indices; +} + +} // namespace + +TEST_CASE("every item resolves once and commits in index order on the calling thread", "[ParallelResolve]") +{ + const size_t count = 200; + std::vector resolves(count, 0); + std::vector committed, values; + std::vector on_caller; + const std::thread::id caller = std::this_thread::get_id(); + resolve_then_commit(count, + [&](size_t i) { ++ resolves[i]; return i * 3; }, + [&](size_t i, size_t resolved) { + committed.push_back(i); + values.push_back(resolved); + on_caller.push_back(std::this_thread::get_id() == caller); + }); + + CHECK(committed == first_indices(count)); + for (size_t i = 0; i < count; ++ i) { + CHECK(resolves[i] == 1); + CHECK(values[i] == i * 3); + CHECK(on_caller[i]); + } +} + +TEST_CASE("every item resolves inside one chunk setup", "[ParallelResolve]") +{ + const size_t count = 200; + std::vector live(count, 0); + resolve_then_commit(count, + [&](size_t i) { live[i] = t_live_setups; return 0; }, + [](size_t, int) {}); + + for (size_t i = 0; i < count; ++ i) + CHECK(live[i] == 1); +} + +TEST_CASE("an exception from resolve leaves the batches before it committed", "[ParallelResolve]") +{ + const size_t count = 200, fails_at = 150; + REQUIRE(fails_at >= resolve_batch_size); + std::vector committed; + CHECK_THROWS_AS(resolve_then_commit(count, + [&](size_t i) { + if (i == fails_at) + throw std::runtime_error("resolve failed"); + return i; + }, + [&](size_t i, size_t) { committed.push_back(i); }), + std::runtime_error); + + CHECK(committed == first_indices(fails_at / resolve_batch_size * resolve_batch_size)); +} + +TEST_CASE("an exception from commit stops at the item that threw", "[ParallelResolve]") +{ + const size_t count = 200, fails_at = 90; + std::vector committed; + CHECK_THROWS_AS(resolve_then_commit(count, + [](size_t i) { return i; }, + [&](size_t i, size_t) { + if (i == fails_at) + throw std::runtime_error("commit failed"); + committed.push_back(i); + }), + std::runtime_error); + + CHECK(committed == first_indices(fails_at)); +} + +TEST_CASE("a canceled task group stops before committing an item it did not resolve", "[ParallelResolve]") +{ + std::vector committed; + bool threw = false; + tbb::task_group_context context; + tbb::parallel_for(tbb::blocked_range(0, 1), [&](const tbb::blocked_range&) { + context.cancel_group_execution(); + try { + resolve_then_commit(200, + [](size_t i) { return i + 1; }, + [&](size_t, size_t resolved) { committed.push_back(resolved); }); + } catch (const std::runtime_error&) { + threw = true; + } + }, context); + + CHECK(threw); + CHECK(committed.empty()); +} diff --git a/tests/libslic3r/test_preset_bundle_loading.cpp b/tests/libslic3r/test_preset_bundle_loading.cpp index e71c8c5451..49f0f90214 100644 --- a/tests/libslic3r/test_preset_bundle_loading.cpp +++ b/tests/libslic3r/test_preset_bundle_loading.cpp @@ -59,6 +59,16 @@ void write_preset_with_inherits(const DynamicPrintConfig &default_config, const config.save_to_json(file.string(), name, "User", "1.0.0"); } +// A user preset file stating nothing but the preset it inherits, so every value it +// ends up with came from resolving that parent. +void write_minimal_child(const fs::path &file, const std::string &name, const std::string &inherits) +{ + fs::create_directories(file.parent_path()); + std::ofstream(file.string()) + << R"({"type":"process","name":")" << name << R"(","from":"User","version":"1.0.0","inherits":")" + << inherits << R"("})"; +} + // Add an in-memory preset (no file) with the given inherits value (empty => root preset). Preset &add_inmemory_preset(PresetCollection &coll, const std::string &name, const std::string &inherits = {}) { @@ -261,6 +271,162 @@ TEST_CASE("Selected printer uses its default or saved bed type", "[Preset][Bundl CHECK(app_config.get_printer_setting("Test Printer", "curr_bed_type") == std::to_string(static_cast(expected_bed_type))); } +TEST_CASE("A directory of user presets loads with each one resolved against its parent", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + RenameTestCollection coll; + + Preset &parent = add_inmemory_preset(coll, "Parent Process"); + parent.config.option("layer_height", true)->value = 0.24; + parent.is_system = true; + + constexpr int children = 400; + for (int i = 0; i < children; ++ i) + write_minimal_child(temp_dir.path() / PRESET_PRINT_NAME / ("Child " + std::to_string(i) + ".json"), + "Child " + std::to_string(i), "Parent Process"); + + PresetsConfigSubstitutions substitutions; + coll.load_presets(temp_dir.path().string(), PRESET_PRINT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::Disable); + + CHECK(coll.size() == size_t(children) + 2); // the children, the default preset and the parent + CHECK(coll.error_count() == 0); + for (int i = 0; i < children; ++ i) { + const Preset *child = coll.find_preset("Child " + std::to_string(i)); + REQUIRE(child != nullptr); + CHECK(child->inherits() == "Parent Process"); + CHECK(child->alias == "Child " + std::to_string(i)); + CHECK(child->loaded); + REQUIRE(child->config.option("layer_height") != nullptr); + CHECK_THAT(child->config.opt_float("layer_height"), Catch::Matchers::WithinAbs(0.24, 1e-9)); + } +} + +TEST_CASE("Repeated loads of a user preset directory produce the same presets", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + + auto seed_directory = [&]() { + for (int i = 0; i < 200; ++ i) + write_minimal_child(temp_dir.path() / PRESET_PRINT_NAME / ("Child " + std::to_string(i) + ".json"), + "Child " + std::to_string(i), "Parent Process"); + }; + + std::vector> names_per_run; + for (int run = 0; run < 3; ++ run) { + RenameTestCollection coll; + Preset &parent = add_inmemory_preset(coll, "Parent Process"); + parent.is_system = true; + if (run == 0) + seed_directory(); + + PresetsConfigSubstitutions substitutions; + coll.load_presets(temp_dir.path().string(), PRESET_PRINT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::Disable); + + std::vector names; + for (auto it = coll.begin(); it != coll.end(); ++ it) + names.push_back(it->name + "|" + it->alias + "|" + it->inherits()); + names_per_run.push_back(std::move(names)); + } + + REQUIRE(names_per_run[0].size() > 200); + CHECK(names_per_run[1] == names_per_run[0]); + CHECK(names_per_run[2] == names_per_run[0]); +} + +TEST_CASE("An unreadable user preset is counted and removed while the rest still load", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + RenameTestCollection coll; + const fs::path dir = temp_dir.path() / PRESET_PRINT_NAME; + + for (int i = 0; i < 20; ++ i) + write_preset_with_inherits(coll.default_preset().config, dir / ("Good " + std::to_string(i) + ".json"), + "Good " + std::to_string(i), std::string()); + fs::create_directories(dir); + std::ofstream((dir / "Broken.json").string()) << "{not-json"; + + PresetsConfigSubstitutions substitutions; + coll.load_presets(temp_dir.path().string(), PRESET_PRINT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + CHECK(coll.error_count() == 1); + CHECK(coll.find_preset("Broken") == nullptr); + CHECK_FALSE(fs::exists(dir / "Broken.json")); + for (int i = 0; i < 20; ++ i) + CHECK(coll.find_preset("Good " + std::to_string(i)) != nullptr); +} + +TEST_CASE("A user filament naming no compatible printer gets the one after its @, in memory and on disk", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + PresetBundle bundle; + const fs::path file = temp_dir.path() / PRESET_FILAMENT_NAME / "My PLA @Test Printer.json"; + REQUIRE(bundle.filaments.default_preset().config.option("compatible_printers")->values.empty()); + write_preset_with_inherits(bundle.filaments.default_preset().config, file, "My PLA @Test Printer", std::string()); + + PresetsConfigSubstitutions substitutions; + bundle.filaments.load_presets(temp_dir.path().string(), PRESET_FILAMENT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + const std::vector expected { "Test Printer" }; + const Preset *preset = bundle.filaments.find_preset("My PLA @Test Printer"); + REQUIRE(preset != nullptr); + CHECK(preset->config.option("compatible_printers")->values == expected); + + DynamicPrintConfig saved; + std::map key_values; + std::string reason; + saved.load_from_json(file.string(), ForwardCompatibilitySubstitutionRule::EnableSilent, key_values, reason); + REQUIRE(reason.empty()); + REQUIRE(saved.option("compatible_printers") != nullptr); + CHECK(saved.option("compatible_printers")->values == expected); +} + +TEST_CASE("A user preset's setting id equal to its base id is dropped in memory, not in the .info written back", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + PresetBundle bundle; + const fs::path file = temp_dir.path() / PRESET_FILAMENT_NAME / "My PLA @Test Printer.json"; + write_preset_with_inherits(bundle.filaments.default_preset().config, file, "My PLA @Test Printer", std::string()); + fs::path info = file; + info.replace_extension(".info"); + std::ofstream(info.string()) << "sync_info = \nuser_id = \nsetting_id = PFUS1\nbase_id = PFUS1\nupdated_time = 0\n"; + + PresetsConfigSubstitutions substitutions; + bundle.filaments.load_presets(temp_dir.path().string(), PRESET_FILAMENT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + const Preset *preset = bundle.filaments.find_preset("My PLA @Test Printer"); + REQUIRE(preset != nullptr); + CHECK(preset->setting_id.empty()); + CHECK(preset->base_id == "PFUS1"); + Preset reloaded(Preset::TYPE_FILAMENT, "My PLA @Test Printer"); + reloaded.load_info(info.string()); + CHECK(reloaded.setting_id == "PFUS1"); +} + +TEST_CASE("A user preset that is not loaded still reports its substituted values", "[Preset][Bundle]") +{ + ScopedTemporaryDir temp_dir; + RenameTestCollection coll; + const fs::path dir = temp_dir.path() / PRESET_PRINT_NAME; + fs::create_directories(dir); + std::ofstream((dir / "Orphan.json").string()) + << R"({"type":"process","name":"Orphan","from":"User","version":"1.0.0","inherits":"No Such Parent",)" + << R"("wall_generator":"no_such_generator"})"; + + PresetsConfigSubstitutions substitutions; + coll.load_presets(temp_dir.path().string(), PRESET_PRINT_NAME, substitutions, + ForwardCompatibilitySubstitutionRule::Enable); + + CHECK(coll.find_preset("Orphan") == nullptr); + CHECK(coll.error_count() == 1); + REQUIRE(substitutions.size() == 1); + CHECK(substitutions.front().preset_name == "Orphan"); +} + TEST_CASE("find_preset resolves a system preset's renamed_from", "[Preset][Rename]") { RenameTestCollection coll; diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp index 93d2b2e87a..585595ace9 100644 --- a/tests/libslic3r/test_vendor_cache.cpp +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -3,12 +3,21 @@ #include #include #include +#include +#include #include #include #include +#include +#include +#include #include #include +#include +#include +#include + #include "libslic3r/PresetBundle.hpp" #include "libslic3r/PresetCacheFormat.hpp" #include "libslic3r/Preset.hpp" @@ -52,6 +61,27 @@ void write_vendor_tree(const fs::path& dir, const std::string& vendor, const std << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; } +// A vendor whose process list is `processes`, each a preset name and the text of +// its sub-file, listed in that order. +void write_process_vendor(const fs::path& dir, const std::string& vendor, + const std::vector>& processes) +{ + fs::create_directories(dir / vendor / "process"); + std::ofstream index((dir / (vendor + ".json")).string()); + index << R"({"version":"1.0.0","name":")" << vendor << R"(","process_list":[)"; + for (size_t i = 0; i < processes.size(); ++ i) { + const std::string sub_path = "process/p" + std::to_string(i) + ".json"; + index << (i ? "," : "") << R"({"name":")" << processes[i].first << R"(","sub_path":")" << sub_path << R"("})"; + std::ofstream((dir / vendor / sub_path).string()) << processes[i].second; + } + index << "]}"; +} + +std::string process_json(const std::string& name, const std::string& extra = std::string()) +{ + return R"({"type":"process","name":")" + name + R"(","from":"system",)" + extra + R"("layer_height":"0.2"})"; +} + // A small but complete vendor: one machine model, one process, a non-instantiated // base filament with an instantiated child that inherits it and includes a // dual-extruder template, a second standalone filament carrying explicit @@ -128,6 +158,80 @@ void write_lib_tree(const fs::path& dir, const std::string& version, const std:: << R"("filament_id":"GFL99","filament_cost":")" << cost << R"("})"; } +struct FixtureEntry { + std::string name, sub_path; + std::vector deps; // indices of the entries this one inherits or includes +}; + +// Filaments with two levels of inheritance, a template to include, and `leaves` +// presets that inherit either level, some including the template. Only the +// subfiles are written; write_filament_list lists them. +std::vector write_layered_filaments(const fs::path& dir, const std::string& vendor, int leaves) +{ + fs::create_directories(dir / vendor / "filament"); + std::vector entries { + { vendor + " Base PLA", "filament/base.json", {} }, + { vendor + " Mid PLA", "filament/mid.json", {0} }, + { vendor + " dual template", "filament/template.json", {} }, + }; + std::ofstream((dir / vendor / "filament" / "base.json").string()) + << R"({"type":"filament","name":")" << entries[0].name << R"(","from":"system","instantiation":"false",)" + << R"("filament_id":"GFA_base","filament_cost":"42","filament_max_volumetric_speed":["12"]})"; + std::ofstream((dir / vendor / "filament" / "mid.json").string()) + << R"({"type":"filament","name":")" << entries[1].name << R"(","from":"system","instantiation":"false",)" + << R"("inherits":")" << entries[0].name << R"(","filament_flow_ratio":"0.95"})"; + std::ofstream((dir / vendor / "filament" / "template.json").string()) + << R"({"type":"filament","name":")" << entries[2].name << R"(","from":"system","instantiation":"false",)" + << R"("filament_extruder_variant":["Direct Drive Standard","Direct Drive High Flow"],)" + << R"("filament_max_volumetric_speed":["20","22"]})"; + for (int i = 0; i < leaves; ++ i) { + const size_t parent = i % 2; + const bool include = i % 3 == 0; + entries.push_back({ vendor + " PLA " + std::to_string(i) + " @0.4", "filament/leaf" + std::to_string(i) + ".json", + include ? std::vector{parent, 2} : std::vector{parent} }); + std::ofstream((dir / vendor / entries.back().sub_path).string()) + << R"({"type":"filament","name":")" << entries.back().name << R"(","from":"system","instantiation":"true",)" + << R"("inherits":")" << entries[parent].name << R"(",)" + << (include ? R"("include":[")" + entries[2].name + R"("],)" : std::string()) + << R"("nozzle_temperature":[")" << (200 + i % 40) << R"("]})"; + } + return entries; +} + +// The vendor profile for write_layered_filaments, listing its entries in `order`, +// or as they were written when `order` is empty. +void write_filament_list(const fs::path& dir, const std::string& vendor, const std::vector& entries, + std::vector order = {}) +{ + if (order.empty()) { + order.resize(entries.size()); + std::iota(order.begin(), order.end(), 0); + } + std::ofstream f((dir / (vendor + ".json")).string()); + f << R"({"version":"1.0.0","name":")" << vendor << R"(","filament_list":[)"; + for (size_t k = 0; k < order.size(); ++ k) + f << (k ? "," : "") << R"({"name":")" << entries[order[k]].name << R"(","sub_path":")" + << entries[order[k]].sub_path << R"("})"; + f << "]}"; +} + +// A random order of `entries` that lists each one after everything it depends on. +std::vector shuffled_after_dependencies(const std::vector& entries, std::mt19937& rng) +{ + std::vector order; + std::vector listed(entries.size(), false); + while (order.size() < entries.size()) { + std::vector ready; + for (size_t i = 0; i < entries.size(); ++ i) + if (! listed[i] && std::all_of(entries[i].deps.begin(), entries[i].deps.end(), [&](size_t d) { return listed[d]; })) + ready.push_back(i); + const size_t next = ready[std::uniform_int_distribution(0, ready.size() - 1)(rng)]; + listed[next] = true; + order.push_back(next); + } + return order; +} + // A vendor whose one filament inherits the library's base and states nothing of // its own — everything it shows comes from the library it is resolved against. void write_vendor_with_lib_filament(const fs::path& dir, const std::string& vendor, const std::string& version) @@ -624,6 +728,299 @@ TEST_CASE("a cache-loaded vendor is indistinguishable from a JSON-loaded one", " CHECK(pr->config.opt_string("machine_start_gcode") == "G28 ; template"); // through the include } +TEST_CASE("a wide vendor loads from its cache exactly as it loads from JSON", "[VendorCache]") +{ + InstallDirs dirs; + constexpr int leaves = 400; + write_filament_list(dirs.system, "Acme", write_layered_filaments(dirs.system, "Acme", leaves)); + + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(dirs.system / "Acme.opc")); + + fs::remove_all(dirs.system / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + auto a = presets_for(from_json.filaments, "Acme"); + auto b = presets_for(from_cache.filaments, "Acme"); + REQUIRE(a.size() == size_t(leaves)); + REQUIRE(b.size() == a.size()); + for (size_t i = 0; i < a.size(); ++ i) { + CHECK(a[i]->name == b[i]->name); + CHECK(preset_deep_equal(*a[i], *b[i])); + } + CHECK(from_cache.error_count() == from_json.error_count()); + // The two-level chain resolved, so the leaf has the middle profile's flow ratio + // and the base profile's filament id, neither of which it states. + const Preset* leaf = from_cache.filaments.find_preset("Acme PLA 7 @0.4", false); + REQUIRE(leaf != nullptr); + CHECK(leaf->filament_id == "GFA_base"); + const auto* flow = leaf->config.option("filament_flow_ratio"); + REQUIRE(flow != nullptr); + CHECK_THAT(flow->values.front(), WithinAbs(0.95, 1e-9)); +} + +TEST_CASE("presets install the same in any order that lists each after what it depends on", "[VendorCache]") +{ + InstallDirs dirs; + const std::vector entries = write_layered_filaments(dirs.system, "Acme", 150); + auto load = [&](const std::vector& order) { + write_filament_list(dirs.system, "Acme", entries, order); + auto bundle = std::make_unique(); + bundle->load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + return bundle; + }; + const auto reference = load({}); + CHECK(reference->error_count() == 0); + const auto expected = presets_for(reference->filaments, "Acme"); + REQUIRE(expected.size() == 150); + + // Every seed checks against one reference load, which GENERATE would repeat per seed. + for (unsigned seed = 0; seed < 8; ++ seed) { + CAPTURE(seed); + std::mt19937 rng(seed); + const auto shuffled = load(shuffled_after_dependencies(entries, rng)); + CHECK(shuffled->error_count() == 0); + const auto actual = presets_for(shuffled->filaments, "Acme"); + REQUIRE(actual.size() == expected.size()); + for (size_t i = 0; i < actual.size(); ++ i) + CHECK(preset_deep_equal(*actual[i], *expected[i])); + } +} + +TEST_CASE("a key misplaced into a vendor preset is reported and removed", "[VendorCache]") +{ + InstallDirs dirs; + write_process_vendor(dirs.system, "Acme", { + { "Acme template", R"({"type":"process","name":"Acme template","from":"system","instantiation":"false","wall_loops":"5","filament_density":"1.2"})" }, + { "Acme base", R"({"type":"process","name":"Acme base","from":"system","instantiation":"false","filament_cost":"5"})" }, + { "0.20mm Standard @Acme", process_json("0.20mm Standard @Acme", + R"("instantiation":"true","inherits":"Acme base","include":["Acme template"],"nozzle_temperature":["210"],)") } }); + PresetBundle bundle; + bundle.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + CHECK(bundle.error_count() == 3); + const Preset* preset = bundle.prints.find_preset("0.20mm Standard @Acme", false); + REQUIRE(preset != nullptr); + CHECK_FALSE(preset->config.has("filament_cost")); + CHECK_FALSE(preset->config.has("filament_density")); + CHECK_FALSE(preset->config.has("nozzle_temperature")); + CHECK(preset->config.opt_int("wall_loops") == 5); +} + +TEST_CASE("each vendor loads from the directory it is listed with", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.profiles, "Zeta", "2.0.0"); + PresetBundle bundle; + bundle.load_vendors({ { "Acme", dirs.system }, { "Zeta", dirs.profiles } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/false); + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Zeta") == 1); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(bundle.prints.find_preset("0.20mm Standard @Zeta", false) != nullptr); +} + +TEST_CASE("a vendor that fails to load is left out, reported, and the others still load", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_process_vendor(dirs.system, "Broken", { { "Broken A", "{not-json" } }); + PresetBundle bundle; + std::vector failed; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { "Broken", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false, nullptr, &failed).second; + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Broken") == 0); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(errors.find("Broken") != std::string::npos); + CHECK(failed == std::vector{ "Broken" }); +} + +TEST_CASE("a filament library that fails partway is reported, and vendors inheriting from it are left out", "[VendorCache]") +{ + InstallDirs dirs; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(dirs.system, "1.0.0", "20"); + std::ofstream((dirs.system / (lib + ".json")).string()) + << R"({"version":"1.0.0","name":")" << lib << R"(","filament_list":[)" + << R"({"name":"Generic PLA","sub_path":"filament/generic_pla.json"},)" + << R"({"name":"Generic PETG","sub_path":"filament/generic_petg.json"}]})"; + std::ofstream((dirs.system / lib / "filament" / "generic_petg.json").string()) << "{not-json"; + write_vendor_with_lib_filament(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.system, "Zeta", "1.0.0"); + + PresetBundle bundle; + std::vector failed; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { lib, dirs.system }, { "Zeta", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false, nullptr, &failed).second; + + CHECK(failed == std::vector{ lib, "Acme" }); + CHECK(errors.find("generic_petg.json") != std::string::npos); + CHECK(bundle.vendors.count(lib) == 1); + CHECK(bundle.vendors.count("Acme") == 0); + CHECK(bundle.filaments.find_preset("Acme PLA @0.4", false) == nullptr); + CHECK(bundle.vendors.count("Zeta") == 1); + CHECK(bundle.prints.find_preset("0.20mm Standard @Zeta", false) != nullptr); +} + +TEST_CASE("a vendor whose profile cannot be read is left out, reported, and the others still load", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + std::ofstream((dirs.system / "Broken.json").string()) << "{not-json"; + PresetBundle bundle; + const std::string errors = bundle.load_vendors({ { "Acme", dirs.system }, { "Broken", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false).second; + + CHECK(bundle.vendors.count("Acme") == 1); + CHECK(bundle.vendors.count("Broken") == 0); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); + CHECK(errors.find("Broken.json") != std::string::npos); +} + +TEST_CASE("a canceled vendor load starts no vendor", "[VendorCache]") +{ + InstallDirs dirs; + write_vendor_tree(dirs.system, "Acme", "1.0.0"); + write_vendor_tree(dirs.system, "Zeta", "1.0.0"); + const std::atomic cancel { true }; + PresetBundle bundle; + bundle.load_vendors({ { "Acme", dirs.system }, { "Zeta", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/false, &cancel); + + CHECK(bundle.vendors.empty()); + CHECK(bundle.prints.find_preset("0.20mm Standard @Acme", false) == nullptr); +} + +TEST_CASE("a preset two vendors both define is kept from the first listed and reported under the others", "[VendorCache]") +{ + InstallDirs dirs; + for (const std::string vendor : { "Acme", "Mira", "Zeta" }) + write_process_vendor(dirs.system, vendor, { + { "Shared", process_json("Shared", R"("instantiation":"true",)") }, + { "Own @" + vendor, process_json("Own @" + vendor, R"("instantiation":"true",)") } }); + PresetBundle bundle; + const std::string errors = bundle.load_vendors({ { "Mira", dirs.system }, { "Acme", dirs.system }, { "Zeta", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, + /*allow_cache=*/false).second; + + const Preset* shared = bundle.prints.find_preset("Shared", false); + REQUIRE(shared != nullptr); + REQUIRE(shared->vendor != nullptr); + CHECK(shared->vendor->id == "Mira"); + CHECK(errors.find("vendor Mira") == std::string::npos); + CHECK(errors.find("Found duplicated settings in vendor Acme's json file lists: Shared") != std::string::npos); + CHECK(errors.find("Found duplicated settings in vendor Zeta's json file lists: Shared") != std::string::npos); + CHECK(bundle.error_count() == 2); + for (const std::string vendor : { "Acme", "Mira", "Zeta" }) { + const Preset* own = bundle.prints.find_preset("Own @" + vendor, false); + REQUIRE(own != nullptr); + CHECK(own->vendor == &bundle.vendors.at(vendor)); + } +} + +TEST_CASE("filaments merged from several vendors come out generic first, then by name", "[VendorCache]") +{ + InstallDirs dirs; + auto write_filaments = [&](const std::string& vendor, const std::vector& names) { + fs::create_directories(dirs.system / vendor / "filament"); + std::ofstream index((dirs.system / (vendor + ".json")).string()); + index << R"({"version":"1.0.0","name":")" << vendor << R"(","filament_list":[)"; + for (size_t i = 0; i < names.size(); ++ i) { + const std::string sub_path = "filament/f" + std::to_string(i) + ".json"; + index << (i ? "," : "") << R"({"name":")" << names[i] << R"(","sub_path":")" << sub_path << R"("})"; + std::ofstream((dirs.system / vendor / sub_path).string()) + << R"({"type":"filament","name":")" << names[i] << R"(","from":"system","instantiation":"true",)" + << R"("filament_id":"GF)" << vendor << i << R"("})"; + } + index << "]}"; + }; + write_filaments("Zeta", { "Zeta PLA @0.4", "Generic PETG @Zeta" }); + write_filaments("Acme", { "Acme PLA @0.4", "Generic PLA @Acme" }); + PresetBundle bundle; + bundle.load_vendors({ { "Zeta", dirs.system }, { "Acme", dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/false); + + std::vector names; + for (const Preset& preset : bundle.filaments.get_presets()) + if (! preset.is_default) + names.push_back(preset.name); + CHECK(names == std::vector{ "Generic PETG @Zeta", "Generic PLA @Acme", "Acme PLA @0.4", "Zeta PLA @0.4" }); + for (const std::string& name : names) + CHECK(bundle.filaments.find_preset(name, false) != nullptr); +} + +TEST_CASE("a vendor read while the filament library loads resolves against it, from JSON and from its cache", "[VendorCache]") +{ + InstallDirs dirs; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(dirs.system, "1.0.0", "20"); + write_vendor_with_lib_filament(dirs.system, "Acme", "1.0.0"); + auto load = [&] { + auto bundle = std::make_unique(); + bundle->set_generate_vendor_caches(true); + bundle->load_vendors({ { "Acme", dirs.system }, { lib, dirs.system } }, + ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/true); + return bundle; + }; + + const auto from_json = load(); + REQUIRE(fs::exists(dirs.system / "Acme.opc")); + fs::remove_all(dirs.system / "Acme"); + const auto from_cache = load(); + + for (const PresetBundle* bundle : { from_json.get(), from_cache.get() }) { + const Preset* pla = bundle->filaments.find_preset("Acme PLA @0.4", false); + REQUIRE(pla != nullptr); + CHECK(pla->filament_id == "GFL99"); + CHECK_THAT(pla->config.option("filament_cost")->values.front(), WithinAbs(20., 1e-9)); + CHECK(bundle->error_count() == 0); + } +} + +TEST_CASE("repeated cache loads of one vendor produce the same presets", "[VendorCache]") +{ + InstallDirs dirs; + write_filament_list(dirs.system, "Acme", write_layered_filaments(dirs.system, "Acme", 400)); + + PresetBundle seed; + seed.set_generate_vendor_caches(true); + seed.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(dirs.system / "Acme.opc")); + fs::remove_all(dirs.system / "Acme"); + + std::vector loads(3); + for (PresetBundle& bundle : loads) + bundle.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + auto first = presets_for(loads[0].filaments, "Acme"); + REQUIRE(!first.empty()); + for (size_t run = 1; run < loads.size(); ++ run) { + auto again = presets_for(loads[run].filaments, "Acme"); + REQUIRE(again.size() == first.size()); + for (size_t i = 0; i < first.size(); ++ i) { + CHECK(first[i]->name == again[i]->name); + CHECK(preset_deep_equal(*first[i], *again[i])); + } + CHECK(loads[run].error_count() == loads[0].error_count()); + } +} + TEST_CASE("a cache-served vendor reports the errors its parse counted", "[VendorCache]") { TempDir tmp; @@ -866,6 +1263,27 @@ TEST_CASE("a vendor installed as its cache alone still loads after a library upd CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); } +TEST_CASE("a cache install stopped by a canceled task group throws instead of rejecting the cache", "[VendorCache]") +{ + TempDir tmp; + const std::string cache = (tmp.path / "Acme.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Acme"), "Acme", "1.0.0", {filament_entry("Acme PLA @0.4")})); + + PresetBundle bundle; + bool threw = false; + tbb::task_group_context context; + tbb::parallel_for(tbb::blocked_range(0, 1), [&](const tbb::blocked_range&) { + context.cancel_group_execution(); + try { + bundle.load_vendor_cache(cache, "Acme", Semver(1, 0, 0)); + } catch (const std::runtime_error&) { + threw = true; + } + }, context); + + CHECK(threw); +} + TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSONs", "[VendorCache]") { TempDir tmp; @@ -893,6 +1311,12 @@ TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSON user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); CHECK(presets_loaded == 1); CHECK(out.vendors.at("Acme").name == "Acme"); // the profile's name, not the cache's + + // Through load_vendors, which reads the cache before installing it. + PresetBundle several; + several.load_vendors({ { "Acme", user } }, ForwardCompatibilitySubstitutionRule::EnableSilent, /*allow_cache=*/true); + CHECK(several.vendors.at("Acme").name == "Acme"); + CHECK(several.prints.find_preset("0.20mm Standard @Acme", false) != nullptr); } TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]") @@ -1691,6 +2115,31 @@ TEST_CASE("an include listed after the preset that names it is an error, and the CHECK(silk->config.option("filament_max_volumetric_speed")->values == std::vector{12.}); } +TEST_CASE("an include listed after the preset that names it is missing from the cache load too", "[VendorCache]") +{ + InstallDirs dirs; + write_full_vendor_tree(dirs.system, "Acme", "1.0.0", /*templates_last=*/true); + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(dirs.system / "Acme.opc")); + + fs::remove_all(dirs.system / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(from_cache.error_count() == 2); + const Preset* pr = from_cache.printers.find_preset("Acme 0.4 nozzle", false); + REQUIRE(pr != nullptr); + CHECK(pr->config.opt_string("machine_start_gcode") != "G28 ; template"); + const Preset* silk = from_cache.filaments.find_preset("Acme Silk PLA @0.4", false); + REQUIRE(silk != nullptr); + const auto& speed = silk->config.option("filament_max_volumetric_speed")->values; + REQUIRE(speed.size() == 1); + CHECK_THAT(speed.front(), WithinAbs(12., 1e-9)); +} + TEST_CASE("a G-code template that states no instantiation is included, not loaded as a preset", "[VendorCache]") { InstallDirs dirs; @@ -1712,3 +2161,35 @@ TEST_CASE("a G-code template that states no instantiation is included, not loade CHECK(pr->config.opt_string("machine_start_gcode") == "G28 ; template"); CHECK(presets_for(bundle.printers, "Acme").size() == 1); } + +TEST_CASE("a sub-file that fails to parse leaves the ones listed before it installed", "[VendorCache]") +{ + InstallDirs dirs; + write_process_vendor(dirs.system, "Acme", { + { "Acme A", process_json("Acme A", R"("instantiation":"true",)") }, + { "Acme B", "{not-json" }, + { "Acme C", process_json("Acme C", R"("instantiation":"true",)") } }); + PresetBundle bundle; + CHECK_THROWS_AS(bundle.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent), + ConfigurationError); + CHECK(bundle.prints.find_preset("Acme A", false) != nullptr); + CHECK(bundle.prints.find_preset("Acme C", false) == nullptr); +} + +TEST_CASE("a load that fails to install partway counts no errors of the sub-files after it", "[VendorCache]") +{ + InstallDirs dirs; + // B names a parent nobody defines. C states no instantiation, an error of its own. + write_process_vendor(dirs.system, "Acme", { + { "Acme A", process_json("Acme A", R"("instantiation":"true",)") }, + { "Acme B", process_json("Acme B", R"("instantiation":"true","inherits":"Nobody",)") }, + { "Acme C", process_json("Acme C") } }); + PresetBundle bundle; + CHECK_THROWS_AS(bundle.load_vendor_configs_from_json(dirs.system.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent), + ConfigurationError); + // B's missing parent and B's failed install. + CHECK(bundle.error_count() == 2); + CHECK(bundle.prints.find_preset("Acme A", false) != nullptr); +}