diff --git a/.gitattributes b/.gitattributes index 4cab1f4d26..441bdfe1eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Set the default behavior, in case people don't have core.autocrlf set. * text=auto + +# Shell scripts are run by Git Bash on Windows CI, which cannot read a script +# with CRLF line endings: it fails on the first line. Windows checkouts default +# to core.autocrlf=true, so keep these LF whatever the platform. +*.sh text eol=lf diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index 1c392e4bc6..3de2a9184b 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -14,6 +14,7 @@ on: - 'localization/**' - 'resources/**' - ".github/workflows/build_*.yml" + - 'scripts/build_preset_cache.*' - 'scripts/flatpak/**' - 'scripts/msix/**' - 'tests/**' @@ -33,6 +34,7 @@ on: - 'build_release_vs.bat' - 'build_release_vs2022.bat' - 'build_release_macos.sh' + - 'scripts/build_preset_cache.*' - 'scripts/flatpak/**' - 'scripts/msix/**' - 'tests/**' diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index a7652c3bd6..112c0b279b 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -162,6 +162,14 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (macOS) + if: runner.os == 'macOS' && !inputs.macos-combine-only + working-directory: ${{ github.workspace }} + shell: bash + # The bundle was already packed from resources/, so the caches have to be + # installed into it here; the source tree keeps its JSONs for later jobs. + run: ./scripts/build_preset_cache.sh -b build/${{ inputs.arch }} build/${{ inputs.arch }}/OrcaSlicer/OrcaSlicer.app/Contents/Resources/profiles + - name: Pack macOS app bundle ${{ inputs.arch }} if: runner.os == 'macOS' && !inputs.macos-combine-only working-directory: ${{ github.workspace }} @@ -390,6 +398,13 @@ jobs: if ($arch -eq "arm64") { .\build_release_vs.bat slicer arm64 tests } else { .\build_release_vs.bat slicer tests } shell: pwsh + - name: Build system preset cache (Windows) + if: runner.os == 'Windows' + shell: cmd + # Shipped into both the already-installed tree (portable zip, MSIX) and + # the checkout cpack re-installs from when it builds the NSIS installer. + run: scripts\build_preset_cache.bat --prune-source "%BUILD_DIR%" "resources\profiles" "%BUILD_DIR%\OrcaSlicer\resources\profiles" + - name: Pack unit tests Win if: runner.os == 'Windows' working-directory: ${{ github.workspace }} @@ -539,6 +554,20 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + # Both were packed from resources/ before the caches existed, so the + # AppImage is unpacked first and the caches shipped into it and into + # the package tree; the source tree keeps its JSONs for later steps. + appimage=$(find build -maxdepth 1 -name "OrcaSlicer_Linux_AppImage*.AppImage" | head -1) + chmod +x "$appimage" + "$appimage" --appimage-extract + ./scripts/build_preset_cache.sh -b build build/package/resources/profiles squashfs-root/resources/profiles + appimagetool=$(find build -name "appimagetool.AppImage" | head -1) + ARCH=$(uname -m) "$appimagetool" --appimage-extract-and-run squashfs-root "$appimage" + rm -rf squashfs-root # Ship the freshly-built validator so slice_check_linux (build_all.yml) # can slice-sweep the shipped profiles with this PR's engine. Taken from # the aarch64 leg so the sweep also exercises the arm build; x86_64 on diff --git a/.gitignore b/.gitignore index 916c7207b7..4d3ccb5c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ internal_docs/ # Python bytecode __pycache__/ *.pyc +*.opc diff --git a/build_linux.sh b/build_linux.sh index 72ea742f1a..6d65a10e41 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -567,6 +567,8 @@ if [[ -n "${BUILD_ORCA}" ]] || [[ -n "${BUILD_TESTS}" ]] ; then print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer echo "Building OrcaSlicer_profile_validator .." print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer_profile_validator + echo "Building generate_system_cache ..." + print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target generate_system_cache ./scripts/run_gettext.sh fi if [[ -n "${BUILD_TESTS}" ]] ; then diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md new file mode 100644 index 0000000000..6e693dbd6f --- /dev/null +++ b/docs/HLSD/preset-cache.md @@ -0,0 +1,402 @@ +# System Preset Cache — High Level Design + +## Why it exists + +OrcaSlicer ships tens of thousands of system preset JSON files. Every launch used to +parse all of them: read each vendor profile, walk its machine, process and filament +sub-files, resolve inheritance, and build the preset collections from scratch. That +parse dominated startup, and it produced the same result every time, because system +presets only change when the app is updated or a profile update is installed. + +The preset cache replaces that parse with a read. Each vendor's presets are serialized +once — at build time, in CI — into a single binary file the app reads in one pass. The +read replaces the file walk and the JSON parsing, which is where the time went; +resolving inheritance and registering the presets still runs at load, through the same +code the JSON path uses, so the result is the parse's result without the parse. + +The cache is **only ever an optimization**. Every rule below exists to guarantee that a +cache is either provably equivalent to parsing the JSONs, or rejected. There is no +"mostly right" cache. + +## The unit is one vendor + +A cache covers exactly one vendor. `BBL.opc` sits beside `BBL.json` and holds +everything `BBL.json` and the `BBL/` sub-file tree would have produced. + +Per-vendor granularity is what makes the system practical: + +- A vendor whose profile is bumped invalidates only its own cache. The other 60-odd + vendors keep theirs — even when the bumped vendor is the shared Orca filament + library everyone else inherits from. +- The setup wizard, which loads vendors one at a time, gets the same speedup as + startup without a second code path. +- A vendor with no cache, or a broken one, costs only that vendor a parse. + +A cache holds *system* presets only. User presets, project settings and modified +presets are never serialized — they have their own storage and their own lifecycle. + +## Where the files live + +| Location | Contents on a shipped build | Role | +|---|---|---| +| `resources/profiles/` | `.opc` alone — the profile and its preset JSONs both pruned | What the app ships with; what installing copies from, and the only thing it is read for | +| `/system/` | `.opc` alone, or `.json` + `/` after an update | What the user has installed | +| `/system/` (dev build) | `.json` + `/` + `.opc` written at runtime | A developer tree caches as it parses | +| `/cache/wizard_profile_data.json` | The wizard's derived vendor catalog plus the stamps it was built from | Written and read by the setup wizard only; never shipped (see "The wizard's profile-data cache") | + +Two forms of the same vendor therefore exist, and the system's central rule is that +**a vendor's cache is the whole of it**. Where a cache ships or is installed, no profile +and no preset JSONs sit beside it: the cache carries the presets, the vendor profile, +and the version stamp that says which release it came from. A vendor is "installed" if +either form is present *and usable*, and its installed version is read from whichever +form a load would serve. + +What stays beside the caches in `resources/profiles/` is everything that is not a +preset: each vendor's directory of printer thumbnails, cover images, bed models and +hotend meshes, which are read from disk by path and were never part of the cache. Files +that are not vendors at all, `blacklist.json` chief among them, are untouched. + +The alternative — shipping both and treating the cache as a sidecar — was rejected. It +doubles the installed size, and it creates a class of bug where the two disagree and +the app's behavior depends on which one a given code path happened to read. + +## What a cache file is + +A fixed-size header followed by one binary stream. + +The header carries a magic number, the cache format version, the payload size and a +CRC32 of the payload. It exists so that a truncated download, a half-written file or a +file from an entirely different program is rejected in microseconds, before anything +tries to interpret it. + +The payload opens with the stamps that decide whether the cache may be used at all — +format version, vendor name, vendor version — then a dictionary, and then the vendor's +data: its vendor profile, three lists of preset entries (process, filament, machine), +and the count of errors the original parse hit. + +Each entry is one preset **in source form**: what its JSON sub-file states and nothing +that resolving it derives — the preset's own config diff, the name of the preset it +inherits, and the parse metadata (name, sub-path, description, instantiation, setting +and filament ids, renames). Non-instantiated base presets are stored too; the children +that inherit from them cannot resolve without them. + +**The payload names its own keys.** The dictionary holds the distinct `opt_key`s the +file uses, the `ConfigOptionType` each was written as, and the distinct enum *value +names*; an option in an entry's config is then a `uint16` index into that dictionary +plus its value. Names are written once per file rather than once per occurrence, and a +reader resolves the dictionary against this build's `print_config_def` once, after +which reading an option is a vector index. + +This is what makes the cache survive config-schema drift. The alternative — keying an +option by its `serialization_key_ordinal`, the position `ConfigDef::add` assigns by +declaration order at static init — cannot: inserting one option into the middle of +`PrintConfig.cpp` shifts every later ordinal, and the lookup on the way back in then +*succeeds on the wrong option*, silently, wherever the two share a type. Because a +name-keyed payload instead drops the individual options this build cannot place, the +file as a whole stays readable, and there is no schema fingerprint — no checksum over +the option schema that would reject every cache on every release. An option this build +no longer defines, or now defines with a different type, gets exactly what it gets from +a JSON profile: read, dropped, and the rest of the preset loads. + +The ordinal-keyed cereal hooks in `PrintConfig.hpp` are untouched — they are also the +undo/redo wire format, where the process cannot change underneath them. The cache has +its own serialization in `PresetCacheFormat.{hpp,cpp}`. + +Three deliberate choices in the layout: + +- **Stamps come first**, so the question "what version is this vendor installed at?" + can be answered by reading the first kilobyte. The updater asks that question for + every vendor on every launch; reading tens of megabytes to answer it would give back + the startup time the cache saved. The dictionary sits behind them, ahead of the + entries, so a reader that does go on resolves it once and then indexes. +- **Nothing inherited is baked in.** A filament preset that inherits from the shared + library is stored as its own diff plus its parent's name, and the parent is looked up + when the entry is installed, against whatever library is loaded then. A cache + therefore carries no other vendor's values, and no other vendor's update — the + library's included — can make it stale. +- **Nothing derived is stored.** Default presets, flattened configs, aliases and + lookup maps are all reconstructed at load by the same code the JSON path runs, and + state that path never fills (obsolete-preset lists) is not stored either. This keeps + the cache a record of the vendor's data, not a memory image of the program's state. + +## When a cache may be used + +A cache is accepted only if every gate below passes. Any failure means "parse the +JSONs instead" — never a hard error, never a partial load. + +**1. Integrity.** Magic number, a declared body size that is exactly the rest of the +file, CRC32 over the payload. The size is checked against the file's real length before +anything is allocated on the strength of it, so an eight-byte field in an unauthenticated +file cannot ask for a gigabyte. + +**2. Cache format version.** A single integer bumped by hand whenever the binary layout +changes in a way nothing else would catch: reordering or retyping a hand-written +serialized field, or changing what the cache's own stamps mean. Config-schema drift is +explicitly *not* such a change — the dictionary handles it — so this no longer moves +every release. + +**3. Vendor identity and version.** The cache names the vendor it holds and the profile +version it was built from. It is accepted only if that version is at least as new as +the profile now on disk. Where no profile sits beside the cache — the shipped, +cache-only form — the comparison is skipped, because nothing on disk can be newer than +a cache that is the installation. + +**4. Every entry installs.** Entries are installed as they are read, and an entry that +cannot be — typically one that inherits a parent the currently loaded filament library +no longer provides — rejects the whole cache, never just the entry. A partial vendor is +not a vendor. + +There is deliberately no stamp for the shared filament library. A cache stores its +filaments' inheritance by name and resolves it at load, so a library update changes +what a cache load *produces*, never whether the cache is *valid* — the same file yields +the updated result. This matters most on a shipped build, where a vendor is its cache +and nothing else: a profile update that delivered only the library would otherwise have +stranded every other vendor with a cache it invalidated and no JSONs to fall back on. + +A vendor profile with no parsable version is never cached and never served from a +cache. There would be no way to tell later whether the cache had gone stale, and a +cache nothing can invalidate is worse than no cache. + +## How a vendor is loaded + +Vendors load in a fixed order, because filament inheritance crosses exactly one +boundary: any vendor's filament may inherit from the shared Orca filament library, +and nothing else reaches across vendors. The library therefore goes first, alone; +every other vendor follows in parallel, resolving against it; and the results are +merged in a stable order: + +```mermaid +flowchart LR + lib["1 · OrcaFilamentLibrary
loaded first, synchronously"] --> par["2 · every other vendor in parallel,
each into its own bundle, filaments
resolving against the loaded library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] +``` + +Whether a vendor comes from its cache or from a parse changes nothing in that +order — both produce the same bundle, so cached and parsed vendors mix freely in +one startup. + +**A vendor is loaded from where it is installed and nowhere else.** For startup that +is `/system/`; resources reaches the app by being *installed* into that +directory first, never by being loaded from. (The setup wizard is the one caller with +a different notion of "where": it also shows vendors the user has not installed, and +loads those from `resources/profiles` — see "The wizard's profile-data cache".) There +is one lookup tier and one parse source: + +``` +load vendor V from /system: + system/V.opc passes CACHE_VERSION + size + CRC + vendor name + version gate? + yes -> serve from it + no -> parse system/V.json, then write system/V.opc back +``` + +The same decision drawn out — "the gates" are the four acceptance checks above: + +```mermaid +flowchart TB + start["load vendor V from a directory dir
— normally <data_dir>/system/"] + start --> stamp["installed version = version of dir/V.json
— or ∞ with no profile there,
the cache then being the installation"] + stamp --> g1{"dir/V.opc
passes all four gates?"} + g1 -- "yes" --> hit(["served from the
installed cache"]) + g1 -- "no" --> pd["parse the JSONs in dir"] + pd --> ver{"profile version
parsable?"} + ver -- "yes" --> save(["loaded; dir/V.opc written back —
the next load takes the top path"]) + ver -- "no" --> raw(["loaded, never cached"]) +``` + +A second tier into `resources/profiles/` used to sit between those two, and a parse +fallback to the same place behind them. Both existed only because an installed cache +died on every app upgrade, when the schema fingerprint rejected it; with the fingerprint +gone there is nothing for them to rescue. They also had a cost: on a developer tree the +shipped cache answered first, so the profile in `/system/` was never parsed +and its cache was never written back. + +Serving from a cache is not a memory-image restore. The entries are deserialized and +then installed one by one — inheritance resolved against the presets installed before +them and the currently loaded filament library, configs flattened onto the collection +defaults, validated and registered — by the same function the JSON path calls straight +after parsing a sub-file. The two paths share everything below the parse, which is what +makes a cache-loaded bundle indistinguishable from a JSON-loaded one by construction +rather than by test coverage. Installation also rebuilds each preset's file path from +the local data directory, so a shipped cache never carries the generating machine's +paths. + +App upgrades work because a cache normally survives one. Only a deliberate +`CACHE_VERSION` bump makes an installed cache unreadable, and that is handled at +install time rather than at load: a vendor whose cache this build cannot read counts +as **not installed**, so the updater lays down a working copy on the next launch (see +below). A vendor that still has its profile JSONs beside the cache is simply parsed +and re-cached. + +If a parse does happen and the vendor's profile carries a version, the app writes the +cache back beside where it looked for the vendor. That is how a developer build warms +itself up on second launch, and how a vendor delivered by a profile update becomes +cached without waiting for the next release. + +## The wizard's profile-data cache + +The setup wizard's printer and filament pages want every vendor in one bundle — the +installed ones *and* the shipped ones the user has not installed yet, because the +wizard is where installing is chosen. Its set therefore spans two directories: +`/system/` for installed vendors (shadowing resources on a name collision), +`resources/profiles` for the rest, each vendor loaded from its own directory. + +What the wizard actually consumes from that bundle is one derived JSON — the model / +machine / filament / process catalog its web pages render — and that JSON is a pure +function of the vendor set: each vendor's name and version, in load order. A profile +change requires a version bump, so name and version determine a vendor's content +wherever its copy sits; which directory served it is deliberately **not** stamped, +and installing or removing a copy at an unchanged version leaves the cache valid. So +the wizard caches the *derived JSON*, not another form of the inputs: +`/cache/wizard_profile_data.json` holds the stamp list and the catalog. On +open, the wizard computes the current stamps (one version peek per vendor) and, when +they match, serves the catalog from the file — no bundle built, no preset installed. +Caching bundle inputs instead was tried and measured: rebuilding the bundle from +per-vendor caches costs ~2 s of preset installation whatever feeds it, so only +skipping the rebuild entirely wins. + +Any change to the set — a vendor added, removed or updated, or its cache-only +`.opc` replaced by a newer one — changes the stamps and retires the whole file; +the wizard then rebuilds the bundle vendor by vendor (per-vendor caches serving where +they cover) and writes the catalog back. Selections, region and per-open decorations +are applied downstream of the cache either way, so a served catalog is +indistinguishable from a rebuilt one. Nothing ships this file and the updater never +touches it; it is a locally written artifact, re-derived whenever stale, written +through a temp file and rename so half a cache is never readable. + +The cache lives under `/cache/`, not beside the vendors: everything that +scans `/system/` treats any `.opc` there as a vendor, so a non-vendor +cache file must not sit in that directory. Relatedly, the stamp reader is hardened: +`read_cache_stamps` validates the cache version before reading anything +variable-length and bounds the stamp strings' lengths, so a reader pointed at a +foreign or damaged `.opc` rejects it cleanly instead of aborting on a garbage +64-bit allocation. + +## How a vendor is installed + +Installing copies from `resources/profiles/` into `/system/`. A shipped build +offers only a cache and a source tree only JSONs, but a partially-generated tree can +have both, at different versions, so the installer picks the form that ships at the +**newer version** and installs only that one: + +- Cache newer or equal, and readable → copy the `.opc`, verify the *copy* is one this + build can read, and only then delete any profile and vendor directory a previous + install left behind, so nothing can shadow it. +- Profile newer, or the cache unreadable or absent → copy the profile and the vendor's + preset JSONs exactly as the app did before caches existed, and delete any stale `.opc` + once the profile is safely in place. + +One vendor that cannot be installed is one vendor missing, not a reason to leave the +rest uninstalled: the installer skips it, records the failure, and carries on with the +batch. A vendor whose cache arrives unreadable falls back to installing its profile, +which is decided by reading the copy rather than by the kilobyte peek that chose the +form. + +**"Installed" means present and usable.** Where the cache is the whole of a vendor's +installation, a `.opc` this build cannot read is not an installation — counted as one, +the vendor would be stranded with nothing to load and the updater would never repair +it. The installed version is likewise whichever form a load would actually serve: the +cache's stamp while it covers the profile beside it, the profile's own version once it +does not. + +The result is that only one form of a vendor is ever present, and it is the newest one +the build has. This matters most for the update check, which compares what is installed +against what installing *would* lay down: if those two disagreed about which form +counts, a vendor could reinstall on every launch forever, or silently never update. + +Profile updates delivered over the air always arrive as JSONs, and they win — an +updated vendor's real profile lands in the data directory, the installed cache beside it +is older and gets rejected, and the vendor is parsed and re-cached. An update that touches only +the filament library needs nothing more: every other vendor's cache stays valid and +simply resolves against the new library on its next load. + +## How the caches are produced + +Cache generation is a build step, not something a user ever runs. + +One script per platform does the whole job, and CI calls it once on each. It builds a +small dev-utility that loads a profiles directory exactly as the app would, with cache +writing enabled, dropping a `.opc` beside every vendor profile it parses; then +it copies those caches into each packaged application it was pointed at and deletes +every preset JSON they replace — the vendor's own profile included. Only a vendor that +actually has a cache is pruned, so a vendor the generator skipped keeps its JSONs and is +simply parsed at startup. + +Caches are generated into the checkout's own `resources/profiles`, because that is what +cpack re-installs from when it builds the NSIS installer — so that directory is also a +prune target in CI. Pruning it deletes the checkout's preset JSONs, which is a packaging +step, not something a build should do to a working tree by surprise: the Windows script +refuses that target unless given `--prune-source`, and CI passes it. + +Generation runs after the build, in the same job, so the caches ship with a build that +can read them. + +The flatpak differs only in where the script is called from. Nothing outside +flatpak-builder ever builds it, so there is no packaged tree for the workflow to point +the script at afterwards: the manifest runs it as a build step instead, against the +profiles the install has already copied into `/app`. + +## Behavior when things go wrong + +The system is designed so that no cache problem is fatal: + +- **Corrupt, truncated or foreign file** — rejected at the header, vendor parsed. A + cache is written to a temp file beside its target and moved into place, so a write + that dies partway leaves the previous cache intact rather than a truncated one. +- **An option this build no longer has, or now types differently** — that option alone + is dropped, exactly as a JSON profile's would be. The preset and the file load. +- **Cache from a build with a different cache layout** — rejected on `CACHE_VERSION`. + A vendor with JSONs beside it is parsed and re-cached; a cache-only vendor reads as + not installed and the updater reinstalls it. +- **Stale cache** — rejected on the vendor version stamp, vendor parsed and re-cached. +- **Failure part-way through loading** — a deserialization error, or any entry that + fails to install — rejects the whole cache, and the bundle is reset to a clean state + before falling back, so a half-loaded cache can never leak into the parsed result. +- **A vendor that can be neither read nor parsed** — logged, and left out. The setup + wizard drops that vendor from its list and opens with the rest; startup records the + error alongside the vendors that did load. One broken vendor never takes the app down. + +The one genuine limit: on a shipped build a vendor is its cache and nothing else, so a +rejected cache has nothing to fall back to for that vendor. This is by design — the +alternative is shipping every preset twice — and it is why the acceptance gates are +conservative and why CI generates the caches with the same build that ships them. The +recovery path is a profile update, which delivers real JSONs. + +It also means nothing may quietly assume a `.json` exists. Discovery, version +checks and the update decision all read whichever form is present, and a code path that +enumerates only `*.json` will find no vendors at all in a packaged build. + +## Maintenance rules + +- **Adding, removing, retyping or reordering a config option** needs nothing. The + payload names its keys and its enum values, so an option a cache carries and this + build does not is dropped; one this build has and the cache does not is simply + absent, as it would be from a JSON that predates it. +- **Changing a hand-written `serialize()`** — `VendorProfile` or its nested types — or + the `CachedPreset` field list — written and read by `visit_entry` in + `PresetCacheFormat.cpp`, one list for the save, the load and the name peek alike — or + the cache's own layout or stamps, requires bumping `CACHE_VERSION` by hand. +- **The dictionary indexes with a `uint16`**, so `print_config_def` may hold at most + 65535 options and one cache at most 65535 distinct enum value names. + `CacheDictionary::save` throws past that, which surfaces when CI generates the + caches rather than on a user's machine. +- **Bumping `CACHE_VERSION` is safe without a resources fallback** because + `is_vendor_installed` means *present and usable*: cache-only vendors read as not + installed after a bump, and the updater reinstalls them from resources. +- **Bumping a vendor profile's version** invalidates that vendor's cache and nothing + else — the filament library's included. Other vendors' caches resolve against the + new library the next time they load. +- **Caches are never committed.** They are build artifacts, generated per build, + ignored by git. + +## Where this lives in the tree + +| Area | Files | +|---|---| +| Everything about the bytes on disk — the dictionary, one config's wire format, the file framing and stamps, entry serialization, `VendorCacheFile` save/load/peeks | `src/libslic3r/PresetCacheFormat.{hpp,cpp}` | +| Serve-or-parse decision, installing cache entries into a bundle, cache write-back | `src/libslic3r/PresetBundle.{hpp,cpp}` | +| Vendor profile serialization | `src/libslic3r/Preset.hpp` | +| Vendor discovery, installed/shipped versions, installation | `src/libslic3r/utils.cpp` (declared in `Utils.hpp`) | +| Update and reinstall decisions | `src/slic3r/Utils/PresetUpdater.cpp` | +| Setup wizard and printer-selection dialog | `src/slic3r/GUI/ConfigWizard.cpp`, `src/slic3r/GUI/WebGuideDialog.cpp` | +| Generator tool | `src/dev-utils/generate_system_cache.cpp` | +| Build and packaging script | `scripts/build_preset_cache.{sh,bat}` | +| Tests | `tests/libslic3r/test_vendor_cache.cpp` | diff --git a/scripts/build_preset_cache.bat b/scripts/build_preset_cache.bat new file mode 100644 index 0000000000..82f3e02723 --- /dev/null +++ b/scripts/build_preset_cache.bat @@ -0,0 +1,141 @@ +@echo off +rem Build the per-vendor system preset caches (one .opc per vendor) by +rem running the generate_system_cache.exe dev tool against a profiles directory, +rem and make every profiles directory named on the command line ship-ready: +rem install the caches into it and delete the preset JSONs they replace, so a +rem build ships one copy of its presets instead of two. +rem +rem scripts\build_preset_cache.bat [build_dir] [target_dir ...] +rem +rem build_dir defaults to "build" +rem target_dir profiles directories to ship into. Caches are generated into +rem the source tree's resources\profiles, which is what every +rem packaging step copies from; a target may be that same +rem directory, which then only gets pruned. +rem --prune-source +rem allow a target that is the directory the caches were generated +rem into (resources\profiles). Pruning it deletes the checkout's +rem own preset JSONs, which is a packaging step - not something a +rem build should do to a working tree by surprise. CI passes it. +rem +rem Shipping deletes, so it is a CI packaging step. A vendor's own .json +rem goes along with its preset JSONs: the cache carries the vendor profile and +rem the version it was built at, so discovery, version checks and installing all +rem read it there. Only a vendor that has a cache is pruned, so non-vendor JSONs +rem (blacklist.json) are left alone, as are the vendor directories themselves - +rem thumbnails, covers and bed models still live there. +rem +rem set CONFIG= to pin the build config for multi-config generators +rem (default: the config of the tool already in the build tree, else Release) +setlocal enabledelayedexpansion + +set "REPO_ROOT=%~dp0.." + +set "PRUNE_SOURCE=" +:parse_flags +if /i "%~1"=="--prune-source" ( + set "PRUNE_SOURCE=1" + shift + goto :parse_flags +) + +set "BUILD_DIR=%~1" +if "%BUILD_DIR%"=="" set "BUILD_DIR=build" +if not exist "%BUILD_DIR%\" ( + echo ERROR: build tree not found: %BUILD_DIR% 1>&2 + exit /b 1 +) +if not "%~1"=="" shift + +rem Newest match wins: a stale binary silently produces a stale cache layout. +call :find_tool +if not defined CONFIG ( + for %%c in (Debug Release RelWithDebInfo MinSizeRel) do ( + echo !TOOL! | findstr /i "\\%%c\\" >nul && set "CONFIG=%%c" + ) +) +if not defined CONFIG set "CONFIG=Release" + +echo Building generate_system_cache in %BUILD_DIR% (%CONFIG%) +cmake --build "%BUILD_DIR%" --config %CONFIG% --target generate_system_cache +if errorlevel 1 ( + echo ERROR: could not build generate_system_cache - configure the build tree with -DORCA_TOOLS=ON: 1>&2 + echo cmake -S "%REPO_ROOT%" -B "%BUILD_DIR%" -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) +call :find_tool +if not defined TOOL ( + echo ERROR: generate_system_cache.exe not found under %BUILD_DIR% - build with -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) + +set "PROFILES=%REPO_ROOT%\resources\profiles" +if not exist "%PROFILES%\" ( + echo ERROR: profiles directory not found: %PROFILES% 1>&2 + exit /b 1 +) +for %%d in ("%PROFILES%") do set "PROFILES=%%~fd" + +rem Add the slicer's runtime DLL directory to PATH so generate_system_cache.exe +rem can resolve its dependencies (TKernel.dll etc.) without a full install step. +set "DLL_DIR=" +for /f "delims=" %%f in ('dir /s /b "%BUILD_DIR%\TKernel.dll" 2^>nul') do ( + if not defined DLL_DIR set "DLL_DIR=%%~dpf" +) +if defined DLL_DIR set "PATH=%DLL_DIR%;%PATH%" + +echo Generating per-vendor preset caches in %PROFILES% +rem Start clean so vendors that went away - and caches written by older tool +rem versions - don't linger next to the freshly generated ones. +del /q "%PROFILES%\*.opc" 2>nul +del /q "%PROFILES%\*.cache" 2>nul +"%TOOL%" --path "%PROFILES%" --log_level 2 +if errorlevel 1 exit /b %errorlevel% + +:next_target +if "%~1"=="" exit /b 0 +call :ship "%~1" +if errorlevel 1 exit /b 1 +shift +goto :next_target + +:ship +set "TARGET=%~1" +if not exist "%TARGET%\" ( + echo ERROR: profiles directory not found: %TARGET% 1>&2 + exit /b 1 +) +for %%d in ("%TARGET%") do set "TARGET=%%~fd" +if /i "%TARGET%"=="%PROFILES%" if not defined PRUNE_SOURCE ( + echo %TARGET%: skipped - this is where the caches were generated. + echo Pass --prune-source to prune it; that deletes this checkout's preset JSONs. + exit /b 0 +) +if /i not "%TARGET%"=="%PROFILES%" copy /y "%PROFILES%\*.opc" "%TARGET%\" >nul + +set /a SHIPPED=0 +set /a PRUNED=0 +for %%c in ("%PROFILES%\*.opc") do ( + set /a SHIPPED+=1 + set "VENDOR=%%~nc" + if exist "%TARGET%\!VENDOR!.json" ( + del /q "%TARGET%\!VENDOR!.json" + set /a PRUNED+=1 + ) + if exist "%TARGET%\!VENDOR!\" ( + for /f %%n in ('dir /s /b "%TARGET%\!VENDOR!\*.json" 2^>nul ^| find /c /v ""') do set /a PRUNED+=%%n + del /s /q "%TARGET%\!VENDOR!\*.json" >nul 2>&1 + rem Deepest first, so a directory the delete above emptied goes too; rd + rem refuses the ones still holding covers or meshes. + for /f "delims=" %%d in ('dir /s /b /ad "%TARGET%\!VENDOR!" 2^>nul ^| sort /r') do rd "%%d" 2>nul + ) +) +echo %TARGET%: !SHIPPED! caches, dropped !PRUNED! preset JSONs +exit /b 0 + +:find_tool +set "TOOL=" +for /f "delims=" %%f in ('dir /s /b /o-d "%BUILD_DIR%\generate_system_cache.exe" 2^>nul') do ( + if not defined TOOL set "TOOL=%%f" +) +exit /b 0 diff --git a/scripts/build_preset_cache.sh b/scripts/build_preset_cache.sh new file mode 100755 index 0000000000..7a874f7e06 --- /dev/null +++ b/scripts/build_preset_cache.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Build the per-vendor system preset caches (one .opc per vendor) by +# running the generate_system_cache dev tool against a profiles directory, and +# make every profiles directory named on the command line ship-ready: install +# the caches into it and delete the preset JSONs they replace, so a build ships +# one copy of its presets instead of two. +# +# ./scripts/build_preset_cache.sh # caches into resources/profiles +# ./scripts/build_preset_cache.sh -b build/arm64 # search this build tree for the tool +# ./scripts/build_preset_cache.sh [ ...] # and ship into these profiles dirs +# +# Caches are generated into the source tree's resources/profiles, which is what +# every packaging step copies from. Shipping deletes, so it is a CI packaging +# step: pass packaged output directories, or the checkout of a build that is +# about to be packaged from it. +# +# A vendor's own .json goes along with its preset JSONs: the cache +# carries the vendor profile and the version it was built at, so discovery, +# version checks and installing all read it there. A shipped vendor is its cache +# and nothing else. Only a vendor that has a cache is pruned, so an ungenerated +# vendor keeps its JSONs and is simply parsed at startup; non-vendor JSONs +# (blacklist.json) are left alone, as are the vendor directories themselves — +# thumbnails, covers and bed models still live there. +# +# -b build tree holding the tool +# (default: build/arm64, build/x86_64, or build — first that exists) +# -p profiles directory to generate caches into +# (default: /resources/profiles) +# -c build config for multi-config generators +# (default: the config of the tool already in the build tree, else +# the build tree's CMAKE_BUILD_TYPE) +# -n skip the rebuild and run the tool already in the build tree +# -l tool log level (default: 2) +# --prune-source +# allow a target that is the directory the caches were generated +# into (resources/profiles). Pruning it deletes the checkout's own +# preset JSONs, which is a packaging step - not something a build +# should do to a working tree by surprise. +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd -P)" +build_dir="" +profiles_dir="" +config="" +build_tool=1 +log_level=2 +prune_source=0 + +# getopts does not do long options; pull this one out first. +args=() +for arg in "$@"; do + if [ "$arg" = "--prune-source" ]; then prune_source=1; else args+=("$arg"); fi +done +set -- ${args+"${args[@]}"} + +while getopts "b:p:c:l:nh" opt; do + case $opt in + b) build_dir="$OPTARG" ;; + p) profiles_dir="$OPTARG" ;; + c) config="$OPTARG" ;; + n) build_tool=0 ;; + l) log_level="$OPTARG" ;; + h) sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "$build_dir" ]; then + for candidate in "$repo_root/build/arm64" "$repo_root/build/x86_64" "$repo_root/build"; do + if [ -d "$candidate" ]; then build_dir="$candidate"; break; fi + done +fi +if [ -z "$build_dir" ] || [ ! -d "$build_dir" ]; then + echo "ERROR: build tree not found (pass -b )" >&2 + exit 1 +fi + +# Newest match wins: multi-config trees keep one binary per config, and a stale +# one silently produces a stale cache layout. +find_tool() { + local best="" f + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ -z "$best" ] || [ "$f" -nt "$best" ]; then best="$f"; fi + done < <(find "$build_dir" -name generate_system_cache -type f 2>/dev/null) + printf '%s' "$best" +} + +tool=$(find_tool) +if [ -z "$config" ]; then + case "$tool" in + */Debug/*) config=Debug ;; + */Release/*) config=Release ;; + */RelWithDebInfo/*) config=RelWithDebInfo ;; + */MinSizeRel/*) config=MinSizeRel ;; + *) config=$(sed -n 's/^CMAKE_BUILD_TYPE:[A-Z]*=\(.\+\)$/\1/p' "$build_dir/CMakeCache.txt" 2>/dev/null | head -1 || true) ;; + esac +fi + +if [ "$build_tool" = 1 ]; then + echo "Building generate_system_cache in $build_dir${config:+ ($config)}" + build_args=(--build "$build_dir" --target generate_system_cache) + if [ -n "$config" ]; then build_args+=(--config "$config"); fi + if ! cmake "${build_args[@]}"; then + echo "ERROR: could not build generate_system_cache — configure the build tree with -DORCA_TOOLS=ON:" >&2 + echo " cmake -S \"$repo_root\" -B \"$build_dir\" -DORCA_TOOLS=ON" >&2 + exit 1 + fi + tool=$(find_tool) +fi + +if [ -z "$tool" ]; then + echo "ERROR: generate_system_cache not found under $build_dir — build with -DORCA_TOOLS=ON" >&2 + exit 1 +fi + +if [ -z "$profiles_dir" ]; then profiles_dir="$repo_root/resources/profiles"; fi +if [ ! -d "$profiles_dir" ]; then + echo "ERROR: profiles directory not found: $profiles_dir" >&2 + exit 1 +fi +profiles_dir=$(cd "$profiles_dir" && pwd -P) + +# Start clean so vendors that went away — and caches written by older tool +# versions — don't linger next to the freshly generated ones. +echo "Generating per-vendor preset caches in $profiles_dir" +rm -f "$profiles_dir"/*.opc "$profiles_dir"/*.cache +"$tool" --path "$profiles_dir" --log_level "$log_level" + +for target in "$@"; do + resolved=$(cd "$target" 2>/dev/null && pwd -P) || { + echo "ERROR: profiles directory not found: $target" >&2 + exit 1 + } + if [ "$resolved" = "$profiles_dir" ] && [ "$prune_source" -eq 0 ]; then + echo "$resolved: skipped - this is where the caches were generated." + echo " Pass --prune-source to prune it; that deletes this checkout's preset JSONs." + continue + fi + if [ "$resolved" != "$profiles_dir" ]; then + cp "$profiles_dir"/*.opc "$resolved"/ + fi + + pruned=0 + shipped=0 + for cache in "$profiles_dir"/*.opc; do + vendor=$(basename "$cache" .opc) + shipped=$(( shipped + 1 )) + if [ -f "$resolved/$vendor.json" ]; then + rm -f "$resolved/$vendor.json" + pruned=$(( pruned + 1 )) + fi + [ -d "$resolved/$vendor" ] || continue + n=$(find "$resolved/$vendor" -name '*.json' | wc -l) + find "$resolved/$vendor" -name '*.json' -delete + find "$resolved/$vendor" -type d -empty -delete + pruned=$(( pruned + n )) + done + echo "$resolved: $shipped caches, dropped $pruned preset JSONs" +done diff --git a/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml b/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml index c33425f23f..94c98121ec 100644 --- a/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml +++ b/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml @@ -347,6 +347,7 @@ modules: - | cmake . -B build_flatpak \ -DFLATPAK=ON \ + -DORCA_TOOLS=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH=/app \ -DCMAKE_INSTALL_PREFIX=/app \ @@ -357,6 +358,13 @@ modules: - ./scripts/run_gettext.sh - cmake --build build_flatpak --target install -j$FLATPAK_BUILDER_N_JOBS + # Per-vendor preset caches. On the other platforms CI runs this script + # itself; the flatpak is built inside flatpak-builder and the generator + # only exists in here, so the swap is a build step instead, against the + # profiles the install above copied into /app. + - cmake --build build_flatpak --target generate_system_cache -j$FLATPAK_BUILDER_N_JOBS + - ./scripts/build_preset_cache.sh -n -b build_flatpak /app/share/OrcaSlicer/profiles + cleanup: - /include @@ -403,6 +411,9 @@ modules: - type: file path: ../run_gettext.sh dest: scripts + - type: file + path: ../build_preset_cache.sh + dest: scripts # AppData metainfo for GNOME Software & Co. - type: file diff --git a/src/dev-utils/CMakeLists.txt b/src/dev-utils/CMakeLists.txt index e3534a024a..2cfce6a7c5 100644 --- a/src/dev-utils/CMakeLists.txt +++ b/src/dev-utils/CMakeLists.txt @@ -20,6 +20,16 @@ if (SLIC3R_ENC_CHECK) ) endif() +if (ORCA_TOOLS) + set(_DEV_DEFS -DBOOST_ALL_NO_LIB -DBOOST_USE_WINAPI_VERSION=0x602 -DBOOST_SYSTEM_USE_UTF8) + + # generate_system_cache: pre-generates per-vendor .opc files under resources/profiles for CI bundling. + add_executable(generate_system_cache generate_system_cache.cpp) + target_link_libraries(generate_system_cache libslic3r boost_headeronly) + target_compile_definitions(generate_system_cache PRIVATE ${_DEV_DEFS}) + +endif() + # Function that adds source file encoding check to a target # using the above encoding-check binary diff --git a/src/dev-utils/generate_system_cache.cpp b/src/dev-utils/generate_system_cache.cpp new file mode 100644 index 0000000000..426ccee997 --- /dev/null +++ b/src/dev-utils/generate_system_cache.cpp @@ -0,0 +1,84 @@ +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/Utils.hpp" + +#include +#include +#include +#include +#include + +using namespace Slic3r; +namespace fs = boost::filesystem; +namespace po = boost::program_options; + +int main(int argc, char* argv[]) +{ + po::options_description desc("OrcaSlicer System Cache Generator\nUsage"); + // clang-format off + desc.add_options() + ("help,h", "Show help") +#ifdef __APPLE__ + ("path,p", po::value()->default_value("../../../../../../../resources/profiles"), "Path to profiles directory") +#else + ("path,p", po::value()->default_value("../../../resources/profiles"), "Path to profiles directory") +#endif + ("log_level,l", po::value()->default_value(2), "Log level (0=trace, 2=info, 4=error)"); + // clang-format on + + po::variables_map vm; + try { + po::store(po::parse_command_line(argc, argv, desc), vm); + if (vm.count("help")) { std::cout << desc << "\n"; return 0; } + po::notify(vm); + } catch (const po::error& e) { + std::cerr << "Error: " << e.what() << "\n" << desc << "\n"; + return 1; + } + + const std::string profiles_path = vm["path"].as(); + const int log_level = vm["log_level"].as(); + + if (!fs::exists(profiles_path) || !fs::is_directory(profiles_path)) { + std::cerr << "Error: '" << profiles_path << "' is not a valid directory\n"; + return 1; + } + + set_logging_level(log_level); + set_data_dir(profiles_path); + set_resources_dir(fs::path(profiles_path).parent_path().make_preferred().string()); + + const fs::path user_dir = fs::path(data_dir()) / PRESET_USER_DIR; + if (!fs::exists(user_dir)) + fs::create_directories(user_dir); + + AppConfig app_config; + app_config.set("preset_folder", "default"); + + auto preset_bundle = std::make_unique(); + preset_bundle->set_is_validation_mode(true); + preset_bundle->set_default_suppressed(true); + preset_bundle->set_generate_vendor_caches(true); + + std::cout << "Loading system presets from: " << profiles_path << "\n"; + + try { + // In validation mode data_dir() is the profiles directory set above, so the + // loader writes each .opc next to its .json as it parses it. + preset_bundle->load_presets(app_config, ForwardCompatibilitySubstitutionRule::EnableSilent); + } catch (const std::exception& ex) { + std::cerr << "Failed to load presets: " << ex.what() << "\n"; + return 1; + } + + size_t cache_count = 0; + for (auto& entry : fs::directory_iterator(profiles_path)) + if (boost::iends_with(entry.path().string(), ".opc")) + ++ cache_count; + if (cache_count == 0) { + std::cerr << "No vendor cache files were generated under " << profiles_path << "\n"; + return 1; + } + std::cout << "Generated " << cache_count << " vendor cache file(s) under " << profiles_path << "\n"; + return 0; +} diff --git a/src/libslic3r/CMakeLists.txt b/src/libslic3r/CMakeLists.txt index f7b4de6e25..333f43a68c 100644 --- a/src/libslic3r/CMakeLists.txt +++ b/src/libslic3r/CMakeLists.txt @@ -348,6 +348,8 @@ set(lisbslic3r_sources Polyline.hpp PresetBundle.cpp PresetBundle.hpp + PresetCacheFormat.cpp + PresetCacheFormat.hpp Preset.cpp Preset.hpp PrincipalComponents2D.cpp diff --git a/src/libslic3r/Config.hpp b/src/libslic3r/Config.hpp index ef93f0d509..9e4344820d 100644 --- a/src/libslic3r/Config.hpp +++ b/src/libslic3r/Config.hpp @@ -28,6 +28,9 @@ #include #include +// The serialize() members below archive ConfigOption hierarchies through +// cereal::base_class, whose registration machinery lives in polymorphic.hpp. +#include namespace Slic3r { struct FloatOrPercent diff --git a/src/libslic3r/Preset.cpp b/src/libslic3r/Preset.cpp index 9ae8b86fd8..1334bd4e7a 100644 --- a/src/libslic3r/Preset.cpp +++ b/src/libslic3r/Preset.cpp @@ -147,6 +147,9 @@ Semver get_version_from_json(std::string file_path) return Semver(); //throw ConfigurationError(format("Failed loading configuration file \"%1%\": %2%", file_path, err.what())); } + catch(...) { + return Semver(); + } } //BBS: add a function to load the key-values from xxx.json @@ -261,18 +264,28 @@ void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil } }; + // The four variant sets are immutable after static init and probed for every + // key of every preset loaded; one merged map makes that a single lookup. + // emplace keeps the first insertion, preserving the first-set-wins priority + // of the else-if chain this replaces. + static const std::unordered_map variant_class = [] { + std::unordered_map m; + for (const std::string& k : print_options_with_variant) m.emplace(k, 0); + for (const std::string& k : filament_options_with_variant) m.emplace(k, 1); + for (const std::string& k : printer_options_with_variant_1) m.emplace(k, 2); + for (const std::string& k : printer_options_with_variant_2) m.emplace(k, 3); + return m; + }(); + for(auto& key :config.keys()){ - if(auto iter = print_options_with_variant.find(key); iter != print_options_with_variant.end()){ - replace_nil_and_resize(key, process_variant_length); - } - else if(auto iter = filament_options_with_variant.find(key); iter != filament_options_with_variant.end()){ - replace_nil_and_resize(key, filament_variant_length); - } - else if(auto iter = printer_options_with_variant_1.find(key); iter != printer_options_with_variant_1.end()){ - replace_nil_and_resize(key, machine_variant_length); - } - else if(auto iter = printer_options_with_variant_2.find(key); iter != printer_options_with_variant_2.end()){ - replace_nil_and_resize(key, machine_variant_length * 2); + auto iter = variant_class.find(key); + if (iter == variant_class.end()) + continue; + switch (iter->second) { + case 0: replace_nil_and_resize(key, process_variant_length); break; + case 1: replace_nil_and_resize(key, filament_variant_length); break; + case 2: replace_nil_and_resize(key, machine_variant_length); break; + case 3: replace_nil_and_resize(key, machine_variant_length * 2); break; } } } diff --git a/src/libslic3r/Preset.hpp b/src/libslic3r/Preset.hpp index b88b5a5ed5..c9b3197a6f 100644 --- a/src/libslic3r/Preset.hpp +++ b/src/libslic3r/Preset.hpp @@ -131,6 +131,10 @@ public: PrinterVariant() {} PrinterVariant(const std::string &name) : name(name) {} std::string name; + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) { ar(name); } // PrinterVariant }; struct PrinterModel { @@ -139,7 +143,7 @@ public: std::string name; //BBS: this is internal id for the printer. Currently only used for searching in database std::string model_id; - PrinterTechnology technology; + PrinterTechnology technology = ptFFF; std::string family; std::vector variants; std::vector default_materials; @@ -162,6 +166,17 @@ public: } const PrinterVariant* variant(const std::string &name) const { return const_cast(this)->variant(name); } + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // PrinterModel + { + ar(id, name, model_id, technology, family, variants, default_materials, + not_support_bed_types, bed_model, bed_texture, image_bed_type, + bottom_texture_end_name, use_double_extruder_default_texture, + bottom_texture_rect, bottom_texture_rect_longer, middle_texture_rect, + hotend_model); + } }; std::vector models; @@ -173,6 +188,14 @@ public: bool valid() const { return ! name.empty() && ! id.empty() && config_version.valid(); } + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // VendorProfile + { + ar(name, id, config_version, config_update_url, changelog_url, + models, default_filaments, default_sla_materials); + } + // Load VendorProfile from an ini file. // If `load_all` is false, only the header with basic info (name, version, URLs) is loaded. static VendorProfile from_ini(const boost::filesystem::path &path, bool load_all=true); @@ -427,10 +450,10 @@ public: Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {} protected: - Preset() = default; - friend class PresetCollection; friend class PresetBundle; + + Preset() = default; }; bool is_compatible_with_print (const PresetWithVendorProfile &preset, const PresetWithVendorProfile &active_print, const PresetWithVendorProfile &active_printer); diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index e66ae064f0..6fcc6e05c1 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -1,7 +1,11 @@ #include +#include #include +#include #include "PresetBundle.hpp" + +#include "PresetCacheFormat.hpp" #include "PrintConfig.hpp" #include "libslic3r.h" #include "I18N.hpp" @@ -307,16 +311,20 @@ std::string PresetBundle::find_preset_vendor(const std::string &preset_name, Pre return ""; } - // Iterate through vendor JSON files in the system directory - for (auto& dir_entry : fs::directory_iterator(system_dir)) { - std::string vendor_file = dir_entry.path().string(); - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " Checking vendor: " << vendor_file; - if (!Slic3r::is_json_file(vendor_file)) + // A vendor is named by its profile or, where the build ships preset caches + // instead of the raw profile JSONs, by its cache alone. + for (const std::string& vendor_name : vendor_names_in(system_dir)) { + const fs::path vendor_json = system_dir / (vendor_name + ".json"); + if (! fs::exists(vendor_json)) { + if (VendorCacheFile::carries_preset((system_dir / (vendor_name + ".opc")).string(), vendor_name, type, preset_name)) { + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " Found preset " << preset_name + << " in vendor cache " << vendor_name; + return vendor_name; + } continue; - - // Get vendor name (filename without .json extension) - std::string vendor_name = dir_entry.path().filename().string(); - vendor_name.erase(vendor_name.size() - 5); // Remove ".json" + } + const std::string vendor_file = vendor_json.string(); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << " Checking vendor: " << vendor_file; try { // Load and parse the vendor JSON file @@ -563,6 +571,8 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, substitution_rule %1%, preferred printer_model_id %2%")%substitution_rule%preferred_selection.printer_model_id; + const auto startup_t0 = std::chrono::steady_clock::now(); + //BBS: change system config to json std::tie(substitutions, errors_cummulative) = this->load_system_presets_from_json(substitution_rule); @@ -588,6 +598,12 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward set_calibrate_printer(""); + { + const auto total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - startup_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: all presets loaded in " << total_ms << " ms"; + } + //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" finished, returned substitutions %1%")%substitutions.size(); return substitutions; @@ -1000,6 +1016,8 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For bundles.m_bundles.clear(); bundles.WriteUnlock(); + const auto user_load_t0 = std::chrono::steady_clock::now(); + // Load bundle metadata from _local directory first fs::path local_dir(folder / PRESET_LOCAL_DIR); if (fs::exists(local_dir)) { @@ -1018,7 +1036,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.filament_presets.clear(); metadata.printer_presets.clear(); - // Add the profiles this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::LocalBundle, metadata.id)); @@ -1055,7 +1072,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.printer_presets.clear(); metadata.is_subscribed = true; - // Load presets from bundle (same logic as __local__) this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::SubscribedBundle, metadata.id)); @@ -1076,34 +1092,41 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For } } - // BBS do not load sla_print - // BBS: change directoties by design - try { - std::string print_selected_preset_name = prints.get_selected_preset().name; - this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); - prints.select_preset_by_name(print_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + // BBS: change directories by design + + { + const auto json_t0 = std::chrono::steady_clock::now(); + try { + std::string sel = prints.get_selected_preset().name; + this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); + prints.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = filaments.get_selected_preset().name; + this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); + filaments.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = printers.get_selected_preset().name; + this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); + printers.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + + const auto json_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - json_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user presets loaded from JSON in " << json_ms << " ms"; } - try { - std::string filament_selected_preset_name = filaments.get_selected_preset().name; - this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); - filaments.select_preset_by_name(filament_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + + { + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - user_load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user + bundle presets loaded in " << ms << " ms"; } - try { - std::string printer_selected_preset_name = printers.get_selected_preset().name; - this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); - printers.select_preset_by_name(printer_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); - } - if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + this->update_multi_material_filament_presets(); this->update_compatible(PresetSelectCompatibleType::Never); - set_calibrate_printer(""); return PresetsConfigSubstitutions(); @@ -1209,13 +1232,10 @@ bool PresetBundle::apply_vendor_config( : std::map(); // Find vendors that need installation - const auto vendor_dir = (fs::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - std::vector install_bundles; for (const auto &it : new_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir / (it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -2223,6 +2243,16 @@ void PresetBundle::remove_users_preset(AppConfig &config, std::mapprints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); +} //BBS: add json related logic, load system presets from json std::pair PresetBundle::load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule) @@ -2242,22 +2272,19 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); + const auto load_t0 = std::chrono::steady_clock::now(); + + // The vendors below are loaded whole and against each other — the filament + // library first, then every other vendor with it as the base — so each parse + // is complete enough to be worth caching. + m_generate_vendor_caches = m_generate_vendor_caches || ! validation_mode; + PresetsConfigSubstitutions substitutions; std::string errors_cummulative; - bool first = true; - std::vector vendor_names; - // store all vendor names in vendor_names - for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { - std::string vendor_file = dir_entry.path().string(); - if (!Slic3r::is_json_file(vendor_file)) - continue; - - std::string vendor_name = dir_entry.path().filename().string(); - - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - vendor_names.push_back(vendor_name); - } + bool first = true; + // Sorted, so any duplicate-preset warning below comes out in the same order on + // every run. + const std::set vendor_names = vendor_names_in(dir); // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded // first because other vendors' filaments may inherit from it via the // `base_bundle` lookup in parse_subfile. The remaining vendors are @@ -2273,8 +2300,13 @@ std::pair PresetBundle::load_system_pre } // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (!orca_lib_vendor.empty()) { + if (! orca_lib_vendor.empty()) { try { + // Match a fresh launch before parsing: hold aliases and the error + // counter survive reset(), and would otherwise carry prior-cycle + // state into this load. + this->clear_printer_hold_aliases(); + this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json(dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule).first); first = false; } catch (const std::runtime_error &err) { @@ -2297,10 +2329,10 @@ std::pair PresetBundle::load_system_pre for (size_t i = range.begin(); i < range.end(); ++i) { auto bundle = std::make_unique(); bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); try { auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, - compatibility_rule, this); + dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this); parallel_substitutions[i] = std::move(result.first); parallel_bundles[i] = std::move(bundle); } catch (const std::runtime_error &err) { @@ -2345,6 +2377,11 @@ std::pair PresetBundle::load_system_pre } this->update_system_maps(); + + const auto load_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; + //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; return std::make_pair(std::move(substitutions), errors_cummulative); @@ -4759,30 +4796,287 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(": finished"); } +// Orca: load one source-form preset entry — parsed from its JSON subfile just +// now, or deserialized from the vendor's cache; the code is shared so a +// cache-loaded bundle cannot come out different from a JSON-loaded one. +// Resolves `inherits` against the presets loaded before this one +// (config_maps) or against base_bundle's filament library, flattens, validates +// and registers the preset. Returns the reason loading failed, empty on +// success. +std::string PresetBundle::load_vendor_preset( + const CachedPreset& entry, + const std::string& path, const std::string& vendor_name, + const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, + ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, + std::map& config_maps, std::map& filament_id_maps, + PresetCollection* presets_collection, size_t& count, bool is_from_lib, + const std::set* retain_configs) +{ + const VendorProfile* current_vendor_profile = &this->vendors.at(vendor_name); + const std::string subfile = path + "/" + vendor_name + "/" + entry.sub_path; + const std::string& preset_name = entry.name; + std::string alias_name, filament_id = entry.filament_id; + std::vector renamed_from = entry.renamed_from; + DynamicPrintConfig config; + const DynamicPrintConfig* default_config = nullptr; + std::string reason; + + //check whether it inherits other preset or not + if (! entry.inherits.empty()) { + auto it2 = config_maps.find(entry.inherits); + if (it2 != config_maps.end()) + default_config = &(it2->second); + if (default_config == nullptr && base_bundle != nullptr) { + auto base_it2 = base_bundle->m_config_maps.find(entry.inherits); + if (base_it2 != base_bundle->m_config_maps.end()) + default_config = &(base_it2->second); + } + if (default_config != nullptr) { + if (filament_id.empty() && (presets_collection->type() == Preset::TYPE_FILAMENT)) { + auto filament_id_map_iter = filament_id_maps.find(entry.inherits); + if (filament_id_map_iter != filament_id_maps.end()) { + filament_id = filament_id_map_iter->second; + } + if (filament_id.empty() && base_bundle != nullptr) { + auto base_filament_id_map_iter = base_bundle->m_filament_id_maps.find(entry.inherits); + if (base_filament_id_map_iter != base_bundle->m_filament_id_maps.end()) { + filament_id = base_filament_id_map_iter->second; + } + } + } + } + else { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << preset_name; + // throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); + reason = "Can not find inherits: " + entry.inherits; + return reason; + } + } + else { + if (presets_collection->type() == Preset::TYPE_PRINTER) + default_config = &presets_collection->default_preset_for(entry.config_src).config; + else + default_config = &presets_collection->default_preset().config; + } + config = *default_config; + config.apply(entry.config_src); + extend_default_config_length(config, true, *default_config); + if (entry.instantiation == "false" && "Template" != vendor_name) { + // Report configuration fields, which are misplaced into a wrong group. + std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); + if (!incorrect_keys.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys + << ", which were removed"; + } + + if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) + config_maps.emplace(preset_name, std::move(config)); + if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!filament_id.empty())) + filament_id_maps.emplace(preset_name, filament_id); + return reason; + } + if (config.has("alias")) + alias_name = (dynamic_cast(config.option("alias")))->value; + Preset::normalize(config); + + // Report configuration fields, which are misplaced into a wrong group. + std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); + if (!incorrect_keys.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys + << ", which were removed"; + } + + if (presets_collection->type() == Preset::TYPE_PRINTER) { + // Filter out printer presets, which are not mentioned in the vendor profile. + // These presets are considered not installed. + auto printer_model = config.opt_string("printer_model"); + if (printer_model.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines no printer model, it will be ignored."; + reason = std::string("can not find printer_model"); + return reason; + } + auto printer_variant = config.opt_string("printer_variant"); + if (printer_variant.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines no printer variant, it will be ignored."; + reason = std::string("can not find printer_variant"); + return reason; + } + auto it_model = std::find_if(current_vendor_profile->models.cbegin(), current_vendor_profile->models.cend(), + [&](const VendorProfile::PrinterModel &m) { return m.id == printer_model; } + ); + if (it_model == current_vendor_profile->models.end()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines invalid printer model \"" << printer_model << "\", it will be ignored."; + reason = std::string("can not find printer model in vendor profile"); + return reason; + } + auto it_variant = it_model->variant(printer_variant); + if (it_variant == nullptr) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines invalid printer variant \"" << printer_variant << "\", it will be ignored."; + reason = std::string("can not find printer_variant in vendor profile"); + return reason; + } + // An instantiation printer profile's nozzle_diameter must match the numeric (diameter) + // prefix of its printer_variant: "0.4" -> {0.4}, "0.8HF" -> {0.8} (a trailing + // non-numeric suffix such as "HF"/"HS" distinguishes a hardware sub-variant and is + // ignored here), and for multi-nozzle printers "0.4+0.6" -> {0.4, 0.6}. + // Note: a variant may legitimately repeat across presets of the same model (e.g. speed + // modes, IDEX copy/mirror, or different control boards), so only the diameter is + // validated, not variant uniqueness. Validation-only so the app keeps loading existing + // profiles unchanged. + if (validation_mode && entry.instantiation == "true") { + const auto *nd = config.option("nozzle_diameter"); + std::set nozzles, variant_nozzles; + if (nd != nullptr) + nozzles.insert(nd->values.begin(), nd->values.end()); + std::vector variant_tokens; + boost::algorithm::split(variant_tokens, printer_variant, boost::algorithm::is_any_of("+")); + bool variant_ok = true; // printer_variant is already guaranteed non-empty above + for (const std::string &tok : variant_tokens) { + size_t consumed = 0; + double d = string_to_double_decimal_point(tok, &consumed); + // Require a leading numeric diameter; a trailing suffix (e.g. "HF") is allowed. + if (consumed == 0) { variant_ok = false; break; } + variant_nozzles.insert(d); + } + if (!variant_ok || variant_nozzles != nozzles) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" has printer_variant \"" << printer_variant << + "\" that does not match its nozzle_diameter \"" << (nd ? nd->serialize() : std::string()) << "\". " + "printer_variant must begin with the nozzle diameter, optionally followed by a non-numeric suffix " + "(e.g. \"0.4\", \"0.8HF\"); for multi-nozzle printers, join the per-nozzle diameters with \"+\" in " + "nozzle order (e.g. \"0.4+0.6\")."; + } + } + } + const Preset *preset_existing = presets_collection->find_preset(preset_name, false); + if (preset_existing != nullptr) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" has already been loaded from another Config Bundle."; + reason = std::string("duplicated defines"); + return reason; + } + + auto file_path = (boost::filesystem::path(data_dir()) /PRESET_SYSTEM_DIR/ vendor_name / entry.sub_path).make_preferred(); + if(validation_mode) + file_path = (boost::filesystem::path(data_dir()) / vendor_name / entry.sub_path).make_preferred(); + + // Load the preset into the list of presets, save it to disk. + Preset &loaded = presets_collection->load_preset(file_path.string(), preset_name, std::move(config), false); + if (flags.has(LoadConfigBundleAttribute::LoadSystem)) { + loaded.is_system = true; + loaded.vendor = current_vendor_profile; + loaded.version = current_vendor_profile->config_version; + loaded.description = entry.description; + loaded.setting_id = entry.setting_id; + // Derive the preset setting_id on the fly when a profile ships without one, + // matching scripts/assign_vendor_setting_ids.py. Only instantiated presets + // carry an id; non-instantiated base profiles return earlier above. This never + // touches the per-user cloud-sync setting_id written into user .info files. + if (loaded.setting_id.empty() && entry.instantiation == "true") + loaded.setting_id = generate_preset_setting_id( + vendor_name, Preset::get_type_string(presets_collection->type()), preset_name); + loaded.filament_id = filament_id; + loaded.m_from_orca_filament_lib = is_from_lib; + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << " " << __LINE__ << ", " << loaded.name << " load filament_id: " << filament_id; + if (presets_collection->type() == Preset::TYPE_FILAMENT) { + if (filament_id.empty() && "Template" != vendor_name) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": can not find filament_id for " << preset_name; + //throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); + reason = "Can not find filament_id for " + preset_name; + return reason; + } + else { + filament_id_maps.emplace(preset_name, filament_id); + } + } + } + + // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. + if (alias_name.empty()) { + size_t end_pos = preset_name.find_first_of("@"); + if (end_pos != std::string::npos) { + alias_name = preset_name.substr(0, end_pos); + if (renamed_from.empty()) + // Add the preset name with the '@' character removed into the "renamed_from" list. + renamed_from.emplace_back(alias_name + preset_name.substr(end_pos + 1)); + boost::trim_right(alias_name); + } + } + if (alias_name.empty()) + loaded.alias = preset_name; + else { + loaded.alias = std::move(alias_name); + filaments.set_printer_hold_alias(loaded.alias, loaded); + } + loaded.renamed_from = std::move(renamed_from); + if (! substitution_context.empty()) + substitutions.push_back({ + preset_name, presets_collection->type(), PresetConfigSubstitutions::Source::ConfigBundle, + std::string(), std::move(substitution_context.substitutions) }); + if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) + config_maps.emplace(preset_name, loaded.config); + ++count; + //BBS: add config related logs + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", got preset %1%, from %2%")%loaded.name %subfile; + return reason; +} + //BBS: Load a config bundle file from json std::pair PresetBundle::load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) { // Enable substitutions for user config bundle, throw an exception when loading a system profile. ConfigSubstitutionContext substitution_context { compatibility_rule }; PresetsConfigSubstitutions substitutions; + // Errors already on this bundle when the load began; the cache stamp below + // counts only what this parse adds. + const int errors_at_entry = m_errors; //BBS: add config related logs - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%path.c_str()%compatibility_rule; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; if (flags.has(LoadConfigBundleAttribute::ResetUserProfile) || flags.has(LoadConfigBundleAttribute::LoadSystem)) // Reset this bundle, delete user profile files if SaveImported. this->reset(flags.has(LoadConfigBundleAttribute::SaveImported)); + // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only + // scans want a slice of one. Validation reads the JSONs whatever is cached. + const boost::filesystem::path dir_path(dir); + const bool cacheable = flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name, base_bundle)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); - std::string root_file = path + "/" + vendor_name + ".json"; + std::string root_file = dir + "/" + vendor_name + ".json"; std::vector> machine_model_subfiles; std::vector> process_subfiles; std::vector> filament_subfiles; std::vector> machine_subfiles; auto get_name_and_subpath = [this](json::iterator& it, std::vector>& subfile_map) { if (it.value().is_array()) { - for (auto iter1 = it.value().begin(); iter1 != it.value().end(); iter1++) { + size_t index = 0; + for (auto iter1 = it.value().begin(); iter1 != it.value().end(); iter1++, index++) { if (iter1.value().is_object()) { std::string name, subpath; for (auto iter2 = iter1.value().begin(); iter2 != iter1.value().end(); iter2++) { @@ -4802,7 +5096,10 @@ std::pair PresetBundle::load_vendor_configs_ subfile_map.push_back(std::make_pair(name, subpath)); } else { ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": invalid type for " << iter1.key(); + // An array element has no key, and asking one for it throws + // nlohmann's invalid_iterator — not a parse_error, so it would + // escape the catch around this parse. Say where it is instead. + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": invalid type for " << it.key() << "[" << index << "]"; } } } else { @@ -4823,7 +5120,7 @@ std::pair PresetBundle::load_vendor_configs_ if (! config_version) { ++m_errors; throw ConfigurationError((boost::format("vendor %1%'s config version: %2% invalid\nSuggest cleaning the directory %3% firstly") - % vendor_name % version_str % path).str()); + % vendor_name % version_str % dir).str()); } else { vendor_profile.config_version = std::move(*config_version); } @@ -4861,7 +5158,7 @@ std::pair PresetBundle::load_vendor_configs_ catch(nlohmann::detail::parse_error &err) { BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": parse "< PresetBundle::load_vendor_configs_ //2) paste the machine model for (auto& machine_model : machine_model_subfiles) { - std::string subfile = path + "/" + vendor_name + "/" + machine_model.second; + std::string subfile = dir + "/" + vendor_name + "/" + machine_model.second; VendorProfile::PrinterModel model; model.id = machine_model.first; try { @@ -4976,7 +5273,7 @@ std::pair PresetBundle::load_vendor_configs_ catch(nlohmann::detail::parse_error &err) { BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": parse "<< subfile <<" got a nlohmann::detail::parse_error, reason = " << err.what(); throw ConfigurationError((boost::format("Failed loading configuration file %1%: %2%\nSuggest cleaning the directory %3% firstly") - %subfile %err.what() % path).str()); + %subfile %err.what() % dir).str()); } if (! model.id.empty() && ! model.variants.empty()) @@ -4985,7 +5282,6 @@ std::pair PresetBundle::load_vendor_configs_ //insert the vendor profile this->vendors.emplace(vendor_name, vendor_profile); - const VendorProfile* current_vendor_profile = &this->vendors[vendor_name]; BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", loaded vendor profile, name %1%, id %2%, version %3%")%vendor_profile.name%vendor_profile.id%vendor_profile.config_version.to_string(); @@ -4996,123 +5292,65 @@ std::pair PresetBundle::load_vendor_configs_ PresetCollection *presets = nullptr; size_t presets_loaded = 0; - auto parse_subfile = [this, path, vendor_name, presets_loaded, current_vendor_profile, base_bundle]( + // Parse one subfile into a source-form entry — everything the JSON states, + // nothing resolved. Loading the entry (load_vendor_preset) is the + // same code whether the entry was parsed just now or deserialized from the + // vendor's cache. + auto parse_subfile = [this, dir, vendor_name]( ConfigSubstitutionContext& substitution_context, - PresetsConfigSubstitutions& substitutions, - LoadConfigBundleAttributes& flags, - std::pair& subfile_iter, - std::map& config_maps, - std::map& filament_id_maps, - PresetCollection* presets_collection, - size_t& count, bool is_from_lib = false) -> std::string { + const std::pair& subfile_iter, + CachedPreset& entry) -> std::string { - std::string subfile = path + "/" + vendor_name + "/" + subfile_iter.second; - // Load the print, filament or printer preset. - std::string preset_name; - DynamicPrintConfig config; - std::string alias_name, inherits, description, instantiation, setting_id, filament_id; - std::vector renamed_from; - const DynamicPrintConfig* default_config = nullptr; - std::string reason; + std::string subfile = dir + "/" + vendor_name + "/" + subfile_iter.second; + std::string reason; try { std::map key_values; substitution_context.substitutions.clear(); //parse the json elements - DynamicPrintConfig config_src; - std::string _renamed_from_str; - config_src.load_from_json(subfile, substitution_context, false, key_values, reason); + entry.sub_path = subfile_iter.second; + entry.config_src.load_from_json(subfile, substitution_context, false, key_values, reason); if (!reason.empty()) { ++m_errors; BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": load config file "<second; + entry.setting_id = setting_it->second; auto filament_it = key_values.find(BBL_JSON_KEY_FILAMENT_ID); if (filament_it != key_values.end()) - filament_id = filament_it->second; - //check whether it inherits other preset or not + entry.filament_id = filament_it->second; auto it1 = key_values.find(BBL_JSON_KEY_INHERITS); if (it1 != key_values.end()) { - inherits = it1->second; - auto it2 = config_maps.find(inherits); - default_config = nullptr; - if (it2 != config_maps.end()) - default_config = &(it2->second); - if(default_config == nullptr && base_bundle != nullptr) { - auto base_it2 = base_bundle->m_config_maps.find(inherits); - if (base_it2 != base_bundle->m_config_maps.end()) - default_config = &(base_it2->second); - } - if (default_config != nullptr) { - if (filament_id.empty() && (presets_collection->type() == Preset::TYPE_FILAMENT)) { - auto filament_id_map_iter = filament_id_maps.find(inherits); - if (filament_id_map_iter != filament_id_maps.end()) { - filament_id = filament_id_map_iter->second; - } - if (filament_id.empty() && base_bundle != nullptr) { - auto filament_id_map_iter = base_bundle->m_filament_id_maps.find(inherits); - if (filament_id_map_iter != base_bundle->m_filament_id_maps.end()) { - filament_id = filament_id_map_iter->second; - } - } - } - } - else { + entry.inherits = it1->second; + // An `inherits` key naming nothing can never resolve; fail it + // here so install can key off the empty string as "no inherits". + if (entry.inherits.empty()) { ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << inherits << " for " << preset_name; - // throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find inherits: " + inherits; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << entry.name; + reason = "Can not find inherits: " + entry.inherits; return reason; } } - else { - if (presets_collection->type() == Preset::TYPE_PRINTER) - default_config = &presets_collection->default_preset_for(config_src).config; - else - default_config = &presets_collection->default_preset().config; - } - config = *default_config; - config.apply(config_src); - extend_default_config_length(config, true, *default_config); - if (instantiation == "false" && "Template" != vendor_name) { - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } - - config_maps.emplace(preset_name, std::move(config)); - if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!filament_id.empty())) - filament_id_maps.emplace(preset_name, filament_id); - return reason; - } - if (config.has("alias")) - alias_name = (dynamic_cast(config.option("alias")))->value; - if (key_values.find(ORCA_JSON_KEY_RENAMED_FROM) != key_values.end()) { - if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], renamed_from)) { - BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << path << "\": The preset \"" << preset_name + if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], entry.renamed_from)) { + BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << dir << "\": The preset \"" << entry.name << "\" contains invalid \"renamed_from\" key, which is being ignored."; } } - Preset::normalize(config); } catch(nlohmann::detail::parse_error &err) { ++m_errors; @@ -5120,195 +5358,60 @@ std::pair PresetBundle::load_vendor_configs_ reason = std::string("json parse error") + err.what(); return reason; } - - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } - - if (presets_collection->type() == Preset::TYPE_PRINTER) { - // Filter out printer presets, which are not mentioned in the vendor profile. - // These presets are considered not installed. - auto printer_model = config.opt_string("printer_model"); - if (printer_model.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer model, it will be ignored."; - reason = std::string("can not find printer_model"); - return reason; - } - auto printer_variant = config.opt_string("printer_variant"); - if (printer_variant.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer variant, it will be ignored."; - reason = std::string("can not find printer_variant"); - return reason; - } - auto it_model = std::find_if(current_vendor_profile->models.cbegin(), current_vendor_profile->models.cend(), - [&](const VendorProfile::PrinterModel &m) { return m.id == printer_model; } - ); - if (it_model == current_vendor_profile->models.end()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer model \"" << printer_model << "\", it will be ignored."; - reason = std::string("can not find printer model in vendor profile"); - return reason; - } - auto it_variant = it_model->variant(printer_variant); - if (it_variant == nullptr) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer variant \"" << printer_variant << "\", it will be ignored."; - reason = std::string("can not find printer_variant in vendor profile"); - return reason; - } - // An instantiation printer profile's nozzle_diameter must match the numeric (diameter) - // prefix of its printer_variant: "0.4" -> {0.4}, "0.8HF" -> {0.8} (a trailing - // non-numeric suffix such as "HF"/"HS" distinguishes a hardware sub-variant and is - // ignored here), and for multi-nozzle printers "0.4+0.6" -> {0.4, 0.6}. - // Note: a variant may legitimately repeat across presets of the same model (e.g. speed - // modes, IDEX copy/mirror, or different control boards), so only the diameter is - // validated, not variant uniqueness. Validation-only so the app keeps loading existing - // profiles unchanged. - if (validation_mode && instantiation == "true") { - const auto *nd = config.option("nozzle_diameter"); - std::set nozzles, variant_nozzles; - if (nd != nullptr) - nozzles.insert(nd->values.begin(), nd->values.end()); - std::vector variant_tokens; - boost::algorithm::split(variant_tokens, printer_variant, boost::algorithm::is_any_of("+")); - bool variant_ok = true; // printer_variant is already guaranteed non-empty above - for (const std::string &tok : variant_tokens) { - size_t consumed = 0; - double d = string_to_double_decimal_point(tok, &consumed); - // Require a leading numeric diameter; a trailing suffix (e.g. "HF") is allowed. - if (consumed == 0) { variant_ok = false; break; } - variant_nozzles.insert(d); - } - if (!variant_ok || variant_nozzles != nozzles) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" has printer_variant \"" << printer_variant << - "\" that does not match its nozzle_diameter \"" << (nd ? nd->serialize() : std::string()) << "\". " - "printer_variant must begin with the nozzle diameter, optionally followed by a non-numeric suffix " - "(e.g. \"0.4\", \"0.8HF\"); for multi-nozzle printers, join the per-nozzle diameters with \"+\" in " - "nozzle order (e.g. \"0.4+0.6\")."; - } - } - } - const Preset *preset_existing = presets_collection->find_preset(preset_name, false); - if (preset_existing != nullptr) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" has already been loaded from another Config Bundle."; - reason = std::string("duplicated defines"); - return reason; - } - - auto file_path = (boost::filesystem::path(data_dir()) /PRESET_SYSTEM_DIR/ vendor_name / subfile_iter.second).make_preferred(); - if(validation_mode) - file_path = (boost::filesystem::path(data_dir()) / vendor_name / subfile_iter.second).make_preferred(); - - // Load the preset into the list of presets, save it to disk. - Preset &loaded = presets_collection->load_preset(file_path.string(), preset_name, std::move(config), false); - if (flags.has(LoadConfigBundleAttribute::LoadSystem)) { - loaded.is_system = true; - loaded.vendor = current_vendor_profile; - loaded.version = current_vendor_profile->config_version; - loaded.description = description; - loaded.setting_id = setting_id; - // Derive the preset setting_id on the fly when a profile ships without one, - // matching scripts/assign_vendor_setting_ids.py. Only instantiated presets - // carry an id; non-instantiated base profiles return earlier above. This never - // touches the per-user cloud-sync setting_id written into user .info files. - if (loaded.setting_id.empty() && instantiation == "true") - loaded.setting_id = generate_preset_setting_id( - vendor_name, Preset::get_type_string(presets_collection->type()), preset_name); - loaded.filament_id = filament_id; - loaded.m_from_orca_filament_lib = is_from_lib; - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << " " << __LINE__ << ", " << loaded.name << " load filament_id: " << filament_id; - if (presets_collection->type() == Preset::TYPE_FILAMENT) { - if (filament_id.empty() && "Template" != vendor_name) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": can not find filament_id for " << preset_name; - //throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find filament_id for " + preset_name; - return reason; - } - else { - filament_id_maps.emplace(preset_name, filament_id); - } - } - } - - // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. - if (alias_name.empty()) { - size_t end_pos = preset_name.find_first_of("@"); - if (end_pos != std::string::npos) { - alias_name = preset_name.substr(0, end_pos); - if (renamed_from.empty()) - // Add the preset name with the '@' character removed into the "renamed_from" list. - renamed_from.emplace_back(alias_name + preset_name.substr(end_pos + 1)); - boost::trim_right(alias_name); - } - } - if (alias_name.empty()) - loaded.alias = preset_name; - else { - loaded.alias = std::move(alias_name); - filaments.set_printer_hold_alias(loaded.alias, loaded); - } - loaded.renamed_from = std::move(renamed_from); - if (! substitution_context.empty()) - substitutions.push_back({ - preset_name, presets_collection->type(), PresetConfigSubstitutions::Source::ConfigBundle, - std::string(), std::move(substitution_context.substitutions) }); - config_maps.emplace(preset_name, loaded.config); - ++count; - //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", got preset %1%, from %2%")%loaded.name %subfile; return reason; }; std::map configs; std::map filament_id_maps; + // Orca: whether to (re)write the vendor's cache after this parse, leaving it + // in step with the profile so the next run reads it instead. It is written + // where the vendor was looked for, even when the profile came from resources, + // and stamped with the version that profile claims — a profile without one + // cannot be judged for staleness later, and a cache nothing can invalidate is + // worse than none. + const bool will_cache = cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); + VendorCacheData cache_data; + // Errors added by install are counted apart: a cache load runs install again, + // so the parse_errors stamped into the cache must hold only what a cache load + // will not recount. + int install_errors = 0; + auto load_subfiles = [&](std::vector>& subfiles, + std::vector& entries, const char* kind, bool is_from_lib = false) { + configs.clear(); + filament_id_maps.clear(); + for (auto& subfile : subfiles) { + CachedPreset entry; + std::string reason = parse_subfile(substitution_context, subfile, entry); + if (reason.empty()) { + const int errors_before_install = m_errors; + reason = load_vendor_preset(entry, dir, vendor_name, base_bundle, flags, + substitution_context, substitutions, configs, filament_id_maps, presets, + presets_loaded, is_from_lib); + install_errors += m_errors - errors_before_install; + } + if (!reason.empty()) { + ++m_errors; + //parse error + std::string subfile_path = dir + "/" + vendor_name + "/" + subfile.second; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % kind % subfile_path; + throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % dir).str()); + } + if (will_cache) + entries.emplace_back(std::move(entry)); + } + }; + + // The section order below — process, filaments (with the ORCA-lib map copy), + // printers — is mirrored by load_vendor_cache's install loops; keep the two + // in lockstep. //3.1) paste the process presets = &this->prints; - configs.clear(); - filament_id_maps.clear(); - for (auto& subfile : process_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, presets_loaded); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse process setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } - } + load_subfiles(process_subfiles, cache_data.process_entries, "process"); //3.2) paste the filaments presets = &this->filaments; - configs.clear(); - filament_id_maps.clear(); const auto is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; - for (auto& subfile : filament_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, - presets_loaded, is_orca_lib); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse filament setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } - } + load_subfiles(filament_subfiles, cache_data.filament_entries, "filament", is_orca_lib); if (is_orca_lib) { m_config_maps = configs; m_filament_id_maps = filament_id_maps; @@ -5316,18 +5419,16 @@ std::pair PresetBundle::load_vendor_configs_ //3.3) paste the printers presets = &this->printers; - configs.clear(); - filament_id_maps.clear(); - for (auto& subfile : machine_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, presets_loaded); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse printer setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } + load_subfiles(machine_subfiles, cache_data.machine_entries, "printer"); + + if (will_cache) { + // Clamped: the count is a difference of three tallies, and a stamp that + // wrapped would be added to every future load of this vendor. + cache_data.parse_errors = uint64_t(std::max(0, m_errors - errors_at_entry - install_errors)); + cache_data.vendors = this->vendors; + if (! VendorCacheFile::save((dir_path / (vendor_name + ".opc")).string(), vendor_name, + vendor_profile.config_version.to_string(), cache_data)) + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; } //BBS: add config related logs @@ -5952,4 +6053,94 @@ bool BundleMetadata::save_to_json(const std::string& path) const return false; } } +// ---- Per-vendor preset cache: install into this bundle ------------------- +// The file format itself lives in PresetCacheFormat.cpp (VendorCacheFile). + +bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle) +{ + // A vendor is loaded from where it is installed and nowhere else; resources + // reaches the app by being installed into `dir` first. The cache there is + // judged against the profile beside it — or, where the cache is the whole + // of the installation, against nothing, since nothing on disk can then be + // newer than it. That state is Semver::inf(), which no real profile carries. + const boost::filesystem::path profile = dir / (vendor_name + ".json"); + const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) + : Semver::inf(); + return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, base_bundle); +} + +bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, const PresetBundle* base_bundle) +{ + // What this bundle had counted before the cache was tried. The caller + // measures its own parse against this same baseline, so a rejection must + // put it back rather than reset it to zero. + const int errors_at_entry = this->m_errors; + // Read and validated before this bundle is touched: a rejected file leaves + // no state to roll back. + VendorCacheData data; + if (! VendorCacheFile::load(cache_path, expected_vendor_name, expected_vendor_version, data)) + return false; + try { + const std::string& vendor_name = expected_vendor_name; // VendorCacheFile::load checked they match + this->vendors = std::move(data.vendors); + + // What the parse counted before install took over; install recounts its + // own below, so m_errors comes out as a JSON parse would leave it. + m_errors += int(data.parse_errors); + + // Install the entries exactly as load_vendor_configs_from_json installs + // them straight after parsing — same code, same order. The substitution + // context stays empty (the entries were substituted when they were + // parsed), so no substitutions are reported, as before. + ConfigSubstitutionContext substitution_context { ForwardCompatibilitySubstitutionRule::EnableSilent }; + PresetsConfigSubstitutions substitutions; + std::map configs; + std::map filament_id_maps; + const std::string path = boost::filesystem::path(cache_path).parent_path().string(); + size_t count = 0; + auto install_entries = [&](const std::vector& entries, PresetCollection* presets, bool is_from_lib) { + configs.clear(); + filament_id_maps.clear(); + // Only configs of presets that other entries inherit are ever looked + // up again; registering just those skips one full config copy for + // every leaf preset. The library's filaments are all retained — they + // become the m_config_maps other vendors resolve against. + std::set inherited; + for (const CachedPreset& entry : entries) + if (! entry.inherits.empty()) + inherited.insert(entry.inherits); + const std::set* retain_configs = is_from_lib ? nullptr : &inherited; + for (const CachedPreset& entry : entries) { + const std::string reason = load_vendor_preset(entry, path, vendor_name, + base_bundle, LoadConfigBundleAttribute::LoadSystem, substitution_context, substitutions, + configs, filament_id_maps, presets, count, is_from_lib, retain_configs); + if (! reason.empty()) + throw std::runtime_error("entry " + entry.name + " failed to install: " + reason); + } + }; + install_entries(data.process_entries, &this->prints, false); + const bool is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; + install_entries(data.filament_entries, &this->filaments, is_orca_lib); + if (is_orca_lib) { + m_config_maps = configs; + m_filament_id_maps = filament_id_maps; + } + install_entries(data.machine_entries, &this->printers, false); + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); + // Restore a clean state so the caller can fall back to the JSON parse. + this->reset(false); + this->vendors.clear(); + this->m_config_maps.clear(); + this->m_filament_id_maps.clear(); + this->m_errors = errors_at_entry; + // A failure partway through installing may have left presets in some + // collections with hold aliases already registered. + this->clear_printer_hold_aliases(); + return false; + } +} + } // namespace Slic3r diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 685687975b..9da8fb4251 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -2,10 +2,12 @@ #define slic3r_PresetBundle_hpp_ #include "Preset.hpp" +#include "PresetCacheFormat.hpp" #include "AppConfig.hpp" #include "enum_bitmask.hpp" #include +#include #include #include #include @@ -170,6 +172,31 @@ struct PresetBundleMetadata class PresetBundle { public: + // ---- Per-vendor preset cache -------------------------------------------- + // One cache file per vendor (plus the Orca filament library), stamped with + // the vendor's own profile version rather than a directory scan. The bytes + // on disk are VendorCacheFile's business (PresetCacheFormat.hpp); what + // lives here is how a cache's contents install into a bundle. + + // The cache is not something a caller loads from: a vendor is loaded with + // load_vendor_configs_from_json, which comes from the cache whenever one covers + // it. What is public here is what the cache's own tests drive directly. + + // Load a per-vendor cache into this bundle by installing its entries, with + // base_bundle's filament library as the inheritance base. Rejects (returns + // false, with this bundle left clean) unless VendorCacheFile::load accepts + // the file — see its contract for the version and identity checks — and + // every entry installs. Options this build no longer defines are dropped, + // not fatal — the payload names its own keys. + bool load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, const PresetBundle* base_bundle = nullptr); + + // Enable writing a per-vendor cache after a JSON parse (off by default). Cache + // content is pure parse output, so the guard is policy, not correctness: only + // the deliberate generators (load_system_presets_from_json, the cache build + // tool) write files, not every incidental load a dialog performs. + void set_generate_vendor_caches(bool enable) { m_generate_vendor_caches = enable; } + static DynamicPrintConfig construct_full_config(Preset &in_printer_preset, Preset &in_print_preset, const DynamicPrintConfig &project_config, @@ -444,8 +471,12 @@ public: /*std::pair load_configbundle( const std::string &path, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule);*/ //Orca: load config bundle from json, pass the base bundle to support cross vendor inheritance + // Orca: `dir` is where the vendor is looked for — its own directory, whether or + // not the profile JSONs are still there. A whole-vendor load comes from the + // vendor's preset cache whenever one covers the profile on disk, and is parsed + // from the JSONs in `dir` only when none does. Nothing here reads resources. std::pair load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); // Export a config bundle file containing all the presets and the names of the active presets. //void export_configbundle(const std::string &path, bool export_system_settings = false, bool export_physical_printers = false); @@ -517,11 +548,49 @@ public: // Orca: for validation only. bool has_errors(bool check_duplicate_filament_subtypes = false) const; + // Errors the last load recorded. What the cache's error accounting promises — + // a cache-served vendor reports what its parse would — is pinned against this. + int error_count() const { return m_errors; } + // Orca: for validation only. Flag any system preset whose inherits / compatible_printers / // compatible_prints references a deleted (unknown) or renamed (old) preset name. bool check_preset_references() const; + // Merge one vendor's presets with the other vendor's presets, report duplicates. + // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a + // bundle out of several per-vendor caches loaded into separate PresetBundle instances. + std::vector merge_presets(PresetBundle &&other); + private: + // Load one vendor from the preset cache installed in `dir`, judged against + // the vendor profile there. False, with this bundle left clean, when there + // is no usable cache and the vendor has to be parsed. This is how + // load_vendor_configs_from_json reads a cache. + bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle); + + // Load one source-form preset entry into this bundle: resolve `inherits`, + // flatten, validate and register the preset. Returns the reason loading + // failed, empty on success. See the definition for the sharing contract + // between the JSON parse and the cache load. + // retain_configs, when non-null, names the only presets registered into + // config_maps (a full config copy each). The cache load passes the names its + // entries inherit — the only ones ever looked up again; the JSON parse + // retains all, not knowing what later subfiles inherit. + std::string load_vendor_preset(const CachedPreset& entry, + const std::string& path, const std::string& vendor_name, + const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, + ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, + std::map& config_maps, std::map& filament_id_maps, + PresetCollection* presets_collection, size_t& count, bool is_from_lib, + const std::set* retain_configs = nullptr); + + // Clear every collection's m_printer_hold_alias, which reset() leaves alone. + void clear_printer_hold_aliases(); + + // Whether to (re)write a per-vendor cache after a JSON parse. + bool m_generate_vendor_caches { false }; + // Orca: validation only - flag any printer with two or more compatible // filament presets sharing one filament_id (ambiguous AMS subtype match). bool check_duplicate_filament_subtypes() const; @@ -529,8 +598,6 @@ private: //std::pair load_system_presets(ForwardCompatibilitySubstitutionRule compatibility_rule); //BBS: add json related logic std::pair load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule); - // Merge one vendor's presets with the other vendor's presets, report duplicates. - std::vector merge_presets(PresetBundle &&other); // Update the multicolor information for filaments. void update_filament_multi_color(); // Update renamed_from and alias maps of system profiles. diff --git a/src/libslic3r/PresetCacheFormat.cpp b/src/libslic3r/PresetCacheFormat.cpp new file mode 100644 index 0000000000..accebba7b4 --- /dev/null +++ b/src/libslic3r/PresetCacheFormat.cpp @@ -0,0 +1,588 @@ +#include "libslic3r/PresetCacheFormat.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libslic3r/Utils.hpp" + +namespace Slic3r { + +CacheDictionary::CacheDictionary() +{ + // ENUM_UNNAMED is index 0 and always the empty name. + m_enum_values.emplace_back(); +} + +// The ints an enum option holds — one for a coEnum, the whole vector for coEnums. +static std::vector enum_ints(const ConfigOptionDef& def, const ConfigOption* opt) +{ + if (def.type == coEnum) + return { opt->getInt() }; + return static_cast(opt)->values; +} + +// The name this build gives one of those ints, empty where it has none — a +// nullable option's nil, or a definition carrying no enum_keys_map. Enums are +// written by name so a build that reorders an enum's values still reads it right. +static std::string enum_name_of(const ConfigOptionDef& def, int value) +{ + if (def.enum_keys_map != nullptr) + for (const auto& kvp : *def.enum_keys_map) + if (kvp.second == value) + return kvp.first; + return {}; +} + +void CacheDictionary::collect(const DynamicPrintConfig& config) +{ + for (auto it = config.cbegin(); it != config.cend(); ++ it) { + const ConfigOptionDef* def = print_config_def.get(it->first); + if (def == nullptr) + continue; // save_config does not write it either + if (m_key_index.try_emplace(it->first, uint16_t(m_keys.size())).second) { + m_keys.push_back(it->first); + m_types.push_back(uint16_t(def->type)); + } + if (def->type != coEnum && def->type != coEnums) + continue; + for (int value : enum_ints(*def, it->second.get())) { + std::string name = enum_name_of(*def, value); + if (! name.empty() && m_enum_index.try_emplace(name, uint16_t(m_enum_values.size())).second) + m_enum_values.push_back(std::move(name)); + } + } +} + +uint16_t CacheDictionary::key_index(const t_config_option_key& key) const +{ + auto it = m_key_index.find(key); + if (it == m_key_index.end()) + throw std::runtime_error("preset cache: option " + key + " was never collected into the dictionary"); + return it->second; +} + +uint16_t CacheDictionary::enum_index(const std::string& name) const +{ + if (name.empty()) + return ENUM_UNNAMED; + auto it = m_enum_index.find(name); + return it == m_enum_index.end() ? ENUM_UNNAMED : it->second; +} + +void CacheDictionary::save(cereal::BinaryOutputArchive& ar) const +{ + // Checked here rather than left to the caller: an index that wrapped would + // be written silently, and nothing downstream could tell. + if (m_keys.size() > MAX_ENTRIES || m_enum_values.size() > MAX_ENTRIES) + throw std::runtime_error("preset cache: the option dictionary outgrew the uint16 it is indexed with"); + ar(m_keys, m_types, m_enum_values); +} + +void CacheDictionary::load(cereal::BinaryInputArchive& ar) +{ + ar(m_keys, m_types, m_enum_values); + if (m_keys.size() != m_types.size()) + throw std::runtime_error("preset cache: dictionary key and type tables differ in length"); + if (m_keys.size() > MAX_ENTRIES || m_enum_values.size() > MAX_ENTRIES) + throw std::runtime_error("preset cache: dictionary is larger than the uint16 it is indexed with"); + if (m_enum_values.empty() || ! m_enum_values.front().empty()) + throw std::runtime_error("preset cache: dictionary is missing its unnamed-enum slot"); + // Resolved once per file: every option read after this is a vector index. + m_defs.resize(m_keys.size()); + for (size_t i = 0; i < m_keys.size(); ++ i) { + const ConfigOptionDef* def = print_config_def.get(m_keys[i]); + m_defs[i] = (def != nullptr && uint16_t(def->type) == m_types[i]) ? def : nullptr; + } +} + +// ---- one config ----------------------------------------------------------- + +static void save_enum_option(cereal::BinaryOutputArchive& ar, const ConfigOptionDef& def, + const ConfigOption* opt, const CacheDictionary& dict) +{ + const std::vector values = enum_ints(def, opt); + ar(uint32_t(values.size())); + for (int value : values) { + const uint16_t idx = dict.enum_index(enum_name_of(def, value)); + ar(idx); + if (idx == CacheDictionary::ENUM_UNNAMED) + ar(int32_t(value)); + } +} + +// `config` may be null, in which case the option is read and dropped. +static void load_enum_option(cereal::BinaryInputArchive& ar, ConfigOptionType type, + const ConfigOptionDef* def, DynamicPrintConfig* config, + const CacheDictionary& dict) +{ + uint32_t cnt = 0; + ar(cnt); + if (type == coEnum && cnt != 1) + throw std::runtime_error("preset cache: a scalar enum carrying more than one value"); + // Every element is read whatever happens, so the stream stays in sync and + // whatever follows this option still loads. + bool usable = def != nullptr && config != nullptr; + std::vector values; + values.reserve(cnt); + for (uint32_t i = 0; i < cnt; ++ i) { + uint16_t idx = 0; + ar(idx); + if (! dict.valid_enum_index(idx)) + throw std::runtime_error("preset cache: enum value index past the end of the dictionary"); + if (idx == CacheDictionary::ENUM_UNNAMED) { + // An int the writer could not name — a nil, or an option whose + // definition carried no enum_keys_map. It travels verbatim. + int32_t raw = 0; + ar(raw); + values.push_back(int(raw)); + continue; + } + if (! usable) + continue; // the index above was this element's whole payload + if (def->enum_keys_map == nullptr) { + usable = false; // this build no longer maps this option's names + continue; + } + const auto it = def->enum_keys_map->find(dict.enum_name_at(idx)); + if (it == def->enum_keys_map->end()) { + usable = false; // a value this build dropped: the option goes with it + continue; + } + values.push_back(it->second); + } + if (! usable) + return; + if (type == coEnum) { + config->set_key_value(def->opt_key, new ConfigOptionEnumGeneric(def->enum_keys_map, values.front())); + } else { + auto* opt = def->nullable ? static_cast(new ConfigOptionEnumsGenericNullable(def->enum_keys_map)) + : static_cast(new ConfigOptionEnumsGeneric(def->enum_keys_map)); + opt->values = std::move(values); + config->set_key_value(def->opt_key, opt); + } +} + +void save_config(cereal::BinaryOutputArchive& ar, const DynamicPrintConfig& config, const CacheDictionary& dict) +{ + struct Written { uint16_t idx; const ConfigOptionDef* def; const ConfigOption* opt; }; + std::vector written; + written.reserve(config.size()); + for (auto it = config.cbegin(); it != config.cend(); ++ it) + if (const ConfigOptionDef* def = print_config_def.get(it->first)) + written.push_back({ dict.key_index(it->first), def, it->second.get() }); + + ar(uint32_t(written.size())); + for (const Written& w : written) { + ar(w.idx); + if (w.def->type == coEnum || w.def->type == coEnums) + save_enum_option(ar, *w.def, w.opt, dict); + else + w.def->save_option_to_archive(ar, w.opt); + } +} + +// `config` null means: read everything, keep nothing. +static void read_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig* config, const CacheDictionary& dict) +{ + uint32_t cnt = 0; + ar(cnt); + if (config != nullptr) + config->clear(); + // Reused across the loop: constructing a ConfigOptionDef per dropped option + // would allocate its strings and vectors for nothing. + ConfigOptionDef scratch; + for (uint32_t i = 0; i < cnt; ++ i) { + uint16_t idx = 0; + ar(idx); + if (! dict.valid_key_index(idx)) + throw std::runtime_error("preset cache: option index past the end of the dictionary"); + const ConfigOptionType type = dict.type_at(idx); + const ConfigOptionDef* def = dict.def_at(idx); + if (type == coEnum || type == coEnums) { + load_enum_option(ar, type, def, config, dict); + } else if (def != nullptr && config != nullptr) { + config->set_key_value(def->opt_key, def->load_option_from_archive(ar)); + } else { + // Read by the type the writer recorded, then drop: the same outcome + // a JSON profile gets for an option this build no longer has. + scratch.type = type; + std::unique_ptr discard(scratch.load_option_from_archive(ar)); + } + } +} + +void load_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig& config, const CacheDictionary& dict) +{ + read_config(ar, &config, dict); +} + +void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict) +{ + read_config(ar, nullptr, dict); +} + +// ---- The per-vendor cache file (.opc) ----------------------------- + +namespace { + +#pragma pack(push, 1) +struct CacheFileHeader { + uint32_t magic; + uint32_t version; + uint64_t data_size; + uint32_t crc32; +}; +#pragma pack(pop) +static_assert(sizeof(CacheFileHeader) == 20, "CacheFileHeader must be 20 bytes"); + +constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ" +// Bump when the wire format changes in a way the payload cannot describe +// itself out of: reordering, removing or retyping a field of a hand-written +// serialize() (VendorProfile and its nested types, CachedPreset via +// save_entries below), or a change to the cache's own layout or the +// meaning of its stamps. Option-schema drift is NOT such a change — the +// dictionary handles it, which is why this no longer moves every release. +constexpr uint32_t CACHE_VERSION = 1; + +// A stamp-string read that refuses an absurd length before allocating anything. +// The stamps are read from files named from the outside (peek_version is +// pointed at whatever .opc a directory holds), so the length word may +// be arbitrary bytes — and a resize to a garbage 64-bit length does not fail as +// a catchable bad_alloc here, it takes the app down through the out-of-memory +// handler. A vendor name or profile version is a short token; anything longer +// is not a cache this build wrote. +std::string read_bounded_string(cereal::BinaryInputArchive& ar) +{ + constexpr uint64_t MAX_STAMP_LEN = 1024; + cereal::size_type len = 0; + ar(cereal::make_size_tag(len)); + if (uint64_t(len) > MAX_STAMP_LEN) + throw std::runtime_error("preset cache: string length out of bounds"); + std::string s(size_t(len), '\0'); + ar(cereal::binary_data(s.data(), size_t(len))); + return s; +} + +// The prologue every cache reader starts with: the format version, then the +// vendor's identity. Returns the vendor version stamped on a body this build can +// read, empty on anything else — which is the same answer as "not this vendor". +std::string read_cache_stamps(cereal::BinaryInputArchive& ar, const std::string& expected_vendor_name) +{ + // The version is judged before anything variable-length is read: on a body + // that is not a per-vendor cache of this version, the bytes where a string + // length would sit may be arbitrary framing. + uint32_t cache_version = 0; + ar(cache_version); + if (cache_version != CACHE_VERSION) + return {}; + const std::string vendor_name = read_bounded_string(ar); + const std::string vendor_version = read_bounded_string(ar); + if (vendor_name != expected_vendor_name) + return {}; + return vendor_version; +} + +// A cache stays usable as long as it was built from a vendor profile at least +// as new as the one now on disk. Profiles whose version is invalid cannot be +// judged this way and are never served from cache; where no profile sits +// beside the cache at all, nothing can be newer than it — that state is passed +// as Semver::inf(), which no real profile can carry (an invalid version could +// not say it apart from "profile there but unjudgeable", and zero would +// collide with a genuine "0.0.0"). This is the serve rule; the install rule +// (cache_covers in PresetBundle.cpp) deliberately reads an unjudgeable profile +// the other way, so the two are not one function. +bool cache_covers_version(const std::string& cached, const Semver& on_disk) +{ + if (on_disk == Semver::inf()) + return true; // before parsing `cached`: nothing exists that the stamp must cover + if (! on_disk.valid()) + return false; + const auto cached_ver = Semver::parse(cached); + return cached_ver && *cached_ver >= on_disk; +} + +// CachedPreset on the wire: all fields, declaration order, in one place. +// `config` writes, reads or skips the config sitting in the middle of that +// order — the three things a reader can want to do with it — so save, load and +// the name peek below cannot drift apart. Keep in sync with the struct in +// PresetCacheFormat.hpp and bump CACHE_VERSION on change. Written here rather +// than as a serialize() member because the config needs the file's dictionary, +// which cereal cannot thread through one. +template +void visit_entry(Archive& ar, Entry& e, ConfigFn&& config) +{ + ar(e.name, e.sub_path); + config(); + ar(e.inherits, e.description, e.instantiation, e.setting_id, e.filament_id, e.renamed_from); +} + +// The count comes from a file that has already passed magic and CRC, but a +// reserve is a promise to allocate: cap it and let push_back grow the rest. +constexpr uint32_t MAX_RESERVED_ENTRIES = 4096; + +void save_entries(cereal::BinaryOutputArchive& ar, + const std::vector& entries, + const CacheDictionary& dict) +{ + ar(uint32_t(entries.size())); + for (const CachedPreset& e : entries) + visit_entry(ar, e, [&] { save_config(ar, e.config_src, dict); }); +} + +void load_entries(cereal::BinaryInputArchive& ar, + std::vector& entries, + const CacheDictionary& dict) +{ + uint32_t cnt = 0; + ar(cnt); + entries.clear(); + entries.reserve(std::min(cnt, MAX_RESERVED_ENTRIES)); + for (uint32_t i = 0; i < cnt; ++ i) { + CachedPreset e; + visit_entry(ar, e, [&] { load_config(ar, e.config_src, dict); }); + entries.push_back(std::move(e)); + } +} + +// Read a raw cache body: verify magic, size, CRC. +bool read_cache_blob(const std::string& path, std::string& out_blob) +{ + try { + boost::nowide::ifstream ifs(path, std::ios::binary); + if (!ifs.is_open()) + return false; + CacheFileHeader fhdr; + if (!ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr))) + return false; + if (fhdr.magic != CACHE_MAGIC) + return false; + // data_size is 8 bytes from a file nothing has authenticated yet, and + // it is about to size an allocation. The body is the whole of the file + // behind the header — anything else is not a cache this build wrote. + ifs.seekg(0, std::ios::end); + const std::streamoff file_size = ifs.tellg(); + if (file_size < std::streamoff(sizeof(fhdr)) || + fhdr.data_size == 0 || + fhdr.data_size != uint64_t(file_size) - sizeof(fhdr)) + return false; + ifs.seekg(sizeof(fhdr), std::ios::beg); + out_blob.assign(fhdr.data_size, '\0'); + if (!ifs.read(&out_blob[0], static_cast(fhdr.data_size))) + return false; + boost::crc_32_type crc; + crc.process_bytes(out_blob.data(), out_blob.size()); + if (crc.checksum() != fhdr.crc32) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: CRC mismatch: " << path; + return false; + } + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: read failed (" << path << "): " << e.what(); + return false; + } +} + +// Write a cache body behind the standard 20-byte file header. False when the +// file could not be opened or written whole. +bool write_cache_blob(const std::string& path, const std::string& blob) +{ + boost::crc_32_type crc; + crc.process_bytes(blob.data(), blob.size()); + // Written beside the target and moved into place, as AppConfig::save does: + // a cache is truncated and rewritten in full, so a write that dies partway + // would otherwise leave a header claiming more body than the file holds. + // The PID suffix also keeps two instances writing the same vendor from + // interleaving. + const std::string tmp_path = path + "." + std::to_string(get_current_pid()) + ".tmp"; + try { + boost::filesystem::create_directories(boost::filesystem::path(path).parent_path()); + { + boost::nowide::ofstream ofs(tmp_path, std::ios::binary | std::ios::trunc); + if (!ofs.is_open()) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: cannot open for writing: " << tmp_path; + return false; + } + CacheFileHeader fhdr; + fhdr.magic = CACHE_MAGIC; + fhdr.version = CACHE_VERSION; + fhdr.data_size = static_cast(blob.size()); + fhdr.crc32 = crc.checksum(); + ofs.write(reinterpret_cast(&fhdr), sizeof(fhdr)); + ofs.write(blob.data(), static_cast(blob.size())); + ofs.close(); // flush; close() raises failbit on error + if (! ofs.good()) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: write failed (" << tmp_path << ")"; + boost::system::error_code ec; + boost::filesystem::remove(tmp_path, ec); + return false; + } + } + if (const std::error_code ec = rename_file(tmp_path, path)) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: could not move " << tmp_path << " into place: " << ec.message(); + boost::system::error_code rm; + boost::filesystem::remove(tmp_path, rm); + return false; + } + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: write failed (" << path << "): " << e.what(); + boost::system::error_code ec; + boost::filesystem::remove(tmp_path, ec); + return false; + } +} + +} // anonymous namespace + +// static +bool VendorCacheFile::save(const std::string& path, const std::string& vendor_name, + const std::string& vendor_version, const VendorCacheData& data) +{ + try { + // Collected before anything is written: the dictionary sits ahead of the + // entries so a reader resolves it once and then indexes. + CacheDictionary dict; + for (const std::vector* entries : { &data.process_entries, &data.filament_entries, &data.machine_entries }) + for (const CachedPreset& e : *entries) + dict.collect(e.config_src); + + std::ostringstream body(std::ios::binary); + { + cereal::BinaryOutputArchive ar(body); + ar(CACHE_VERSION); + ar(vendor_name, vendor_version); + dict.save(ar); + ar(data.vendors); + save_entries(ar, data.process_entries, dict); + save_entries(ar, data.filament_entries, dict); + save_entries(ar, data.machine_entries, dict); + ar(data.parse_errors); + } + return write_cache_blob(path, body.str()); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: failed to save vendor cache " << path << ": " << e.what(); + return false; + } +} + +// static +bool VendorCacheFile::load(const std::string& path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, VendorCacheData& data) +{ + std::string blob; + if (! read_cache_blob(path, blob)) + return false; + try { + // Read in place: an istringstream would copy the blob once more just to + // stream over it. + boost::iostreams::stream body(blob.data(), blob.size()); + cereal::BinaryInputArchive ar(body); + const std::string vendor_version = read_cache_stamps(ar, expected_vendor_name); + if (vendor_version.empty() || ! cache_covers_version(vendor_version, expected_vendor_version)) + return false; + CacheDictionary dict; + dict.load(ar); + ar(data.vendors); + load_entries(ar, data.process_entries, dict); + load_entries(ar, data.filament_entries, dict); + load_entries(ar, data.machine_entries, dict); + ar(data.parse_errors); + if (data.vendors.find(expected_vendor_name) == data.vendors.end()) + throw std::runtime_error("vendor cache does not carry its own vendor profile"); + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: rejecting vendor cache " << path << ": " << e.what(); + return false; + } +} + +// static +std::string VendorCacheFile::peek_version(const std::string& path, const std::string& expected_vendor_name) +{ + try { + boost::nowide::ifstream ifs(path, std::ios::binary); + CacheFileHeader fhdr; + if (! ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr)) || fhdr.magic != CACHE_MAGIC) + return {}; + // Only the head of the body is read, and its CRC left unverified: the + // stamps sit at the front, and this answers "what version is this?" + // without paying for tens of megabytes. Callers that need to know the + // file is whole use usable_version instead. + std::string head(static_cast(std::min(fhdr.data_size, 1024)), '\0'); + if (! ifs.read(&head[0], static_cast(head.size()))) + return {}; + std::istringstream body(head, std::ios::binary); + cereal::BinaryInputArchive ar(body); + return read_cache_stamps(ar, expected_vendor_name); + } catch (const std::exception&) { + return {}; + } +} + +// static +Semver VendorCacheFile::usable_version(const std::string& path, const std::string& expected_vendor_name) +{ + std::string blob; + if (! read_cache_blob(path, blob)) + return Semver::invalid(); + try { + boost::iostreams::stream body(blob.data(), blob.size()); + cereal::BinaryInputArchive ar(body); + const auto ver = Semver::parse(read_cache_stamps(ar, expected_vendor_name)); + return ver ? *ver : Semver::invalid(); + } catch (const std::exception&) { + return Semver::invalid(); + } +} + +// static +bool VendorCacheFile::carries_preset(const std::string& path, const std::string& vendor_name, + Preset::Type type, const std::string& preset_name) +{ + std::string blob; + if (! read_cache_blob(path, blob)) + return false; + try { + boost::iostreams::stream body(blob.data(), blob.size()); + cereal::BinaryInputArchive ar(body); + if (read_cache_stamps(ar, vendor_name).empty()) + return false; + CacheDictionary dict; + dict.load(ar); + VendorMap vendors; + ar(vendors); + // Reused: every entry overwrites it, and only its name is ever looked at. + CachedPreset entry; + // Written in this order by save. The list that could carry the preset + // is the last one worth reading. + for (Preset::Type kind : { Preset::TYPE_PRINT, Preset::TYPE_FILAMENT, Preset::TYPE_PRINTER }) { + uint32_t cnt = 0; + ar(cnt); + for (uint32_t i = 0; i < cnt; ++ i) { + visit_entry(ar, entry, [&] { skip_config(ar, dict); }); + if (kind == type && entry.name == preset_name) + return true; + } + if (kind == type) + return false; + } + return false; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "VendorCacheFile: could not read preset names from " << path << ": " << e.what(); + return false; + } +} + +} // namespace Slic3r diff --git a/src/libslic3r/PresetCacheFormat.hpp b/src/libslic3r/PresetCacheFormat.hpp new file mode 100644 index 0000000000..b200ec9911 --- /dev/null +++ b/src/libslic3r/PresetCacheFormat.hpp @@ -0,0 +1,192 @@ +#ifndef slic3r_PresetCacheFormat_hpp_ +#define slic3r_PresetCacheFormat_hpp_ + +#include +#include +#include +#include + +#include +#include +#include + +#include "libslic3r/Config.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/PrintConfig.hpp" +#include "libslic3r/Semver.hpp" + +namespace Slic3r { + +// How the preset cache writes a DynamicPrintConfig. +// +// Not through the global cereal hooks in PrintConfig.hpp: those key an option by +// its serialization_key_ordinal, which ConfigDef::add assigns by declaration +// order at static-init time. Inserting one option into the middle of +// PrintConfig.cpp shifts every later ordinal, and the lookup on the way back in +// then SUCCEEDS on the wrong option — where the two share a type, and hundreds +// of coFloat/coBool/coInt options do, the bytes deserialize cleanly into the +// wrong key. Silently wrong print settings, no error. Those hooks are also the +// undo/redo wire format, where the process cannot change underneath them, so +// they stay as they are and the cache keys by name instead. +// +// Names are not repeated per preset. Each cache file carries one dictionary of +// the distinct opt_keys it uses, the type each was written as, and the distinct +// enum value names; an option on the wire is then a uint16 index into it plus +// its value. The dictionary is resolved to this build's option definitions once +// per file, after which reading an option is a vector index. +class CacheDictionary +{ +public: + CacheDictionary(); + + // Index reserved in the enum table for an int the writing build could not + // name — a nullable option's nil, or a definition carrying no + // enum_keys_map. The raw int32 follows it on the wire and is loaded + // verbatim, so those values survive too. + static constexpr uint16_t ENUM_UNNAMED = 0; + + // ---- writing ---- + + // Record every key and enum value `config` uses. Call for every config that + // will be written, before writing the dictionary. + void collect(const DynamicPrintConfig& config); + + uint16_t key_index(const t_config_option_key& key) const; + // ENUM_UNNAMED for an empty name or one that was never collected. + uint16_t enum_index(const std::string& name) const; + + // ---- reading ---- + + // The definition an index resolves to in THIS build, or nullptr where the + // key is unknown here or is now defined with a different type. A nullptr + // entry's value is still read — using type_at(idx), the type the writer + // recorded — and then dropped, which is what a JSON profile gets for an + // option this build no longer has. + const ConfigOptionDef* def_at(uint16_t idx) const { return m_defs[idx]; } + ConfigOptionType type_at(uint16_t idx) const { return ConfigOptionType(m_types[idx]); } + const std::string& enum_name_at(uint16_t idx) const { return m_enum_values[idx]; } + // m_defs, not m_keys: only load() sizes it, so this is false for every index + // on a dictionary that was collected rather than read. + bool valid_key_index(uint16_t idx) const { return size_t(idx) < m_defs.size(); } + bool valid_enum_index(uint16_t idx) const { return size_t(idx) < m_enum_values.size(); } + + // The layout these two agree on is covered by CACHE_VERSION (PresetCacheFormat.cpp); + // bump it when they change. + // Throws when either table outgrew the uint16 the wire format indexes it + // with. Both are bounded by the option count (912 at the time of writing), so + // that is a build-time failure in CI, not a runtime one. + void save(cereal::BinaryOutputArchive& ar) const; + // Throws on a dictionary that cannot be indexed as written. + void load(cereal::BinaryInputArchive& ar); + +private: + // Indices are uint16, so a table may hold at most this many entries. + static constexpr size_t MAX_ENTRIES = 0xFFFF; + + std::vector m_keys; + // ConfigOptionType, as written. Sixteen bits, not eight: coVectorType is + // 0x4000, so every vector type — coFloats, coEnums, coStrings — is above + // 255, and a byte would fold each one onto its scalar counterpart. + std::vector m_types; + std::vector m_enum_values; // [ENUM_UNNAMED] is always empty + + // Writing. + std::unordered_map m_key_index; + std::unordered_map m_enum_index; + // Reading, resolved once by load(). + std::vector m_defs; +}; + +// One config, keyed through `dict`. Options print_config_def does not know are +// not written: nothing could give them a type on the way back in. +void save_config(cereal::BinaryOutputArchive& ar, const DynamicPrintConfig& config, const CacheDictionary& dict); +// Throws only on a payload that cannot be indexed; an option this build cannot +// place is dropped, not fatal. +void load_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig& config, const CacheDictionary& dict); +// Consume one config without building it, for a reader that only wants what +// comes after. +void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict); + +// One preset as its JSON subfile states it: the config diff, the name of the +// preset it inherits, and the parse metadata — everything the parse phase of +// load_vendor_configs_from_json extracts and nothing it derives. Inheritance +// is resolved when the entry is installed, against whatever filament library +// is loaded then, so a cache carries no other vendor's values and no other +// vendor's update can make it stale. +// Written and read by visit_entry in PresetCacheFormat.cpp, which lists every +// field below in this order — once, for the save, the load and the name peek alike. +struct CachedPreset +{ + std::string name; + std::string sub_path; // path under the vendor's directory + DynamicPrintConfig config_src; // the preset's own diff, nothing inherited + std::string inherits; + std::string description; + std::string instantiation; // "true"/"false" as stated; anything else was already counted as a parse error + std::string setting_id; + std::string filament_id; + std::vector renamed_from; +}; + +// What one per-vendor cache file carries besides its stamps: the vendor profile +// map, the presets in source form, and how many errors their parse counted. +struct VendorCacheData +{ + VendorMap vendors; + std::vector process_entries; + std::vector filament_entries; + std::vector machine_entries; + uint64_t parse_errors = 0; +}; + +// A per-vendor preset cache file (.opc): a 20-byte header (magic, format +// version, body size, CRC) framing one cereal body — stamps (format version, +// vendor name, vendor profile version), the option dictionary, then the +// VendorCacheData. Everything about those bytes lives here; when a vendor is +// served from its cache, and how entries install into a bundle, is +// PresetBundle's business. +class VendorCacheFile +{ +public: + // Save one vendor (vendor_name at vendor_version). False when the file + // could not be written whole. + static bool save(const std::string& path, const std::string& vendor_name, + const std::string& vendor_version, const VendorCacheData& data); + + // Read a whole cache into `data`. False — with `data` in an unspecified + // state — unless the file is a cache this build wrote, its CRC holds, it + // names this vendor, it was built from a vendor profile at least as new as + // `expected_vendor_version`, and it carries its own vendor profile. An + // invalid expected version (a profile whose version + // cannot be judged) is never served from cache; Semver::inf() (no profile + // beside the cache at all) accepts whatever is cached. + static bool load(const std::string& path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, VendorCacheData& data); + + // Read the profile version a cache was stamped with, without deserializing + // its presets. Empty if the file is unreadable, not a cache this build + // understands, or not this vendor's. This is how an installed vendor's + // version is known when only its cache is installed. + static std::string peek_version(const std::string& path, const std::string& expected_vendor_name); + + // The profile version an installed cache can actually be served at, or an + // invalid Semver when the file is not a cache this build can read. Unlike + // peek_version this verifies the body's CRC, at the cost of reading the + // whole file: where the cache is the vendor's whole installation, "a file + // is there" is not enough to call it installed, and a vendor wrongly + // believed installed is never repaired. + static Semver usable_version(const std::string& path, const std::string& expected_vendor_name); + + // Whether a cache carries a preset of `type` under `preset_name`, without + // installing any of them. False when the file is not a cache this build can + // read. The three kinds are written in one stream, so reaching the machines + // means reading past the processes and filaments — their configs are consumed + // and dropped rather than built. This is how a build that ships caches instead + // of preset JSONs answers "which vendor carries this preset?". + static bool carries_preset(const std::string& path, const std::string& vendor_name, + Preset::Type type, const std::string& preset_name); +}; + +} // namespace Slic3r + +#endif // slic3r_PresetCacheFormat_hpp_ diff --git a/src/libslic3r/PrintConfig.hpp b/src/libslic3r/PrintConfig.hpp index 26a708b78d..255c8721b9 100644 --- a/src/libslic3r/PrintConfig.hpp +++ b/src/libslic3r/PrintConfig.hpp @@ -2488,7 +2488,8 @@ namespace cereal { archive(serialization_key_ordinal); assert(serialization_key_ordinal > 0); auto it = Slic3r::print_config_def.by_serialization_key_ordinal.find(serialization_key_ordinal); - assert(it != Slic3r::print_config_def.by_serialization_key_ordinal.end()); + if (it == Slic3r::print_config_def.by_serialization_key_ordinal.end()) + throw std::runtime_error("VendorCache: unknown serialization_key_ordinal " + std::to_string(serialization_key_ordinal) + " - cache is stale"); config.set_key_value(it->second->opt_key, it->second->load_option_from_archive(archive)); } } diff --git a/src/libslic3r/Semver.hpp b/src/libslic3r/Semver.hpp index 4d64b1c7db..d3683b4eb8 100644 --- a/src/libslic3r/Semver.hpp +++ b/src/libslic3r/Semver.hpp @@ -190,6 +190,19 @@ public: os << self.to_string(); return os; } + + // cereal: round-trip through the standard 3-part string (major.minor.patch). + // to_string() uses a BBS 4-part format that semver_parse() cannot read back. + template + std::string save_minimal(const Archive&) const { return to_string_sf(); } + template + void load_minimal(const Archive&, const std::string& s) { + auto v = Semver::parse(s); + if (! v) + throw std::runtime_error("Semver: cannot parse serialized version: " + s); + *this = std::move(*v); + } + private: semver_t ver; diff --git a/src/libslic3r/Utils.hpp b/src/libslic3r/Utils.hpp index 62b2eeb78e..55d9b716cf 100644 --- a/src/libslic3r/Utils.hpp +++ b/src/libslic3r/Utils.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include "libslic3r.h" +#include "Semver.hpp" //define CLI errors @@ -722,11 +724,42 @@ void copy_directory_recursively(const boost::filesystem::path& source, std::function filter = nullptr, bool merge_mode = false); -// Install vendor bundles from resources directory to data directory -// bundle_names: vector of vendor bundle names (without .json extension) -// resource_subdir: subdirectory under resources_dir() (default: "profiles") -// data_subdir: subdirectory under data_dir() (default: "system") -// Returns: true if all bundles installed successfully, false otherwise +// ---- Vendor installation on disk ------------------------------------------ +// How a vendor bundle is installed from resources into data_dir()/system: as +// its profile and preset JSONs or, in a build that ships preset caches, as its +// .opc preset cache alone. Loading what is installed is PresetBundle's business; +// the cache file format itself is VendorCacheFile's (PresetCacheFormat.hpp). + +// True if `vendor` is installed in data_dir()/system. A build that ships preset +// caches installs the cache alone, so it — not the profile — marks a vendor +// installed; a cache this build cannot read marks nothing. +bool is_vendor_installed(const std::string& vendor); + +// The version the installed vendor would be loaded at: its cache's stamp while +// that covers the profile beside it, the profile's own version once it does not. +// Invalid Semver if neither form is installed. +Semver installed_vendor_version(const std::string& vendor); + +// Remove every form `vendor` can be installed as from data_dir()/system: its +// profile, its preset cache, and its preset directory. +void remove_installed_vendor(const std::string& vendor); + +// The vendors `dir` holds, sorted: one is named by its profile or, in a build that +// ships preset caches instead of the raw profile JSONs, by its cache alone. +std::set vendor_names_in(const boost::filesystem::path& dir); + +// The version a build ships `vendor` at: whichever of its preset cache and its +// profile is newer, that being the one installing lays down. Invalid Semver if the +// build ships neither. +Semver resource_vendor_version(const std::string& vendor); + +// Install vendors from the resources directory into the data directory, each as +// its preset cache or as its profile and preset JSONs — whichever of the two the +// build ships at the newer version. Anything the previous install of that vendor +// left behind goes, so only the form just installed is there to be loaded. +// bundle_names: vendor names, without extension. +// Every bundle that can be installed is, whatever the others do. Returns false +// if any named bundle could not be installed. bool install_vendor_bundles_from_resources(const std::vector& bundle_names, const std::string& resource_subdir = "profiles", const std::string& data_subdir = "system"); diff --git a/src/libslic3r/utils.cpp b/src/libslic3r/utils.cpp index 5f429f076a..58ec8318a6 100644 --- a/src/libslic3r/utils.cpp +++ b/src/libslic3r/utils.cpp @@ -17,6 +17,10 @@ #include "Platform.hpp" #include "Time.hpp" #include "libslic3r.h" +// For the vendor-installation helpers: the vendor profile version +// (get_version_from_json) and the preset cache stamp (VendorCacheFile). +#include "Preset.hpp" +#include "PresetCacheFormat.hpp" #ifdef __APPLE__ #include "MacUtils.hpp" @@ -1724,6 +1728,85 @@ void copy_directory_recursively(const boost::filesystem::path& source, return; } +// ---- Vendor installation on disk ------------------------------------------ + +// Whether a cache stamped `cache_ver` still speaks for a vendor whose profile on +// disk claims `profile_ver`: it does unless the profile has moved ahead of it. A +// profile that is missing or carries no judgeable version cannot be ahead of +// anything. The one rule behind both "which form gets installed" and "which form +// is installed"; they must not drift apart. Deliberately NOT the serve rule +// (VendorCacheFile::load), which refuses an unjudgeable profile instead. +static bool cache_covers(const Semver& cache_ver, const Semver& profile_ver) +{ + return cache_ver.valid() && (! profile_ver.valid() || cache_ver >= profile_ver); +} + +bool is_vendor_installed(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR; + // A cache is the whole of a cache-only installation, so a file this build + // cannot serve the vendor from is not an installation. Left counted as one, + // the updater would never lay a working copy down. + return boost::filesystem::exists(dir / (vendor + ".json")) + || VendorCacheFile::usable_version((dir / (vendor + ".opc")).string(), vendor).valid(); +} + +Semver installed_vendor_version(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR; + const boost::filesystem::path json = dir / (vendor + ".json"); + // Guarded: get_version_from_json logs an error and throws-and-catches its way + // to an invalid version on a file that is not there, and a cache-only vendor + // never has one. + const Semver from_json = boost::filesystem::exists(json) ? get_version_from_json(json.string()) : Semver(); + const Semver from_cache = VendorCacheFile::usable_version((dir / (vendor + ".opc")).string(), vendor); + // Whichever form a load would serve. + return cache_covers(from_cache, from_json) ? from_cache : from_json; +} + +void remove_installed_vendor(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR; + boost::filesystem::remove(dir / (vendor + ".json")); + boost::filesystem::remove(dir / (vendor + ".opc")); + if (boost::filesystem::exists(dir / vendor)) + boost::filesystem::remove_all(dir / vendor); +} + +std::set vendor_names_in(const boost::filesystem::path& dir) +{ + std::set names; + for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { + const auto& path = dir_entry.path(); + if (Slic3r::is_json_file(path.string()) || path.extension() == ".opc") + names.insert(path.stem().string()); + } + return names; +} + +// A vendor's preset cache is the whole of its installation: it carries the presets, +// the vendor profile and the version they were built at, so where one ships nothing +// else needs copying. Unless the profile beside it claims a newer version — a cache +// generated before that profile was bumped is out of date, and a cache that cannot +// be read is no installation at all — and the vendor is installed the way it was +// before caches existed, as its profile and the preset JSONs it points at. Returns +// the version the cache is stamped with, invalid when it is not the form to install. +static Semver installable_cache_version(const boost::filesystem::path& dir, const std::string& vendor) +{ + const auto cache_ver = Semver::parse(VendorCacheFile::peek_version((dir / (vendor + ".opc")).string(), vendor)); + if (! cache_ver) + return Semver::invalid(); + const Semver profile_ver = get_version_from_json((dir / (vendor + ".json")).string()); + return cache_covers(*cache_ver, profile_ver) ? *cache_ver : Semver::invalid(); +} + +Semver resource_vendor_version(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(resources_dir()) / "profiles"; + const Semver ver = installable_cache_version(dir, vendor); + return ver.valid() ? ver : get_version_from_json((dir / (vendor + ".json")).string()); +} + bool install_vendor_bundles_from_resources( const std::vector& bundle_names, const std::string& resource_subdir, @@ -1736,37 +1819,82 @@ bool install_vendor_bundles_from_resources( BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; + // One vendor that cannot be installed is one vendor missing, not a reason to + // leave the rest uninstalled. The caller is told, and every bundle that can + // be laid down is. + bool all_installed = true; + for (const auto &bundle : bundle_names) { try { + if (bundle.empty()) { + BOOST_LOG_TRIVIAL(warning) << "Refusing to install a bundle with no name"; + all_installed = false; + continue; + } + // Install the JSON file auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); + auto cache_in_rsrc = (rsrc_path / bundle).replace_extension(".opc"); + auto cache_in_vendors = (vendor_path / bundle).replace_extension(".opc"); - if (!fs::exists(path_in_rsrc)) { + // Either form of the vendor will do: a build may ship it as a cache alone. + if (!fs::exists(path_in_rsrc) && !fs::exists(cache_in_rsrc)) { BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; - return false; + all_installed = false; + continue; } // Create target directory if needed if (!fs::exists(vendor_path)) fs::create_directories(vendor_path); - // Copy JSON file std::string error_message; - CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); - if (cfr != CopyFileResult::SUCCESS) { - BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; - return false; + bool installed_cache = false; + if (installable_cache_version(rsrc_path, bundle).valid()) { + installed_cache = copy_file(cache_in_rsrc.string(), cache_in_vendors.string(), error_message, false) == CopyFileResult::SUCCESS; + if (! installed_cache) { + BOOST_LOG_TRIVIAL(warning) << "Failed to copy " << bundle << ".opc: " << error_message; + } else if (! VendorCacheFile::usable_version(cache_in_vendors.string(), bundle).valid()) { + // The copy is what will be loaded, so it — not the kilobyte + // peek that chose this form — decides whether the profile + // beside it can go. + BOOST_LOG_TRIVIAL(warning) << "Installed cache for " << bundle << " cannot be read; installing its profile instead"; + boost::system::error_code ec; + fs::remove(cache_in_vendors, ec); + installed_cache = false; + } + } + + if (! installed_cache) { + CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); + if (cfr != CopyFileResult::SUCCESS) { + BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; + all_installed = false; + continue; + } + // Only now: an earlier install's cache would shadow this profile, + // but removing it before the profile lands would leave neither. + boost::system::error_code ec; + fs::remove(cache_in_vendors, ec); + } else { + // Left in place, an earlier install's profile would shadow the cache. + boost::system::error_code ec; + fs::remove(path_in_vendors, ec); + if (ec) + BOOST_LOG_TRIVIAL(warning) << "Could not remove the superseded profile " << path_in_vendors.string() << ": " << ec.message(); } // Copy the vendor directory (if it exists) auto dir_in_rsrc = rsrc_path / bundle; auto dir_in_vendors = vendor_path / bundle; - if (fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { - // Remove existing directory - if (fs::exists(dir_in_vendors)) - fs::remove_all(dir_in_vendors); + // Whatever is installed came from an earlier version of this vendor and + // would be parsed in place of the one being installed now. + if (fs::exists(dir_in_vendors)) + fs::remove_all(dir_in_vendors); + + if (! installed_cache && fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { fs::create_directories(dir_in_vendors); // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) @@ -1787,11 +1915,11 @@ bool install_vendor_bundles_from_resources( } catch (const std::exception& e) { BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); - return false; + all_installed = false; } } - return true; + return all_installed; } void save_string_file(const boost::filesystem::path& p, const std::string& str) diff --git a/src/slic3r/Config/Snapshot.cpp b/src/slic3r/Config/Snapshot.cpp index 4b071994fc..a7135eac6f 100644 --- a/src/slic3r/Config/Snapshot.cpp +++ b/src/slic3r/Config/Snapshot.cpp @@ -432,14 +432,9 @@ const Snapshot& SnapshotDB::take_snapshot(const AppConfig &app_config, Snapshot: cfg.models_variants_installed.erase(it ++); else ++ it; - // Read the active config bundle, parse the config version. - PresetBundle bundle; - //BBS: change directoties by design - //bundle.load_configbundle((data_dir / PRESET_SYSTEM_DIR / (cfg.name + ".ini")).string(), PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - bundle.load_vendor_configs_from_json((data_dir/PRESET_SYSTEM_DIR).string(), cfg.name, PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - for (const auto &vp : bundle.vendors) - if (vp.second.id == cfg.name) - cfg.version.config_version = vp.second.config_version; + // Orca: the version the vendor is installed at, read from its profile or — + // where the cache is the whole installation — from the cache's own stamp. + cfg.version.config_version = installed_vendor_version(cfg.name); snapshot.vendor_configs.emplace_back(std::move(cfg)); } diff --git a/src/slic3r/GUI/ConfigWizard.cpp b/src/slic3r/GUI/ConfigWizard.cpp index 0bbbc15f87..dba8699105 100644 --- a/src/slic3r/GUI/ConfigWizard.cpp +++ b/src/slic3r/GUI/ConfigWizard.cpp @@ -66,41 +66,41 @@ using Config::SnapshotDB; // Configuration data structures extensions needed for the wizard //BBS: set BBL as default -bool Bundle::load(fs::path source_path, bool ais_in_resources, bool ais_bbl_bundle) +bool Bundle::load(fs::path dir, const std::string &vendor_name, bool ais_in_resources, bool ais_bbl_bundle) { this->preset_bundle = std::make_unique(); this->is_in_resources = ais_in_resources; this->is_bbl_bundle = ais_bbl_bundle; - std::string path_string = source_path.string(); - std::string parent_path = source_path.parent_path().string(); //BBS: add json logic for vendor bundles - std::string vendor_name = source_path.filename().string(); - if (Slic3r::is_json_file(path_string)) { - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - } - else + // Orca: served from the vendor's preset cache where one covers it — which is + // how a shipped build carries its vendors — and parsed from the JSONs otherwise. + // A vendor that can be neither read nor parsed — a cache the build cannot use + // with the preset JSONs behind it pruned, say — is one the wizard cannot offer. + // Every other vendor still can be, so it is left out rather than thrown over. + size_t presets_loaded = 0; + try { + auto [config_substitutions, loaded] = preset_bundle->load_vendor_configs_from_json( + dir.string(), vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + UNUSED(config_substitutions); + // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. + assert(config_substitutions.empty()); + presets_loaded = loaded; + } catch (const std::exception &e) { + BOOST_LOG_TRIVIAL(fatal) << boost::format("Vendor bundle: `%1%`: cannot be loaded, leaving it out: %2%") % vendor_name % e.what(); return false; - - // Throw when parsing invalid configuration. Only valid configuration is supposed to be provided over the air. - //BBS: add json logic for vendor bundles - auto [config_substitutions, presets_loaded] = preset_bundle->load_vendor_configs_from_json( - parent_path, vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); - UNUSED(config_substitutions); - // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. - assert(config_substitutions.empty()); + } auto first_vendor = preset_bundle->vendors.begin(); if (first_vendor == preset_bundle->vendors.end()) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % vendor_name; return false; } if (presets_loaded == 0) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % vendor_name; return false; - } + } - BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % path_string % presets_loaded; + BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % vendor_name % presets_loaded; this->vendor_profile = &first_vendor->second; return true; } @@ -125,15 +125,10 @@ BundleMap BundleMap::load() //Orca: add custom as default //Orca: add json logic for vendor bundle - auto orca_bundle_path = (vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - auto orca_bundle_rsrc = false; - if (!boost::filesystem::exists(orca_bundle_path)) { - orca_bundle_path = (rsrc_vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - orca_bundle_rsrc = true; - } { + const bool from_rsrc = ! is_vendor_installed(PresetBundle::ORCA_DEFAULT_BUNDLE); Bundle bbl_bundle; - if (bbl_bundle.load(std::move(orca_bundle_path), orca_bundle_rsrc, true)) + if (bbl_bundle.load(from_rsrc ? rsrc_vendor_dir : vendor_dir, PresetBundle::ORCA_DEFAULT_BUNDLE, from_rsrc, true)) res.emplace(PresetBundle::ORCA_DEFAULT_BUNDLE, std::move(bbl_bundle)); } @@ -141,18 +136,13 @@ BundleMap BundleMap::load() // and then additionally from resources/profiles. bool is_in_resources = false; for (auto dir : { &vendor_dir, &rsrc_vendor_dir }) { - for (const auto &dir_entry : boost::filesystem::directory_iterator(*dir)) { - //BBS: add json logic for vendor bundle - if (Slic3r::is_json_file(dir_entry.path().string())) { - std::string id = dir_entry.path().stem().string(); // stem() = filename() without the trailing ".json" part + for (const std::string &id : vendor_names_in(*dir)) { + // Don't load this bundle if we've already loaded it. + if (res.find(id) != res.end()) { continue; } - // Don't load this bundle if we've already loaded it. - if (res.find(id) != res.end()) { continue; } - - Bundle bundle; - if (bundle.load(dir_entry.path(), is_in_resources)) - res.emplace(std::move(id), std::move(bundle)); - } + Bundle bundle; + if (bundle.load(*dir, id, is_in_resources)) + res.emplace(id, std::move(bundle)); } is_in_resources = true; diff --git a/src/slic3r/GUI/ConfigWizard_private.hpp b/src/slic3r/GUI/ConfigWizard_private.hpp index 364d378b42..7b9674b216 100644 --- a/src/slic3r/GUI/ConfigWizard_private.hpp +++ b/src/slic3r/GUI/ConfigWizard_private.hpp @@ -71,9 +71,11 @@ struct Bundle Bundle() = default; Bundle(Bundle&& other); + // Load the vendor `vendor_name` as it is installed in `dir`, from its preset + // cache or its profile JSONs, whichever is usable. // Returns false if not loaded. Reason for that is logged as boost::log error. //BBS: set BBL as default - bool load(fs::path source_path, bool is_in_resources, bool is_bbl_bundle = false); + bool load(fs::path dir, const std::string &vendor_name, bool is_in_resources, bool is_bbl_bundle = false); const std::string& vendor_id() const { return vendor_profile->id; } }; diff --git a/src/slic3r/GUI/CreatePresetsDialog.cpp b/src/slic3r/GUI/CreatePresetsDialog.cpp index 1bd80d5f00..33f49c2a38 100644 --- a/src/slic3r/GUI/CreatePresetsDialog.cpp +++ b/src/slic3r/GUI/CreatePresetsDialog.cpp @@ -2201,25 +2201,14 @@ bool CreatePrinterPresetDialog::load_system_and_user_presets_with_curr_model(Pre } else { selected_vendor_id = m_printer_preset_vendor_selected.id; - if (boost::filesystem::exists(boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(); - } else if (boost::filesystem::exists(boost::filesystem::path(Slic3r::resources_dir()) / "profiles" / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::resources_dir()) / "profiles").string(); - } - - if (preset_path.empty()) { - BOOST_LOG_TRIVIAL(info) << "Preset path was not found"; - MessageDialog dlg(this, _L("Preset path was not found; please reselect vendor."), wxString(SLIC3R_APP_FULL_NAME) + " - " + _L("Info"), - wxYES_NO | wxYES_DEFAULT | wxCENTRE); - dlg.ShowModal(); - return false; - } - try { // Pass the app's preset bundle (which already holds OrcaFilamentLibrary) as the base // bundle so vendor filaments that inherit OFL bases resolve via the existing // cross-vendor inheritance path. - temp_preset_bundle.load_vendor_configs_from_json(preset_path, selected_vendor_id, + // Orca: served from the vendor's preset cache where one covers it — a shipped + // build carries that instead of the raw preset JSONs — and parsed otherwise. + temp_preset_bundle.load_vendor_configs_from_json((boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(), + selected_vendor_id, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent, wxGetApp().preset_bundle); diff --git a/src/slic3r/GUI/GUI_App.cpp b/src/slic3r/GUI/GUI_App.cpp index 3a85dc1928..6028ada640 100644 --- a/src/slic3r/GUI/GUI_App.cpp +++ b/src/slic3r/GUI/GUI_App.cpp @@ -6817,6 +6817,12 @@ void GUI_App::add_pending_vendor_preset(const std::pair>(); diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index 0d2f6c724b..6b58fffead 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -1,7 +1,9 @@ #include "WebGuideDialog.hpp" #include "ConfigWizard.hpp" +#include #include +#include #include #include #include @@ -9,7 +11,9 @@ #include "I18N.hpp" #include "libslic3r/AppConfig.hpp" #include "libslic3r/Config.hpp" +#include "libslic3r/Preset.hpp" #include "libslic3r/PresetBundle.hpp" +#include "libslic3r/PresetCacheFormat.hpp" #include "slic3r/GUI/wxExtensions.hpp" #include "slic3r/GUI/GUI_App.hpp" #include "libslic3r_version.h" @@ -41,8 +45,6 @@ using namespace nlohmann; namespace Slic3r { namespace GUI { -json m_ProfileJson; - static wxString update_custom_filaments() { json m_Res = json::object(); @@ -190,12 +192,10 @@ GuideFrame::GuideFrame(GUI_App *pGUI, long style) GuideFrame::~GuideFrame() { - m_destroy = true; - if (m_load_task && m_load_task->joinable()) { + *m_cancel_token = true; // stop the loading thread and any queued CallAfter lambdas before join + if (m_load_task && m_load_task->joinable()) m_load_task->join(); - delete m_load_task; - m_load_task = nullptr; - } + m_load_task.reset(); if (m_browser) { delete m_browser; m_browser = nullptr; @@ -301,15 +301,71 @@ void GuideFrame::OnNavigationRequest(wxWebViewEvent &evt) /** * Callback invoked when a navigation request was accepted */ +// The empty shape every profile-loading path starts from or falls back to. +void GuideFrame::reset_profile_json() +{ + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); +} + +void GuideFrame::init_guide_paths() +{ + m_ProfileJson = json::parse("{}"); + reset_profile_json(); + + vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); + rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); + orca_bundle_rsrc = true; + + if (boost::filesystem::exists(vendor_dir)) { + for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { + if (!boost::filesystem::is_directory(entry) && + boost::iequals(entry.path().extension().string(), ".json") && + !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { + orca_bundle_rsrc = false; + break; + } + } + } + + auto lib_json = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + m_OrcaFilaLibPath = boost::filesystem::exists(vendor_dir / lib_json) + ? (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string() + : (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); +} + +void GuideFrame::on_profile_loaded() +{ + // Must be called on the main thread. + SaveProfileData(); + const std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents:\n" << strAll; + json res; + res["command"] = "userguide_profile_load_finish"; + res["sequence_id"] = "10001"; + RunScript(wxString::Format("HandleStudio(%s)", res.dump(-1, ' ', true))); +} + void GuideFrame::OnNavigationComplete(wxWebViewEvent &evt) { //wxLogMessage("%s", "Navigation complete; url='" + evt.GetURL() + "'"); if (!bFirstComplete) { - m_load_task = new boost::thread(boost::bind(&GuideFrame::LoadProfileData, this)); - // boost::thread LoadProfileThread(boost::bind(&GuideFrame::LoadProfileData, this)); - //LoadProfileThread.detach(); - bFirstComplete = true; + try { + init_guide_paths(); + if (BuildProfileDataFromPresetBundle()) { + if (!*m_cancel_token) + on_profile_loaded(); + } else { + // Presets not yet in memory — delegate to background thread. + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", init error: " << e.what(); + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } } m_browser->Show(); @@ -762,11 +818,9 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle bool check_unsaved_preset_changes = false; std::vector install_bundles; std::vector remove_bundles; - const auto vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); for (const auto &it : enabled_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir/(it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -777,8 +831,7 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle if (it.second.size() > 0) { if (enabled_vendors.find(it.first) != enabled_vendors.end()) continue; - auto vendor_file = vendor_dir/(it.first + ".json"); - if (fs::exists(vendor_file)) { + if (is_vendor_installed(it.first)) { remove_bundles.emplace_back(it.first); } } @@ -1127,99 +1180,324 @@ int GuideFrame::GetFilamentInfo( std::string VendorDirectory, json & pFilaList, return status; } -int GuideFrame::LoadProfileData() +bool GuideFrame::BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors) { try { - m_ProfileJson = json::parse("{}"); - m_ProfileJson["model"] = json::array(); - m_ProfileJson["machine"] = json::object(); - m_ProfileJson["filament"] = json::object(); - m_ProfileJson["process"] = json::array(); + // Models from vendor profiles + for (const auto& [vendor_id, vp] : bundle.vendors) { + for (const auto& model : vp.models) { + std::string nozzle_str; + for (const auto& v : model.variants) { + if (!nozzle_str.empty()) nozzle_str += ";"; + nozzle_str += v.name; + } + const std::string materials_str = boost::algorithm::join(model.default_materials, ";"); + boost::filesystem::path cover_path = + (boost::filesystem::path(resources_dir()) / "profiles" / vp.id / (model.id + "_cover.png")) + .make_preferred(); + if (!boost::filesystem::exists(cover_path)) + cover_path = + (boost::filesystem::path(resources_dir()) / "web/image/printer" / (model.id + "_cover.png")) + .make_preferred(); - vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); - - // Orca: add custom as default - // Orca: add json logic for vendor bundle - orca_bundle_rsrc = true; - - // search if there exists a .json file in vendor_dir folder, if exists, set orca_bundle_rsrc to false - for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { - if (!boost::filesystem::is_directory(entry) && boost::iequals(entry.path().extension().string(), ".json") && !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { - orca_bundle_rsrc = false; - break; + json entry; + entry["model"] = model.id; + entry["name"] = model.name; + entry["vendor"] = vp.id; + entry["nozzle_diameter"] = nozzle_str; + entry["materials"] = materials_str; + entry["cover"] = cover_path.string(); + entry["nozzle_selected"] = ""; + entry["sub_path"] = ""; + m_ProfileJson["model"].push_back(entry); } } - // load the default filament library first - std::set loaded_vendors; - auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); - if (boost::filesystem::exists(vendor_dir / filament_library_name)) { - m_OrcaFilaLibPath = (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); - } else { - m_OrcaFilaLibPath = (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); - } - loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); + // Machine map: preset name -> {model, nozzle variant} + for (const Preset& p : bundle.printers()) { + if (!p.is_system || !p.vendor) continue; + const auto* printer_model = p.config.option("printer_model"); + const auto* printer_variant = p.config.option("printer_variant"); + if (!printer_model || printer_model->value.empty() || !printer_variant) continue; - //load custom bundle from user data path - boost::filesystem::directory_iterator endIter; - for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if(strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } - if (m_destroy) - return 0; + json mach; + mach["model"] = printer_model->value; + mach["nozzle"] = printer_variant->value; + m_ProfileJson["machine"][p.name] = mach; } - boost::filesystem::directory_iterator others_endIter; - for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; + // Filament map from system filament presets (vendor/type already resolved in config) + const json& machines = m_ProfileJson["machine"]; + for (const Preset& p : bundle.filaments()) { + if (!p.is_system || !p.vendor) continue; + const auto* fila_vendor = p.config.option("filament_vendor"); + const auto* fila_type = p.config.option("filament_type"); + const auto* compat_printers = p.config.option("compatible_printers"); - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); + std::string vendor = (fila_vendor && !fila_vendor->values.empty()) ? fila_vendor->values[0] : ""; + std::string type = (fila_type && !fila_type->values.empty()) ? fila_type->values[0] : ""; + + std::string model_list; + if (compat_printers) { + for (const std::string& pname : compat_printers->values) { + auto it = machines.find(pname); + if (it != machines.end()) { + const std::string m = (*it)["model"]; + const std::string n = (*it)["nozzle"]; + model_list += "[" + m + "++" + n + "]"; + } + } } - if (m_destroy) - return 0; + + json ff; + ff["name"] = p.name; + ff["sub_path"] = p.file; + ff["vendor"] = vendor; + ff["type"] = type; + ff["models"] = model_list; + ff["selected"] = 0; + m_ProfileJson["filament"][p.name] = ff; } - wxGetApp().CallAfter([this] { - if (!m_destroy) { - //sync to appconfig first to populate current selections - SaveProfileData(); + // Process list from visible system print presets + for (const Preset& p : bundle.prints()) { + if (!p.is_system || !p.vendor || !p.is_visible) continue; + json entry; + entry["name"] = p.name; + entry["sub_path"] = p.file; + m_ProfileJson["process"].push_back(entry); + } - //sync to web after selections are populated - std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); + if (require_all_resource_vendors) { + // If rsrc_vendor_dir has vendors (profile JSONs, or the preset caches a + // packaged build ships instead) not covered by the current bundle, the + // bundle is incomplete (e.g. dev env where data_dir/system only has + // OrcaFilamentLibrary+Custom). Fall back so the slow path reads both dirs. + try { + for (const std::string& name : vendor_names_in(rsrc_vendor_dir)) { + if (bundle.vendors.find(name) == bundle.vendors.end()) { + BOOST_LOG_TRIVIAL(info) << "GuideFrame: vendor '" << name + << "' in resources but not in preset_bundle — falling back to JSON loading"; + reset_profile_json(); + return false; + } + } + } catch (const std::exception&) {} + } - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents: " << std::endl << strAll; - json m_Res = json::object(); - m_Res["command"] = "userguide_profile_load_finish"; - m_Res["sequence_id"] = "10001"; - wxString strJS = wxString::Format("HandleStudio(%s)", m_Res.dump(-1, ' ', true)); + BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data (" + << m_ProfileJson["model"].size() << " models, " + << m_ProfileJson["machine"].size() << " machines, " + << m_ProfileJson["filament"].size() << " filaments)"; + return !m_ProfileJson["machine"].empty(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileJson failed: " << e.what() + << " — falling back to JSON loading"; + reset_profile_json(); + return false; + } +} - RunScript(strJS); +bool GuideFrame::BuildProfileDataFromPresetBundle() +{ + PresetBundle* pb = wxGetApp().preset_bundle; + if (!pb || pb->vendors.empty()) + return false; + return BuildProfileJson(*pb, /*require_all_resource_vendors=*/true); +} + +bool GuideFrame::BuildProfileDataFromVendors() +{ + try { + // Same vendor set and precedence as the JSON scan in LoadProfileData: a + // vendor in the user's system dir shadows the bundled one of that name. + // vendor_names_in names a vendor by its profile or, where a build ships + // preset caches instead, by its cache alone. + std::map vendor_sources; + for (const boost::filesystem::path& dir : { vendor_dir, rsrc_vendor_dir }) { + boost::system::error_code ec; + if (boost::filesystem::exists(dir, ec)) + for (const std::string& name : vendor_names_in(dir)) + vendor_sources.emplace(name, dir); // first dir wins + } + + // The load order: the filament library first, because the others' + // filaments inherit from it, then every versioned vendor — each loaded + // from the directory it was found in, so a vendor that is not installed + // is served from the shipped profiles. Each is stamped by name and + // version alone: a profile change requires a version bump, so those two + // determine content wherever the vendor's copy sits. + struct VendorSource { std::string name; boost::filesystem::path dir; std::string version; }; + std::vector ordered; + auto add_vendor = [&ordered](const std::string& name, const boost::filesystem::path& dir) { + // The version a load from `dir` would serve: the profile's where one + // exists (a cache is only served while it covers the profile beside + // it), the cache's own stamp where the cache is the whole vendor. + // A profile without a version (blacklist.json) carries no presets + // and is passed over. + const boost::filesystem::path profile = dir / (name + ".json"); + if (boost::filesystem::exists(profile)) { + const Semver v = get_version_from_json(profile.string()); + if (v.valid()) + ordered.push_back({name, dir, v.to_string()}); + } else { + ordered.push_back({name, dir, + VendorCacheFile::peek_version((dir / (name + ".opc")).string(), name)}); } + }; + const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); + if (auto it = vendor_sources.find(filament_library); it != vendor_sources.end()) + add_vendor(filament_library, it->second); + for (const auto& [name, dir] : vendor_sources) + if (name != filament_library) + add_vendor(name, dir); + if (ordered.empty()) + return false; + json stamps = json::array(); + for (const VendorSource& v : ordered) + stamps.push_back({v.name, v.version}); + + // What this function derives is a pure function of that stamped set, so + // the derived JSON is cached whole: a fresh cache makes an open one + // file read, with no bundle built and no preset installed. Stale or + // absent, the bundle is rebuilt below and the result written back. + const boost::filesystem::path cache_file = + boost::filesystem::path(Slic3r::data_dir()) / "cache" / "wizard_profile_data.json"; + try { + // Slurped whole and parsed from the buffer — nlohmann's fastest + // input path; a stream adapter costs real time on a multi-MB file. + boost::nowide::ifstream ifs(cache_file.string(), std::ios::binary); + if (ifs.is_open()) { + const std::string text{std::istreambuf_iterator(ifs), std::istreambuf_iterator()}; + json cached = json::parse(text); + if (cached.value("format", 0) == 1 && cached["vendors"] == stamps && + ! cached["profile"]["machine"].empty()) { + for (const char* key : { "model", "machine", "filament", "process" }) + m_ProfileJson[key] = std::move(cached["profile"][key]); + BOOST_LOG_TRIVIAL(info) << "GuideFrame: profile data served from " << cache_file; + return true; + } + } + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(info) << "GuideFrame: rejecting cached profile data: " << e.what(); + } + + // Each vendor comes from its preset cache where one covers it, which is + // what makes this worth doing instead of the scan below; loading into a + // bundle per vendor keeps the install order the startup path has. + PresetBundle bundle; + auto load_vendor = [](PresetBundle& into, const std::string& vendor, + const boost::filesystem::path& dir, const PresetBundle* base) { + into.load_vendor_configs_from_json(dir.string(), vendor, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, base); + }; + for (const VendorSource& v : ordered) { + if (*m_cancel_token) + return false; // as in the scan below: a vendor without a cache is parsed, and that takes time + if (v.name == filament_library) { + load_vendor(bundle, v.name, v.dir, nullptr); + } else { + PresetBundle tmp; + load_vendor(tmp, v.name, v.dir, &bundle); + bundle.merge_presets(std::move(tmp)); + } + } + if (bundle.vendors.empty()) + return false; + if (! BuildProfileJson(bundle, /*require_all_resource_vendors=*/false)) + return false; + + // Written through a temp file and moved into place, as the preset caches + // are: half a cache must never be readable, and the PID suffix keeps two + // instances from interleaving on one temp file. + const std::string tmp_path = cache_file.string() + "." + std::to_string(get_current_pid()) + ".tmp"; + try { + json out; + out["format"] = 1; + out["vendors"] = std::move(stamps); + json& profile = out["profile"]; + for (const char* key : { "model", "machine", "filament", "process" }) + profile[key] = m_ProfileJson[key]; + boost::filesystem::create_directories(cache_file.parent_path()); + { + boost::nowide::ofstream ofs(tmp_path, std::ios::binary | std::ios::trunc); + ofs << out.dump(-1, ' ', false, json::error_handler_t::ignore); + ofs.close(); + if (! ofs.good()) + throw std::runtime_error("write failed"); + } + if (const std::error_code ec = rename_file(tmp_path, cache_file.string())) + throw std::runtime_error(ec.message()); + } catch (const std::exception& e) { + boost::system::error_code rm; + boost::filesystem::remove(tmp_path, rm); + BOOST_LOG_TRIVIAL(warning) << "GuideFrame: could not write the profile data cache: " << e.what(); + } + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << " failed: " << e.what(); + reset_profile_json(); + return false; + } +} + +int GuideFrame::LoadProfileData() +{ + // Background thread: the fast path in OnNavigationComplete failed (presets not yet loaded). + // Loading order (fastest to slowest): + // 1. Load every vendor, from its preset cache wherever one covers it + // 2. Read all vendor JSONs by hand + try { + if (!BuildProfileDataFromVendors()) { + // Last resort — read all vendor JSONs + std::set loaded_vendors; + auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + if (boost::filesystem::exists(vendor_dir / filament_library_name)) + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); + else + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); + loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); + + boost::filesystem::directory_iterator endIter; + for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (*m_cancel_token) return 0; + } + + boost::filesystem::directory_iterator others_endIter; + for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (*m_cancel_token) return 0; + } + } + + // Capture the cancel token by value (shared_ptr) so the lambda doesn't + // touch `this` if GuideFrame is destroyed before the event fires. + auto tok = m_cancel_token; + wxGetApp().CallAfter([this, tok] { + if (!*tok) + on_profile_loaded(); }); - } catch (std::exception& e) { - // wxLogMessage("GUIDE: load_profile_error %s ", e.what()); - // wxMessageBox(e.what(), "", MB_OK); - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what() << std::endl; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what(); } filament_info_cache.clear(); diff --git a/src/slic3r/GUI/WebGuideDialog.hpp b/src/slic3r/GUI/WebGuideDialog.hpp index fcdb0841db..b9592d03fe 100644 --- a/src/slic3r/GUI/WebGuideDialog.hpp +++ b/src/slic3r/GUI/WebGuideDialog.hpp @@ -30,10 +30,14 @@ #include "libslic3r/PresetBundle.hpp" #include "slic3r/Utils/PresetUpdater.hpp" +#include +#include #include #include +#include + namespace Slic3r { namespace GUI { class GuideFrame : public DPIDialog @@ -78,6 +82,12 @@ public: int LoadProfileData(); int SaveProfileData(); int LoadProfileFamily(std::string strVendor, std::string strFilePath); + void init_guide_paths(); + void on_profile_loaded(); + bool BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors); + bool BuildProfileDataFromPresetBundle(); + bool BuildProfileDataFromVendors(); + void reset_profile_json(); int SaveProfile(); int GetFilamentInfo( std::string VendorDirectory,json & pFilaList, std::string filepath, std::string &sVendor, std::string &sType); @@ -112,8 +122,11 @@ private: //First Load bool bFirstComplete{false}; - bool m_destroy{false}; - boost::thread* m_load_task{ nullptr }; + // Set once in the destructor. Read through `this` by the loading thread + // (joined before `this` dies) and captured as the shared_ptr by CallAfter + // lambdas so they don't touch `this` after the object is freed. + std::shared_ptr> m_cancel_token{std::make_shared>(false)}; + std::unique_ptr m_load_task; // User Config bool PrivacyUse; @@ -123,6 +136,7 @@ private: bool InstallNetplugin; bool network_plugin_ready {false}; + json m_ProfileJson; json m_OrcaFilaList; std::string m_OrcaFilaLibPath; diff --git a/src/slic3r/Utils/PresetUpdater.cpp b/src/slic3r/Utils/PresetUpdater.cpp index 18a9db4e26..06808e253d 100644 --- a/src/slic3r/Utils/PresetUpdater.cpp +++ b/src/slic3r/Utils/PresetUpdater.cpp @@ -1044,46 +1044,42 @@ void PresetUpdater::priv::check_installed_vendor_profiles() const std::set bundles; // Orca: always install filament library bundles.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - for (auto &dir_entry : boost::filesystem::directory_iterator(rsrc_path)) { - const auto &path = dir_entry.path(); - std::string file_path = path.string(); - if (is_json_file(file_path)) { - const auto path_in_vendor = vendor_path / path.filename(); - std::string vendor_name = path.filename().string(); - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - if (bundles.find(vendor_name) != bundles.end())continue; + // A vendor is named by its profile or, where the build ships preset caches + // instead of the raw profile JSONs, by its cache alone. + for (const std::string &vendor_name : vendor_names_in(rsrc_path)) { + if (bundles.find(vendor_name) != bundles.end())continue; - const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE - || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); - if (enabled_config_update) { - if ( fs::exists(path_in_vendor)) { - if (is_vendor_enabled) { - Semver resource_ver = get_version_from_json(file_path); - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE + || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); + if (enabled_config_update) { + if (is_vendor_installed(vendor_name)) { + if (is_vendor_enabled) { + // Orca: whichever form of the vendor resources ships at the newer + // version is the one installing lays down, and the one to judge + // what is installed against. + Semver resource_ver = resource_vendor_version(vendor_name); + // Orca: a vendor installed as a preset cache has no profile + // beside it; the version it was installed at is in the cache. + Semver vendor_ver = installed_vendor_version(vendor_name); - if (vendor_ver < resource_ver) { - BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor " << vendor_name << " newer version " - << resource_ver.to_string() << " from resource, old version " << vendor_ver.to_string(); - bundles.insert(vendor_name); - } - } - else { - //need to be removed because not installed - fs::remove(path_in_vendor); - const auto path_of_vendor = vendor_path / vendor_name; - if (fs::exists(path_of_vendor)) - fs::remove_all(path_of_vendor); + if (vendor_ver < resource_ver) { + BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor " << vendor_name << " newer version " + << resource_ver.to_string() << " from resource, old version " << vendor_ver.to_string(); + bundles.insert(vendor_name); } } - else if (is_vendor_enabled) { - bundles.insert(vendor_name); + else { + //need to be removed because not installed + remove_installed_vendor(vendor_name); } } else if (is_vendor_enabled) { bundles.insert(vendor_name); } } + else if (is_vendor_enabled) { + bundles.insert(vendor_name); + } } if (bundles.size() > 0) { @@ -1163,11 +1159,12 @@ Updates PresetUpdater::priv::get_config_updates(const Semver &old_slic3r_version auto filament_in_cache = (cache_profile_path / vendor_name / PRESET_FILAMENT_NAME); auto machine_in_cache = (cache_profile_path / vendor_name / PRESET_PRINTER_NAME); - if (( fs::exists(path_in_vendor)) + if (is_vendor_installed(vendor_name) || fs::exists(print_in_cache) || fs::exists(filament_in_cache) || fs::exists(machine_in_cache)) { - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + // Orca: a vendor installed as a preset cache carries its version there. + Semver vendor_ver = installed_vendor_version(vendor_name); std::map key_values; std::vector keys(3); diff --git a/tests/libslic3r/CMakeLists.txt b/tests/libslic3r/CMakeLists.txt index bc10bb4f73..28c39c2d6a 100644 --- a/tests/libslic3r/CMakeLists.txt +++ b/tests/libslic3r/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(${_TEST_NAME}_tests test_preset_bundle_loading.cpp test_preset_setting_id.cpp test_preset_diff.cpp + test_vendor_cache.cpp test_elephant_foot_compensation.cpp test_fill_corner_smoothing.cpp test_fill_plane_path.cpp diff --git a/tests/libslic3r/test_geometry.cpp b/tests/libslic3r/test_geometry.cpp index b5f7b7ef98..b4bbe86cc6 100644 --- a/tests/libslic3r/test_geometry.cpp +++ b/tests/libslic3r/test_geometry.cpp @@ -574,11 +574,6 @@ TEST_CASE("Convex polygon intersection on two squares touching one vertex", "[Ge Polygon B = A; B.translate(10 / SCALING_FACTOR, 10 / SCALING_FACTOR); - SVG svg{std::string("one_vertex_touch") + ".svg"}; - svg.draw(A, "blue"); - svg.draw(B, "green"); - svg.Close(); - bool is_inters = Geometry::convex_polygons_intersect(A, B); REQUIRE(is_inters == false); diff --git a/tests/libslic3r/test_preset_bundle_loading.cpp b/tests/libslic3r/test_preset_bundle_loading.cpp index 844ccb6a8b..037a76a805 100644 --- a/tests/libslic3r/test_preset_bundle_loading.cpp +++ b/tests/libslic3r/test_preset_bundle_loading.cpp @@ -1,6 +1,7 @@ #include #include +#include #include "libslic3r/PresetBundle.hpp" #include "libslic3r/AppConfig.hpp" @@ -132,7 +133,7 @@ TEST_CASE("Current vendor type tolerates missing printer model", "[Preset][Bundl { PresetBundle bundle; - VendorProfile orca_vendor("ORCA"); + VendorProfile orca_vendor; orca_vendor.id = "ORCA"; VendorProfile::PrinterModel model; model.name = "Orca Test"; orca_vendor.models.emplace_back(model); @@ -143,6 +144,31 @@ TEST_CASE("Current vendor type tolerates missing printer model", "[Preset][Bundl CHECK(bundle.get_current_vendor_type() == VendorType::Unknown); } +TEST_CASE("A malformed entry in a vendor's preset list is counted, not thrown", "[Preset][Bundle]") +{ + ScopedTemporaryDir dir; + + // A bare number where the list wants an object. An array element has no key, + // so reporting one as if it did throws nlohmann's invalid_iterator - which is + // not a parse_error, and escapes the catch around the vendor profile parse. + std::ofstream((dir.path() / "Acme.json").string()) + << R"({"version":"1.0.0","name":"Acme","process_list":[123,)" + << R"({"name":"0.20mm Standard @Acme","sub_path":"process/standard.json"}]})"; + fs::create_directories(dir.path() / "Acme" / "process"); + std::ofstream((dir.path() / "Acme" / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @Acme","from":"system",)" + << R"("instantiation":"true","layer_height":"0.2"})"; + + PresetBundle bundle; + size_t loaded = 0; + REQUIRE_NOTHROW(loaded = bundle.load_vendor_configs_from_json( + dir.path().string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent).second); + + CHECK(bundle.error_count() > 0); // the malformed element was counted + CHECK(loaded == 1); // the well-formed one beside it still loaded +} + TEST_CASE("Printer extruder count tolerates missing nozzle diameter", "[Preset][Bundle]") { PresetBundle bundle; diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp new file mode 100644 index 0000000000..85e100f4d4 --- /dev/null +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -0,0 +1,1620 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/PresetCacheFormat.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/PrintConfig.hpp" +#include "libslic3r/Utils.hpp" + +using namespace Slic3r; +using Catch::Matchers::WithinAbs; +namespace fs = boost::filesystem; + +namespace { + +struct TempDir { + fs::path path; + TempDir() { + path = fs::temp_directory_path() / fs::unique_path("orca-cache-test-%%%%-%%%%"); + fs::create_directories(path); + } + ~TempDir() { boost::system::error_code ec; fs::remove_all(path, ec); } +}; + +std::string write_vendor_json(const fs::path& dir, const std::string& vendor_id, + const std::string& version = "1.0.0") +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"version":")" << version << R"(","name":")" << vendor_id << R"("})"; + return p.string(); +} + +// One vendor profile with a single process preset beside it, as an install or an +// update lays it down: /.json plus //process/standard.json. +void write_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor + << R"(","process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}]})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; +} + +// A small but complete vendor: one machine model, one process, a non-instantiated +// base filament with an instantiated child inheriting it, a second standalone +// filament carrying explicit metadata, and one machine preset with a rename — so +// the equivalence test below sees every CachedPreset field populated. +void write_full_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + fs::create_directories(dir / vendor / "filament"); + fs::create_directories(dir / vendor / "machine"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor << R"(",)" + << R"("machine_model_list":[{"name":"Test Model","sub_path":"machine/model.json"}],)" + << R"("process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}],)" + << R"("filament_list":[)" + << R"({"name":")" << vendor << R"( Base PLA","sub_path":"filament/base.json"},)" + << R"({"name":")" << vendor << R"( PLA @0.4","sub_path":"filament/pla.json"},)" + << R"({"name":")" << vendor << R"( Silk PLA @0.4","sub_path":"filament/silk.json"}],)" + << R"("machine_list":[{"name":")" << vendor << R"( 0.4 nozzle","sub_path":"machine/printer.json"}]})"; + std::ofstream((dir / vendor / "machine" / "model.json").string()) + << R"({"type":"machine_model","name":"Test Model","nozzle_diameter":"0.4"})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; + std::ofstream((dir / vendor / "filament" / "base.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( Base PLA","from":"system","instantiation":"false","filament_id":"GFA_base","filament_cost":"42"})"; + std::ofstream((dir / vendor / "filament" / "pla.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( PLA @0.4","from":"system","instantiation":"true","filament_id":"GFA00","filament_cost":"20",)" + << R"("setting_id":"GFSA04","description":"Test PLA description"})"; + std::ofstream((dir / vendor / "filament" / "silk.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( Silk PLA @0.4","from":"system","instantiation":"true","inherits":")" << vendor << R"( Base PLA"})"; + std::ofstream((dir / vendor / "machine" / "printer.json").string()) + << R"({"type":"machine","name":")" << vendor + << R"( 0.4 nozzle","from":"system","instantiation":"true","printer_model":"Test Model","printer_variant":"0.4",)" + << R"("renamed_from":")" << vendor << R"( old 0.4 nozzle"})"; +} + +// The filament library: one non-instantiated base filament other vendors inherit +// from. `cost` lets a test bump the library and watch the change flow through. +void write_lib_tree(const fs::path& dir, const std::string& version, const std::string& cost) +{ + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + fs::create_directories(dir / lib / "filament"); + std::ofstream((dir / (lib + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << lib << R"(",)" + << R"("filament_list":[{"name":"Generic PLA","sub_path":"filament/generic_pla.json"}]})"; + std::ofstream((dir / lib / "filament" / "generic_pla.json").string()) + << R"({"type":"filament","name":"Generic PLA","from":"system","instantiation":"false",)" + << R"("filament_id":"GFL99","filament_cost":")" << cost << R"("})"; +} + +// A vendor whose one filament inherits the library's base and states nothing of +// its own — everything it shows comes from the library it is resolved against. +void write_vendor_with_lib_filament(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "filament"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor << R"(",)" + << R"("filament_list":[{"name":")" << vendor << R"( PLA @0.4","sub_path":"filament/pla.json"}]})"; + std::ofstream((dir / vendor / "filament" / "pla.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( PLA @0.4","from":"system","instantiation":"true","inherits":"Generic PLA"})"; +} + +std::string write_versionless_vendor_json(const fs::path& dir, const std::string& vendor_id) +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"name":")" << vendor_id << R"("})"; + return p.string(); +} + +// Whole file as bytes, for the byte-identity comparisons below. +std::string slurp(const fs::path& p) +{ + std::string s; + load_string_file(p, s); + return s; +} + +// Flip one byte of the body. The default lands in the stamps at the front, which +// every reader checks; pass an offset past them to corrupt a file that still +// answers VendorCacheFile::peek_version but cannot survive its CRC. +void corrupt_blob_byte(const std::string& path, std::streamoff at = 30) +{ + std::fstream f(path, std::ios::in | std::ios::out | std::ios::binary); + f.seekp(at); + char b = 0; f.read(&b, 1); + f.seekp(at); + b ^= 0xFF; + f.write(&b, 1); +} + +// Overwrite `n` bytes at `payload_off` into the cache's payload (which starts at +// file offset 20, behind the header) and recompute the header CRC, so the file +// stays authentic and only the deserializer can object to its contents. +void patch_payload_bytes(const std::string& path, size_t payload_off, const void* bytes, size_t n) +{ + constexpr size_t header_size = 20; // magic(4) + version(4) + data_size(8) + crc32(4) + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + REQUIRE(data.size() >= header_size + payload_off + n); + std::memcpy(&data[header_size + payload_off], bytes, n); + boost::crc_32_type crc; + crc.process_bytes(&data[header_size], data.size() - header_size); + const uint32_t new_crc = crc.checksum(); + std::memcpy(&data[16], &new_crc, 4); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); +} + +// Patch cache_version (the payload's first word) so the file passes the CRC +// check but fails the cache_version check in VendorCacheFile::load. +void patch_cache_version(const std::string& path, uint32_t wrong_version) +{ + patch_payload_bytes(path, 0, &wrong_version, sizeof(wrong_version)); +} + +// Truncates the cache's PAYLOAD (everything after the 20-byte header) by +// `truncate_by` bytes and recomputes data_size/crc32 in the header, exactly +// as the cache writer computes them, so the framing's size and CRC checks +// still pass but cereal runs out of bytes partway through deserializing the +// body — exercising VendorCacheFile::load's catch block instead of its early +// (pre-body) rejection paths. +void truncate_payload_and_fix_header(const std::string& path, size_t truncate_by) +{ + constexpr size_t header_size = 20; // magic(4) + version(4) + data_size(8) + crc32(4) + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + REQUIRE(data.size() > header_size + truncate_by); + const size_t new_payload_size = data.size() - header_size - truncate_by; + const uint64_t data_size_field = static_cast(new_payload_size); + boost::crc_32_type crc; + crc.process_bytes(&data[header_size], new_payload_size); + const uint32_t crc_field = crc.checksum(); + std::memcpy(&data[8], &data_size_field, sizeof(data_size_field)); // data_size offset + std::memcpy(&data[16], &crc_field, sizeof(crc_field)); // crc32 offset + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(header_size + new_payload_size)); +} + +// One vendor as a cache's VendorMap. It carries one printer model ("Test Model", +// variant "0.4") so machine entries can pass install's model/variant validation. +VendorMap one_vendor(const std::string& vendor_id, const std::string& name = "", + Semver ver = Semver(1, 0, 0)) +{ + VendorMap vendors; + VendorProfile vp(vendor_id); + vp.name = name.empty() ? vendor_id + " Corp" : name; + vp.config_version = ver; + VendorProfile::PrinterModel model; + model.id = "Test Model"; + model.variants.emplace_back(VendorProfile::PrinterVariant("0.4")); + vp.models.push_back(model); + vendors.emplace(vendor_id, vp); + return vendors; +} + +// Source-form entries as parse_subfile would emit them. The alias is derived by +// install from the '@' in the name, exactly as it is for the JSON parse. +CachedPreset filament_entry(const std::string& name, const std::string& filament_id = "GFA00", + const std::string& inherits = "") +{ + CachedPreset e; + e.name = name; + e.sub_path = "filament/" + name + ".json"; + e.instantiation = "true"; + e.filament_id = filament_id; + e.inherits = inherits; + return e; +} + +CachedPreset printer_entry(const std::string& name) +{ + CachedPreset e; + e.name = name; + e.sub_path = "machine/" + name + ".json"; + e.instantiation = "true"; + e.config_src.set_key_value("printer_model", new ConfigOptionString("Test Model")); + e.config_src.set_key_value("printer_variant", new ConfigOptionString("0.4")); + return e; +} + +static bool save_one_vendor(const std::string& path, const VendorMap& vendors, + const std::string& vendor, const std::string& vendor_version, + const std::vector& filament_entries = {}, + const std::vector& machine_entries = {}, + const std::vector& process_entries = {}) +{ + VendorCacheData data; + data.vendors = vendors; + data.process_entries = process_entries; + data.filament_entries = filament_entries; + data.machine_entries = machine_entries; + return VendorCacheFile::save(path, vendor, vendor_version, data); +} + +// resources_dir()/data_dir() are process-wide, so restore them however the test +// leaves — including through a failed REQUIRE — to stay green under --order rand. +struct ScopedDirs { + std::string prev_data{data_dir()}, prev_rsrc{resources_dir()}; + ScopedDirs(const fs::path& data, const fs::path& rsrc) + { + set_data_dir(data.string()); + set_resources_dir(rsrc.string()); + } + ~ScopedDirs() { set_data_dir(prev_data); set_resources_dir(prev_rsrc); } +}; + +// A data dir and a resources dir, both pointed at by the process-wide accessors, +// with the two directories a vendor is installed into and shipped from already +// created. What every install- and load-order test needs before it starts. +struct InstallDirs { + TempDir data, rsrc; + fs::path system = data.path / PRESET_SYSTEM_DIR; + fs::path profiles = rsrc.path / "profiles"; + ScopedDirs scoped { data.path, rsrc.path }; + + InstallDirs() + { + fs::create_directories(system); + fs::create_directories(profiles); + } +}; + +// Helper: filter a collection by vendor_id. +std::vector presets_for(const PresetCollection& coll, const std::string& vendor_id) +{ + std::vector out; + for (const Preset& p : coll()) + if (p.is_system && p.vendor && p.vendor->id == vendor_id) + out.push_back(&p); + return out; +} + +} // namespace + +namespace Slic3r { +inline bool operator==(const VendorProfile::PrinterVariant& a, const VendorProfile::PrinterVariant& b) { return a.name == b.name; } +inline bool operator==(const VendorProfile::PrinterModel& a, const VendorProfile::PrinterModel& b) +{ + return a.id == b.id && a.name == b.name && a.model_id == b.model_id && a.technology == b.technology + && a.family == b.family && a.variants == b.variants && a.default_materials == b.default_materials + && a.not_support_bed_types == b.not_support_bed_types && a.bed_model == b.bed_model + && a.bed_texture == b.bed_texture && a.image_bed_type == b.image_bed_type + && a.bottom_texture_end_name == b.bottom_texture_end_name + && a.use_double_extruder_default_texture == b.use_double_extruder_default_texture + && a.bottom_texture_rect == b.bottom_texture_rect + && a.bottom_texture_rect_longer == b.bottom_texture_rect_longer + && a.middle_texture_rect == b.middle_texture_rect && a.hotend_model == b.hotend_model; +} +} // namespace Slic3r + +static bool vendor_deep_equal(const VendorProfile& a, const VendorProfile& b) +{ + return a.name == b.name && a.id == b.id && a.config_version == b.config_version + && a.config_update_url == b.config_update_url && a.changelog_url == b.changelog_url + && a.models == b.models && a.default_filaments == b.default_filaments + && a.default_sla_materials == b.default_sla_materials; +} + +static bool preset_deep_equal(const Preset& a, const Preset& b) +{ + return a.type == b.type && a.is_default == b.is_default && a.is_external == b.is_external + && a.is_system == b.is_system && a.is_visible == b.is_visible && a.is_dirty == b.is_dirty + && a.is_compatible == b.is_compatible && a.is_project_embedded == b.is_project_embedded + && a.name == b.name && a.file == b.file && a.loaded == b.loaded + && a.config.equals(b.config) + && a.alias == b.alias && a.renamed_from == b.renamed_from + && a.m_excluded_from == b.m_excluded_from && a.m_from_orca_filament_lib == b.m_from_orca_filament_lib + && a.bundle_id == b.bundle_id && a.version == b.version && a.ini_str == b.ini_str + && a.setting_id == b.setting_id && a.filament_id == b.filament_id && a.user_id == b.user_id + && a.base_id == b.base_id && a.sync_info == b.sync_info && a.description == b.description + && a.updated_time == b.updated_time && a.key_values == b.key_values; +} + +TEST_CASE("a saved cache loads back with names, aliases and filament ids intact", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA @0.4", "GFL_acme_pla")}, + {printer_entry(vid + " Printer 0.4")})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + REQUIRE(out.vendors.count(vid) == 1); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 1); + CHECK(fi[0]->name == vid + " PLA @0.4"); + CHECK(fi[0]->alias == "Acme PLA"); + CHECK(fi[0]->filament_id == "GFL_acme_pla"); + REQUIRE(pr.size() == 1); + CHECK(pr[0]->name == vid + " Printer 0.4"); +} + +TEST_CASE("loading a missing cache file returns false", "[VendorCache]") +{ + TempDir tmp; + PresetBundle out; + REQUIRE(!out.load_vendor_cache((tmp.path / "nonexistent.opc").string(), "Acme", Semver("1.0.0"))); +} + +TEST_CASE("a cache with a corrupted byte is rejected by the CRC check", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); + corrupt_blob_byte(cache.string()); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); +} + +TEST_CASE("two vendors produce two independent cache files", "[VendorCache]") +{ + TempDir tmp; + const fs::path cacheA = tmp.path / "vendorA.opc"; + const fs::path cacheB = tmp.path / "vendorB.opc"; + + REQUIRE(save_one_vendor(cacheA.string(), one_vendor("VendorA"), "VendorA", "1.0.0", + {filament_entry("VendorA PLA")})); + REQUIRE(save_one_vendor(cacheB.string(), one_vendor("VendorB"), "VendorB", "1.0.0", + {filament_entry("VendorB PLA")})); + + // Corrupt only vendor B's file; vendor A's must be unaffected. + corrupt_blob_byte(cacheB.string()); + + PresetBundle outA; + REQUIRE(outA.load_vendor_cache(cacheA.string(), "VendorA", Semver("1.0.0"))); + REQUIRE(outA.vendors.count("VendorA") == 1); + REQUIRE(presets_for(outA.filaments, "VendorA").size() == 1); + + PresetBundle outB; + REQUIRE(!outB.load_vendor_cache(cacheB.string(), "VendorB", Semver("1.0.0"))); + REQUIRE(outB.vendors.empty()); +} + +TEST_CASE("vendor profile fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + VendorMap vendors; + VendorProfile vp(vid); + vp.name = "Acme Corporation"; + vp.config_version = Semver(2, 5, 1); + VendorProfile::PrinterModel model; + model.id = "AcmePro"; + model.name = "Acme Pro"; + VendorProfile::PrinterVariant v0_4; v0_4.name = "0.4"; + model.variants.push_back(v0_4); + vp.models.push_back(model); + vendors.emplace(vid, vp); + REQUIRE(save_one_vendor(cache.string(), vendors, vid, "2.5.1")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("2.5.1"))); + REQUIRE(out.vendors.count(vid) == 1); + const VendorProfile& gvp = out.vendors.at(vid); + REQUIRE(vendor_deep_equal(gvp, vendors.at(vid))); + // Spot-check the fields the old test asserted directly, so a + // vendor_deep_equal regression still points at what actually broke. + CHECK(gvp.id == vid); + CHECK(gvp.name == "Acme Corporation"); + REQUIRE(gvp.models.size() == 1); + CHECK(gvp.models[0].id == "AcmePro"); + CHECK(gvp.models[0].name == "Acme Pro"); + REQUIRE(gvp.models[0].variants.size() == 1); + CHECK(gvp.models[0].variants[0].name == "0.4"); +} + +TEST_CASE("config option values survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + auto entry = filament_entry(vid + " PETG @0.4"); + entry.config_src.set_key_value("filament_type", new ConfigOptionStrings({"PETG"})); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", {entry})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + + auto fi = presets_for(out.filaments, vid); + REQUIRE(fi.size() == 1); + const auto* ft = fi[0]->config.option("filament_type"); + REQUIRE(ft != nullptr); + REQUIRE(ft->values.size() >= 1); + CHECK(ft->values[0] == "PETG"); +} + +TEST_CASE("multiple presets in one collection all round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + const std::vector fi_names = {vid + " PLA", vid + " PETG", vid + " ABS"}; + const std::vector pr_names = {vid + " Printer 0.4", vid + " Printer 0.6"}; + std::vector filament_entries, machine_entries; + for (const auto& n : fi_names) filament_entries.push_back(filament_entry(n)); + for (const auto& n : pr_names) machine_entries.push_back(printer_entry(n)); + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + filament_entries, machine_entries)); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 3); + REQUIRE(pr.size() == 2); + + std::set fi_got, pr_got; + for (const auto* p : fi) fi_got.insert(p->name); + for (const auto* p : pr) pr_got.insert(p->name); + for (const auto& n : fi_names) CHECK(fi_got.count(n) == 1); + for (const auto& n : pr_names) CHECK(pr_got.count(n) == 1); +} + +TEST_CASE("a truncated cache file is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "truncated.opc"; + { + std::ofstream f(cache.string(), std::ios::binary); + const char data[] = {0x4F, 0x52, 0x43}; + f.write(data, sizeof(data)); + } + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); +} + +TEST_CASE("a cache with the wrong magic number is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); + + { + std::fstream f(cache.string(), std::ios::in | std::ios::out | std::ios::binary); + const uint32_t bad = 0xDEADBEEFu; + f.write(reinterpret_cast(&bad), sizeof(bad)); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); +} + +TEST_CASE("a vendor with no presets saves and loads cleanly", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid, "Acme Corporation"), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + REQUIRE(out.vendors.count(vid) == 1); + CHECK(out.vendors.at(vid).id == vid); + CHECK(out.vendors.at(vid).name == "Acme Corporation"); + CHECK(presets_for(out.filaments, vid).empty()); + CHECK(presets_for(out.printers, vid).empty()); + CHECK(presets_for(out.prints, vid).empty()); +} + +TEST_CASE("a cache-loaded vendor is indistinguishable from a JSON-loaded one", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_full_vendor_tree(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); + + // Take the preset JSONs away: were the cache rejected, the load below would + // have nothing to parse — so its success proves the cache answered. + fs::remove_all(user / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + // Both paths run the same install code over the same entries, so everything + // observable must come out identical — the vendor profile and every preset, + // field by field. + REQUIRE(from_cache.vendors.count("Acme") == 1); + REQUIRE(vendor_deep_equal(from_cache.vendors.at("Acme"), from_json.vendors.at("Acme"))); + const std::pair colls[] = { + {&from_json.prints, &from_cache.prints}, + {&from_json.filaments, &from_cache.filaments}, + {&from_json.printers, &from_cache.printers}, + }; + for (const auto& [jc, cc] : colls) { + auto a = presets_for(*jc, "Acme"); + auto b = presets_for(*cc, "Acme"); + REQUIRE(a.size() == b.size()); + REQUIRE(!a.empty()); + for (size_t i = 0; i < a.size(); ++i) { + CHECK(a[i]->name == b[i]->name); + CHECK(preset_deep_equal(*a[i], *b[i])); + } + } + + // Pin the explicit metadata against symmetric loss: dropping a field from + // visit_entry (PresetCacheFormat.cpp) keeps the two bundles equal to each + // other, but not to the fixture. + const Preset* pla = from_cache.filaments.find_preset("Acme PLA @0.4", false); + REQUIRE(pla != nullptr); + CHECK(pla->setting_id == "GFSA04"); + CHECK(pla->description == "Test PLA description"); + const Preset* silk = from_cache.filaments.find_preset("Acme Silk PLA @0.4", false); + REQUIRE(silk != nullptr); + CHECK(silk->filament_id == "GFA_base"); // inherited from the non-instantiated base + const auto* cost = silk->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(42., 1e-9)); + const Preset* pr = from_cache.printers.find_preset("Acme 0.4 nozzle", false); + REQUIRE(pr != nullptr); + CHECK(pr->renamed_from == std::vector{"Acme old 0.4 nozzle"}); +} + +TEST_CASE("a cache-served vendor reports the errors its parse counted", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + // One process preset without the required "instantiation" key — a parse-phase + // error the load survives, so it must reach the cache's parse_errors stamp. + fs::create_directories(user / "Acme" / "process"); + std::ofstream((user / "Acme.json").string()) + << R"({"version":"1.0.0","name":"Acme","process_list":[{"name":"0.20mm Standard @Acme","sub_path":"process/standard.json"}]})"; + std::ofstream((user / "Acme" / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @Acme","from":"system","layer_height":"0.2"})"; + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); + CHECK(from_json.error_count() > 0); + + fs::remove_all(user / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(from_cache.error_count() == from_json.error_count()); + CHECK(presets_for(from_cache.prints, "Acme").size() == 1); +} + +TEST_CASE("a non-instantiated base in a regular vendor's cache resolves its children and stays out of the library maps", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + + // Entry order is the resolution order: the base must install (into the local + // config maps) before the child that inherits it. + auto base = filament_entry("Acme Base PLA", "GFA_base"); + base.instantiation = "false"; + base.config_src.set_key_value("filament_cost", new ConfigOptionFloats({42.})); + auto child = filament_entry("Acme Silk PLA @0.4", "", "Acme Base PLA"); + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0", {base, child})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + auto fi = presets_for(out.filaments, "Acme"); + REQUIRE(fi.size() == 1); // the base never becomes a preset + CHECK(fi[0]->name == "Acme Silk PLA @0.4"); + CHECK(fi[0]->filament_id == "GFA_base"); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(42., 1e-9)); + // Only the filament library's bases persist as the cross-vendor inheritance + // maps; a regular vendor's stay local to its own load. + CHECK(out.m_config_maps.empty()); + CHECK(out.m_filament_id_maps.empty()); +} + +TEST_CASE("a cache with the wrong cache version is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); + patch_cache_version(cache.string(), 0xFFFFFFFFu); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); +} + +TEST_CASE("a cache truncated mid-blob is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); + + { + std::ifstream in(cache.string(), std::ios::binary); + std::vector buf(30); // 20-byte header + 10 bytes of blob + in.read(buf.data(), 30); + in.close(); + std::ofstream out(cache.string(), std::ios::binary | std::ios::trunc); + out.write(buf.data(), 30); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); +} + +TEST_CASE("printer model bed texture fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + VendorMap vendors = one_vendor(vid); + VendorProfile::PrinterModel model; + model.id = "N1"; + model.name = "Neat One"; + model.bottom_texture_rect_longer = "5,5,50,10"; + vendors.at(vid).models.push_back(model); + REQUIRE(save_one_vendor(cache.string(), vendors, vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + REQUIRE(out.vendors.at(vid).models.size() == 2); + REQUIRE(vendor_deep_equal(out.vendors.at(vid), vendors.at(vid))); + CHECK(out.vendors.at(vid).models[1].bottom_texture_rect_longer == "5,5,50,10"); +} + +TEST_CASE("a cache older than the vendor profile on disk is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.1"))); +} + +TEST_CASE("a cache newer than the vendor profile on disk is used", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.2.0", + {filament_entry("Acme PLA")})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + CHECK(presets_for(out.filaments, "Acme").size() == 1); +} + +TEST_CASE("a vendor cache outlives a filament library update and resolves against the new library", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(user, "1.0.0", "20"); + write_vendor_with_lib_filament(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // First launch: the library parses first, then the vendor against it, and + // both caches are written. + PresetBundle base1; + base1.set_generate_vendor_caches(true); + base1.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme1; + acme1.set_generate_vendor_caches(true); + acme1.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base1); + REQUIRE(fs::exists(user / "Acme.opc")); + { + auto fi = presets_for(acme1.filaments, "Acme"); + REQUIRE(fi.size() == 1); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(20., 1e-9)); + } + + // An update delivers a new library only; the vendor stays as it was. + write_lib_tree(user, "2.0.0", "30"); + PresetBundle base2; + base2.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + // Take the vendor's preset JSONs away: were its cache rejected, the load + // below would have nothing to parse — so its success proves the cache + // survived the library bump. + fs::remove_all(user / "Acme"); + PresetBundle acme2; + auto [substitutions, presets_loaded] = acme2.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base2); + CHECK(presets_loaded == 1); + auto fi = presets_for(acme2.filaments, "Acme"); + REQUIRE(fi.size() == 1); + // The cache holds only the vendor's own diff; the library values come from + // the library loaded now, not the one in effect when the cache was written. + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); + CHECK(fi[0]->filament_id == "GFL99"); +} + +TEST_CASE("a vendor installed as its cache alone still loads after a library update", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(user, "1.0.0", "20"); + write_vendor_with_lib_filament(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Generate the vendor's cache, then strip the vendor to the cache alone — + // the shape of a packaged install, which ships each vendor as its .opc and + // nothing else. + PresetBundle base1; + base1.set_generate_vendor_caches(true); + base1.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme1; + acme1.set_generate_vendor_caches(true); + acme1.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base1); + fs::remove(user / "Acme.json"); + fs::remove_all(user / "Acme"); + + // An OTA update then delivers a new library only. With no JSONs anywhere to + // fall back on, the vendor must keep loading from its cache. + write_lib_tree(user, "2.0.0", "30"); + PresetBundle base2; + base2.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme2; + auto [substitutions, presets_loaded] = acme2.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base2); + CHECK(presets_loaded == 1); + REQUIRE(acme2.vendors.count("Acme") == 1); + auto fi = presets_for(acme2.filaments, "Acme"); + REQUIRE(fi.size() == 1); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); +} + +TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_tree(user, "Acme", "1.0.0"); + + // A cache claiming the installed version, but whose entry inherits a preset + // no loaded library provides. + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4", "GFA00", "No Such Base")})); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Directly: the load fails and leaves the bundle clean. + PresetBundle direct; + REQUIRE(!direct.load_vendor_cache((user / "Acme.opc").string(), "Acme", Semver("1.0.0"))); + CHECK(direct.vendors.empty()); + + // Through the vendor load: the JSONs answer instead, as if no cache existed. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").name == "Acme"); // the profile's name, not the cache's +} + +TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + + PresetBundle out; + // An unversioned vendor profile has no version to compare against. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver::invalid())); + // And a cache carrying no version of its own cannot cover a profile that has one. + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "")); + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + REQUIRE(out.vendors.empty()); +} + +TEST_CASE("a versionless profile beside a cache keeps the cache from being served", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4")})); + // The profile beside the cache parses to no usable version, which can no + // more judge the cache's staleness than it could be cached itself. + write_versionless_vendor_json(user, "Acme"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + // Nothing came from the cache: the versionless profile was parsed instead, + // and it carries no presets. + CHECK(presets_loaded == 0); +} + +TEST_CASE("a vendor's cache is its whole installation", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources"; + const fs::path data = tmp.path / "data"; + fs::create_directories(rsrc / "profiles" / "Acme" / "machine"); + write_vendor_json(rsrc / "profiles", "Acme"); + std::ofstream((rsrc / "profiles" / "Acme" / "machine" / "printer.json").string()) << "{}"; + + REQUIRE(save_one_vendor((rsrc / "profiles" / "Acme.opc").string(), one_vendor("Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(data, rsrc); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + // The cache carries the presets, the vendor profile and the version they were + // built at, so it is installed on its own. + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); + CHECK(is_vendor_installed("Acme")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // A vendor with no cache is installed as its profile and preset JSONs instead, + // parsing them being the only way left to load it — and the cache the previous + // install left behind has to go, or it would shadow the profile just installed. + fs::remove(rsrc / "profiles" / "Acme.opc"); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(!fs::exists(data / "system" / "Acme.opc")); + CHECK(fs::exists(data / "system" / "Acme" / "machine" / "printer.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // Installing the cache again takes the profile and its preset JSONs back out. + REQUIRE(save_one_vendor((rsrc / "profiles" / "Acme.opc").string(), one_vendor("Acme"), "Acme", "1.0.0")); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); +} + +TEST_CASE("a vendor shipped as a cache alone is installed and loaded from it", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // A packaged build: every vendor is its cache, with no profile of any kind + // beside it — not even the filament library's. + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + REQUIRE(save_one_vendor((rsrc / (lib + ".opc")).string(), one_vendor(lib, "Shipped Library"), lib, "1.0.0")); + REQUIRE(save_one_vendor((rsrc / "Acme.opc").string(), one_vendor("Acme", "Shipped Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The version the build ships the vendor at comes from the cache, there being + // no profile to read it from. + CHECK(resource_vendor_version("Acme") == Semver(1, 0, 0)); + + // Resources reaches the app by being installed, never by being loaded from. + REQUIRE(install_vendor_bundles_from_resources({lib, "Acme"})); + CHECK(fs::exists(user / "Acme.opc")); + CHECK(!fs::exists(user / "Acme.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + PresetBundle after; + after.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(after.vendors.at("Acme").name == "Shipped Acme"); +} + +TEST_CASE("a vendor with a profile in the data dir is parsed there and cached there, whatever resources ships", "[VendorCache]") +{ + // The reported regression: a valid resources/profiles/.opc answered + // first, so the JSON in system/ was never parsed and system/.opc was + // never written. Main reads system/ and nothing else. + InstallDirs dirs; + + write_vendor_tree(dirs.system, "Shadow", "1.0.0"); + // A cache in resources at the very same version — under the old two-tier + // lookup this was accepted and the parse skipped. + REQUIRE(save_one_vendor((dirs.profiles / "Shadow.opc").string(), one_vendor("Shadow"), "Shadow", "1.0.0")); + + PresetBundle bundle; + bundle.set_generate_vendor_caches(true); + REQUIRE(bundle.load_vendor_configs_from_json(dirs.system.string(), "Shadow", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent).second > 0); + + // Parsed from system/, and its cache written back beside the profile. + CHECK(fs::exists(dirs.system / "Shadow.opc")); + CHECK(presets_for(bundle.prints, "Shadow").size() == 1); +} + +TEST_CASE("a vendor with nothing installed is not loaded from resources", "[VendorCache]") +{ + // Resources reaches the app by being installed into system/ first. A vendor + // that is not installed is not loaded, however completely resources ships it. + InstallDirs dirs; + + write_vendor_tree(dirs.profiles, "Absent", "1.0.0"); + REQUIRE(save_one_vendor((dirs.profiles / "Absent.opc").string(), one_vendor("Absent"), "Absent", "1.0.0")); + + PresetBundle bundle; + REQUIRE_THROWS(bundle.load_vendor_configs_from_json(dirs.system.string(), "Absent", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent)); + CHECK(presets_for(bundle.prints, "Absent").empty()); +} + +TEST_CASE("a cache installed with no profile beside it is used whatever its version", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, "Acme"); + + // Installed at an older version than the one now shipped in resources. Nothing + // sits beside it claiming to be newer, so the cache is what the vendor is. + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Installed Acme"), "Acme", "0.9.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + CHECK(VendorCacheFile::peek_version((user / "Acme.opc").string(), "Acme") == "0.9.0"); + CHECK(VendorCacheFile::peek_version((user / "Acme.opc").string(), "Other").empty()); + CHECK(installed_vendor_version("Acme") == Semver(0, 9, 0)); + + // Loading the vendor takes the installed cache, not the newer shipped profile. + PresetBundle out; + out.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(out.vendors.at("Acme").name == "Installed Acme"); +} + +TEST_CASE("a vendor whose cache covers it is loaded without parsing any JSON", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4")}, + {printer_entry("Acme Printer 0.4")})); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // The cache is the whole installation — no profile, no preset JSONs — and the + // caller asks for the vendor exactly as it would for a JSON install. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + CHECK(substitutions.empty()); + CHECK(presets_loaded == 2); + CHECK(out.vendors.at("Acme").name == "Cached Acme"); + + // Nothing was written back: the presets never came from a parse. + CHECK(!fs::exists(user / "Acme.json")); +} + +TEST_CASE("a vendor whose cache is stale falls back to parsing its JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // An update installed the vendor at 2.0.0; the cache next to it was built from + // the profile before that, so it no longer covers what is on disk. + write_vendor_tree(user, "Acme", "2.0.0"); + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(2, 0, 0)); + + // A one-off parse like this one leaves the stale cache alone: only a bundle + // told its parses are complete writes one. + CHECK(VendorCacheFile::peek_version((user / "Acme.opc").string(), "Acme") == "1.0.0"); + + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(VendorCacheFile::peek_version((user / "Acme.opc").string(), "Acme") + == get_version_from_json((user / "Acme.json").string()).to_string()); +} + +TEST_CASE("a cache with a mismatched vendor name is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("VendorA"), "VendorA", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "VendorB", Semver("1.0.0"))); +} + +TEST_CASE("a cache is rejected against an unparsable version", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + PresetBundle out; + // A profile version that does not parse comes out of get_version_from_json + // as zero, which cannot be judged any more than Semver::invalid() can. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver())); + REQUIRE(out.vendors.empty()); // rejection happens before the body is touched +} + +TEST_CASE("the filament library's inheritance maps are rebuilt on cache load", "[VendorCache]") +{ + // m_config_maps/m_filament_id_maps are the inheritance base other vendors + // resolve against. The cache no longer stores them: they are rebuilt by + // installing the library's entries — including the non-instantiated bases, + // which exist for exactly this and never become presets. + TempDir tmp; + const fs::path cache = tmp.path / "lib.opc"; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + + auto base = filament_entry("Generic PLA", "GFL99"); + base.instantiation = "false"; + base.config_src.set_key_value("filament_cost", new ConfigOptionFloats({20.})); + REQUIRE(save_one_vendor(cache.string(), one_vendor(lib), lib, "1.0.0", {base})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), lib, Semver("1.0.0"))); + REQUIRE(out.m_config_maps.count("Generic PLA") == 1); + const auto* cost = out.m_config_maps.at("Generic PLA").option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(20., 1e-9)); + CHECK(out.m_filament_id_maps.at("Generic PLA") == "GFL99"); + CHECK(presets_for(out.filaments, lib).empty()); // not instantiated, not a preset +} + +TEST_CASE("the same fixture parsed twice serializes byte-identically", "[VendorCache]") +{ + // Shipped caches must be reproducible: the same profiles must produce the + // same bytes on every machine that generates them. + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_full_vendor_tree(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle first; + first.set_generate_vendor_caches(true); + first.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); + const std::string bytes1 = slurp(user / "Acme.opc"); + fs::remove(user / "Acme.opc"); + + PresetBundle second; + second.set_generate_vendor_caches(true); + second.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(slurp(user / "Acme.opc") == bytes1); +} + +TEST_CASE("a cache that fails mid-body deserialization is rejected and leaves the bundle clean", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path valid_cache = tmp.path / "valid.opc"; + const fs::path corrupt_cache = tmp.path / "corrupt.opc"; + + REQUIRE(save_one_vendor(valid_cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA @0.4")}, + {printer_entry(vid + " Printer 0.4")})); + // Truncate the tail (machine entries + parse_errors, per VendorCacheFile::save's + // field order) so the header's size/CRC still validate but cereal runs out of + // bytes partway through the body. Grow the cut if a given size ever stops + // throwing (e.g. after an unrelated field-order change to the cache format). + size_t truncate_by = 40; + bool throws = false; + for (; truncate_by <= 200; truncate_by += 8) { + fs::copy_file(valid_cache, corrupt_cache, fs::copy_option::overwrite_if_exists); + truncate_payload_and_fix_header(corrupt_cache.string(), truncate_by); + PresetBundle probe_bundle; + if (!probe_bundle.load_vendor_cache(corrupt_cache.string(), vid, Semver("1.0.0"))) { + throws = true; + break; + } + } + REQUIRE(throws); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(corrupt_cache.string(), vid, Semver("1.0.0"))); + // The catch block put the bundle back the way a failed parse would leave it. + CHECK(out.vendors.empty()); + CHECK(out.m_config_maps.empty()); + CHECK(presets_for(out.filaments, vid).empty()); + + // The recovery must leave a bundle a caller can still load a good cache into. + REQUIRE(out.load_vendor_cache(valid_cache.string(), vid, Semver("1.0.0"))); + CHECK(out.vendors.count(vid) == 1); + CHECK(presets_for(out.filaments, vid).size() == 1); +} + +TEST_CASE("a cache rejected mid-body leaves the error count where it found it", "[VendorCache]") +{ + InstallDirs dirs; + + // A vendor whose root profile counts a parse error, so the bundle carries a + // non-zero tally into the load below. Without one there is nothing for a + // rejected cache to zero, and nothing to underflow. + std::ofstream((dirs.system / "Noisy.json").string()) + << R"({"version":"1.0.0","name":"Noisy","process_list":"not a list"})"; + + write_vendor_tree(dirs.system, "Counted", "1.0.0"); + // A cache that passes every stamp and then dies in the entries. + REQUIRE(save_one_vendor((dirs.system / "Counted.opc").string(), one_vendor("Counted"), "Counted", "1.0.0", + {filament_entry("Counted PLA @0.4")})); + truncate_payload_and_fix_header((dirs.system / "Counted.opc").string(), 8); + + PresetBundle bundle; + bundle.set_generate_vendor_caches(true); + bundle.load_vendor_configs_from_json(dirs.system.string(), "Noisy", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(bundle.error_count() > 0); + + // The same bundle: the cache is tried, fails mid-body, and the parse that + // follows must be measured against the tally the cache found rather than + // against zero. + REQUIRE(bundle.load_vendor_configs_from_json(dirs.system.string(), "Counted", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent).second > 0); + + // The rewritten cache must carry the parse's own error count, not an + // underflowed one. Reload it and check the bundle does not inherit a + // nonsensical tally. + PresetBundle reloaded; + REQUIRE(reloaded.load_vendor_cache((dirs.system / "Counted.opc").string(), "Counted", Semver(1, 0, 0))); + CHECK(reloaded.error_count() == 0); +} + +TEST_CASE("a preset is traced to its vendor in a build that ships caches alone", "[VendorCache]") +{ + InstallDirs dirs; + + std::vector filaments { filament_entry("Cached PLA @0.4") }; + std::vector printers { printer_entry("Cached 0.4 nozzle") }; + REQUIRE(save_one_vendor((dirs.profiles / "Cached.opc").string(), one_vendor("Cached"), "Cached", "1.0.0", + filaments, printers)); + + CHECK(PresetBundle::find_preset_vendor("Cached PLA @0.4", Preset::TYPE_FILAMENT) == "Cached"); + CHECK(PresetBundle::find_preset_vendor("Cached 0.4 nozzle", Preset::TYPE_PRINTER) == "Cached"); + CHECK(PresetBundle::find_preset_vendor("Nobody's PLA", Preset::TYPE_FILAMENT).empty()); +} + +TEST_CASE("a bundle that cannot be installed does not drop the others", "[VendorCache]") +{ + InstallDirs dirs; + + write_vendor_tree(dirs.profiles, "Good", "1.0.0"); + + // An empty name sorts first out of a std::map, and a name resources does not + // carry can appear anywhere. Neither may cost the batch the vendors it can + // install. + CHECK_FALSE(install_vendor_bundles_from_resources({"", "Absent", "Good"})); + CHECK(fs::exists(dirs.system / "Good.json")); +} + +TEST_CASE("a cache that arrives unusable leaves the profile fallback in place", "[VendorCache]") +{ + InstallDirs dirs; + + write_vendor_tree(dirs.profiles, "Torn", "1.0.0"); + const std::string cache = (dirs.profiles / "Torn.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Torn"), "Torn", "1.0.0")); + // Past the stamps at the front, so the 1 KB peek that chooses the cache form + // still succeeds — only the CRC, which decides whether it can be served, + // catches this. + corrupt_blob_byte(cache, std::streamoff(fs::file_size(cache)) - 4); + REQUIRE(VendorCacheFile::peek_version(cache, "Torn") == "1.0.0"); + + CHECK(install_vendor_bundles_from_resources({"Torn"})); + CHECK(fs::exists(dirs.system / "Torn.json")); + CHECK_FALSE(fs::exists(dirs.system / "Torn.opc")); +} + +TEST_CASE("a vendor installed as an unreadable cache alone counts as not installed", "[VendorCache]") +{ + InstallDirs dirs; + + const std::string cache = (dirs.system / "Broken.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Broken"), "Broken", "1.0.0")); + REQUIRE(is_vendor_installed("Broken")); + + // A cache this build cannot serve is not an installation: there is no + // profile beside it and, since the single-tier load, nowhere else to load + // the vendor from. + corrupt_blob_byte(cache); + CHECK_FALSE(is_vendor_installed("Broken")); + CHECK_FALSE(installed_vendor_version("Broken").valid()); +} + +TEST_CASE("a stale profile beside a newer cache does not hide the cache's version", "[VendorCache]") +{ + InstallDirs dirs; + + write_vendor_json(dirs.system, "Both", "1.0.0"); + REQUIRE(save_one_vendor((dirs.system / "Both.opc").string(), one_vendor("Both"), "Both", "2.0.0")); + + // The cache covers the profile, so the cache is what a load serves — and + // 2.0.0 is the version installed, not the 1.0.0 the profile still claims. + CHECK(installed_vendor_version("Both") == Semver(2, 0, 0)); +} + +TEST_CASE("a profile newer than the cache beside it is the installed version", "[VendorCache]") +{ + InstallDirs dirs; + + write_vendor_json(dirs.system, "Both", "3.0.0"); + REQUIRE(save_one_vendor((dirs.system / "Both.opc").string(), one_vendor("Both"), "Both", "2.0.0")); + + // The cache no longer covers the profile, so the profile is parsed — and + // its version is the one in force. + CHECK(installed_vendor_version("Both") == Semver(3, 0, 0)); +} + +TEST_CASE("a header claiming more body than the file holds is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string cache = (tmp.path / "Bounded.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Bounded"), "Bounded", "1.0.0")); + + // Claim a body far larger than the file. Nothing may be allocated on the + // strength of that number. + { + std::fstream f(cache, std::ios::in | std::ios::out | std::ios::binary); + const uint64_t huge = 400ull * 1024ull * 1024ull; + f.seekp(8); + f.write(reinterpret_cast(&huge), sizeof(huge)); + } + + PresetBundle bundle; + REQUIRE_FALSE(bundle.load_vendor_cache(cache, "Bounded", Semver(1, 0, 0))); +} + +TEST_CASE("a failed write leaves the previous cache in place", "[VendorCache]") +{ + TempDir tmp; + const std::string cache = (tmp.path / "Durable.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Durable"), "Durable", "1.0.0")); + const std::string before = slurp(cache); + + // A directory where the temp file wants to go: the write cannot complete, + // and must not have destroyed what was already there to find that out. + const fs::path blocker = fs::path(cache + "." + std::to_string(get_current_pid()) + ".tmp"); + fs::create_directories(blocker); + + REQUIRE_FALSE(save_one_vendor(cache, one_vendor("Durable"), "Durable", "2.0.0")); + CHECK(slurp(cache) == before); + + fs::remove_all(blocker); +} + +TEST_CASE("a cache written by another build's option ordering still loads", "[VendorCache]") +{ + // The regression the fingerprint used to prevent by refusing the file + // outright: nothing in the payload depends on serialization_key_ordinal, so + // a build that inserted an option ahead of these reads them back correctly. + TempDir tmp; + const std::string cache = (tmp.path / "Ordinal.opc").string(); + + auto e = filament_entry("Ordinal PLA @0.4"); + e.config_src.set_key_value("filament_cost", new ConfigOptionFloats({42.})); + e.config_src.set_key_value("filament_type", new ConfigOptionStrings({"PLA"})); + REQUIRE(save_one_vendor(cache, one_vendor("Ordinal"), "Ordinal", "1.0.0", {e})); + + PresetBundle bundle; + REQUIRE(bundle.load_vendor_cache(cache, "Ordinal", Semver(1, 0, 0))); + const auto filaments = presets_for(bundle.filaments, "Ordinal"); + REQUIRE(filaments.size() == 1); + const auto* cost = filaments.front()->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(42., 1e-9)); + CHECK(filaments.front()->config.option("filament_type")->values.front() == "PLA"); +} + +// ---- CacheDictionary and the name-keyed config payload ------------------- + +namespace { + +// Round-trip one config through the dictionary payload, optionally mutating the +// dictionary between write and read to stand in for another build's schema. +DynamicPrintConfig roundtrip_config(const DynamicPrintConfig& in, + const std::function& mutate_blob = {}) +{ + CacheDictionary wdict; + wdict.collect(in); + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + wdict.save(ar); + save_config(ar, in, wdict); + } + std::string blob = os.str(); + if (mutate_blob) + mutate_blob(blob); + std::istringstream is(blob, std::ios::binary); + cereal::BinaryInputArchive ar(is); + CacheDictionary rdict; + rdict.load(ar); + DynamicPrintConfig out; + load_config(ar, out, rdict); + return out; +} + +} // namespace + +TEST_CASE("a config round-trips through the cache dictionary", "[VendorCache]") +{ + DynamicPrintConfig in; + in.set_key_value("layer_height", new ConfigOptionFloat(0.28)); + in.set_key_value("printer_model", new ConfigOptionString("Test Model")); + in.set_key_value("nozzle_diameter", new ConfigOptionFloats({0.4, 0.6})); + in.set_key_value("spiral_mode", new ConfigOptionBool(true)); + + const DynamicPrintConfig out = roundtrip_config(in); + + CHECK_THAT(out.opt_float("layer_height"), WithinAbs(0.28, 1e-9)); + CHECK(out.opt_string("printer_model") == "Test Model"); + REQUIRE(out.option("nozzle_diameter") != nullptr); + CHECK(out.option("nozzle_diameter")->values.size() == 2); + CHECK(out.opt_bool("spiral_mode") == true); +} + +TEST_CASE("an enum option round-trips by name, not by index", "[VendorCache]") +{ + // top_surface_pattern is a coEnum; its stored int is an index into an enum + // whose order is not a wire contract. Assert on the NAME, so a reordering + // of the enum in PrintConfig.cpp cannot make this test pass by accident. + const ConfigOptionDef* def = print_config_def.get("top_surface_pattern"); + REQUIRE(def != nullptr); + REQUIRE(def->type == coEnum); + REQUIRE(def->enum_keys_map != nullptr); + const int monotonic = def->enum_keys_map->at("monotonic"); + + DynamicPrintConfig in; + in.set_key_value("top_surface_pattern", new ConfigOptionEnumGeneric(def->enum_keys_map, monotonic)); + + const DynamicPrintConfig out = roundtrip_config(in); + REQUIRE(out.option("top_surface_pattern") != nullptr); + CHECK(out.opt_enum("top_surface_pattern") == InfillPattern(monotonic)); + CHECK(out.option("top_surface_pattern")->serialize() == "monotonic"); +} + +TEST_CASE("a nullable vector enum round-trips by name, nil included", "[VendorCache]") +{ + // coEnums carries a vector of ints and, unlike coEnum, its ConfigOptionType + // does not fit in a byte - a truncated type in the dictionary would make a + // reader take this for a scalar enum and run off the end of the stream. + // nozzle_type is also nullable, and nil is an int no enum_keys_map names, + // so this covers the dictionary's unnamed-value escape hatch too. + const ConfigOptionDef* def = print_config_def.get("nozzle_type"); + REQUIRE(def != nullptr); + REQUIRE(def->type == coEnums); + REQUIRE(def->nullable); + REQUIRE(def->enum_keys_map != nullptr); + const int brass = def->enum_keys_map->at("brass"); + const int nil = ConfigOptionInts::nil_value(); + + DynamicPrintConfig in; + auto* opt = new ConfigOptionEnumsGenericNullable(def->enum_keys_map); + opt->values = { brass, nil }; + in.set_key_value("nozzle_type", opt); + in.set_key_value("printer_model", new ConfigOptionString("Test Model")); + + const DynamicPrintConfig out = roundtrip_config(in); + const auto* got = out.option("nozzle_type"); + REQUIRE(got != nullptr); + CHECK(got->values == std::vector{brass, nil}); + CHECK(out.opt_string("printer_model") == "Test Model"); +} + +TEST_CASE("an option the build no longer knows is dropped, and the rest still load", "[VendorCache]") +{ + DynamicPrintConfig in; + in.set_key_value("layer_height", new ConfigOptionFloat(0.28)); + in.set_key_value("printer_model", new ConfigOptionString("Test Model")); + + // Rename the key in the dictionary the reader sees: "layer_height" becomes + // "layer_heighX", a key no build defines. Same length, so the blob's + // offsets are untouched - this is exactly what a removed or renamed option + // looks like to a reader. + const DynamicPrintConfig out = roundtrip_config(in, [](std::string& blob) { + const size_t at = blob.find("layer_height"); + REQUIRE(at != std::string::npos); + blob[at + 11] = 'X'; + }); + + CHECK(out.option("layer_height") == nullptr); + CHECK(out.opt_string("printer_model") == "Test Model"); +} + +TEST_CASE("an option whose type changed is dropped, and the rest still load", "[VendorCache]") +{ + // A payload from a build where layer_height was a coString. This one has it + // as a coFloat, so nothing can be done with the value - but the dictionary + // says how it was written, so its bytes are still consumed and printer_model + // behind it still lands. Hand-written rather than round-tripped: only a + // dictionary this build did not produce can disagree with it. + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + const std::vector keys { "layer_height", "printer_model" }; + const std::vector types { uint16_t(coString), uint16_t(coString) }; + const std::vector enums { std::string() }; // the ENUM_UNNAMED slot + ar(keys, types, enums); + ar(uint32_t(2)); + ar(uint16_t(0)); ar(ConfigOptionString("0.28")); + ar(uint16_t(1)); ar(ConfigOptionString("Test Model")); + } + + std::istringstream is(os.str(), std::ios::binary); + cereal::BinaryInputArchive ar(is); + CacheDictionary rdict; + rdict.load(ar); + DynamicPrintConfig out; + REQUIRE_NOTHROW(load_config(ar, out, rdict)); + CHECK(out.option("layer_height") == nullptr); + CHECK(out.opt_string("printer_model") == "Test Model"); +} + +TEST_CASE("skip_config consumes a config without building one", "[VendorCache]") +{ + DynamicPrintConfig in; + in.set_key_value("layer_height", new ConfigOptionFloat(0.28)); + in.set_key_value("printer_model", new ConfigOptionString("Test Model")); + + CacheDictionary wdict; + wdict.collect(in); + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + wdict.save(ar); + save_config(ar, in, wdict); + ar(std::string("sentinel")); // must still be reachable after the skip + } + + std::istringstream is(os.str(), std::ios::binary); + cereal::BinaryInputArchive ar(is); + CacheDictionary rdict; + rdict.load(ar); + skip_config(ar, rdict); + std::string sentinel; + ar(sentinel); + CHECK(sentinel == "sentinel"); +} + +TEST_CASE("a dictionary index past the end of the table is refused", "[VendorCache]") +{ + DynamicPrintConfig in; + in.set_key_value("layer_height", new ConfigOptionFloat(0.28)); + + CacheDictionary wdict; + wdict.collect(in); + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + wdict.save(ar); + save_config(ar, in, wdict); + } + std::string blob = os.str(); + // The payload's tail is the option count (uint32), the key index (uint16) + // and the double. Point the key index somewhere the table does not go. + const uint16_t bad = 0xFFFE; + std::memcpy(&blob[blob.size() - sizeof(double) - sizeof(uint16_t)], &bad, sizeof(bad)); + + std::istringstream is(blob, std::ios::binary); + cereal::BinaryInputArchive ar(is); + CacheDictionary rdict; + rdict.load(ar); + DynamicPrintConfig out; + REQUIRE_THROWS(load_config(ar, out, rdict)); +} + +TEST_CASE("a stamp string with an absurd length is rejected, not allocated", "[VendorCache]") +{ + // The stamps are read from whatever .opc a directory holds, and a + // string resize to a garbage 64-bit length does not fail as a catchable + // bad_alloc — it takes the app down through the out-of-memory handler. A + // CRC-valid body opening with the right cache version but foreign framing + // where the name's length word sits must be refused before anything is + // allocated. + TempDir tmp; + const std::string cache = (tmp.path / "Evil.opc").string(); + REQUIRE(save_one_vendor(cache, one_vendor("Evil"), "Evil", "1.0.0")); + + // The vendor name's length word sits right behind the payload's version + // word; make it claim a ~9-exabyte name. + const uint64_t huge = 0x7FFFFFFFFFFFFFFFull; + patch_payload_bytes(cache, sizeof(uint32_t), &huge, sizeof(huge)); + + PresetBundle out; + REQUIRE(! out.load_vendor_cache(cache, "Evil", Semver::inf())); + CHECK(out.vendors.empty()); + CHECK(VendorCacheFile::peek_version(cache, "Evil").empty()); +} +