diff --git a/.gitattributes b/.gitattributes index 4cab1f4d26..441bdfe1eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Set the default behavior, in case people don't have core.autocrlf set. * text=auto + +# Shell scripts are run by Git Bash on Windows CI, which cannot read a script +# with CRLF line endings: it fails on the first line. Windows checkouts default +# to core.autocrlf=true, so keep these LF whatever the platform. +*.sh text eol=lf diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index 93071d8041..3a42983ce4 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -162,14 +162,13 @@ jobs: retention-days: 5 if-no-files-found: error - - name: Generate system presets cache (macOS) + - name: Build system preset cache (macOS) if: runner.os == 'macOS' && !inputs.macos-combine-only working-directory: ${{ github.workspace }} shell: bash - run: | - tool=$(find build/${{ inputs.arch }} -name generate_system_cache -type f | head -1) - profiles=$(find build/${{ inputs.arch }} -path "*/Resources/profiles" -type d | head -1) - "$tool" --path "$profiles" --log_level 2 + # The bundle was already packed from resources/, so the caches have to be + # installed into it here; the source tree keeps its JSONs for later jobs. + run: ./scripts/build_preset_cache.sh -b build/${{ inputs.arch }} build/${{ inputs.arch }}/OrcaSlicer/OrcaSlicer.app/Contents/Resources/profiles - name: Pack macOS app bundle ${{ inputs.arch }} if: runner.os == 'macOS' && !inputs.macos-combine-only @@ -399,6 +398,13 @@ jobs: if ($arch -eq "arm64") { .\build_release_vs.bat slicer arm64 tests } else { .\build_release_vs.bat slicer tests } shell: pwsh + - name: Build system preset cache (Windows) + if: runner.os == 'Windows' + shell: cmd + # Shipped into both the already-installed tree (portable zip, MSIX) and + # the checkout cpack re-installs from when it builds the NSIS installer. + run: scripts\build_preset_cache.bat "%BUILD_DIR%" "resources\profiles" "%BUILD_DIR%\OrcaSlicer\resources\profiles" + - name: Pack unit tests Win if: runner.os == 'Windows' working-directory: ${{ github.workspace }} @@ -415,23 +421,6 @@ jobs: retention-days: 5 if-no-files-found: error - - name: Generate system presets cache (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - $buildDir = $env:BUILD_DIR - $tool = Get-ChildItem -Recurse -Path $buildDir -Filter "generate_system_cache.exe" | Select-Object -First 1 - if (-not $tool) { Write-Error "generate_system_cache.exe not found in $buildDir"; exit 1 } - $profiles = Get-ChildItem -Recurse -Path $buildDir -Directory -Filter profiles | - Where-Object { $_.FullName -match 'resources' } | Select-Object -First 1 - if (-not $profiles) { Write-Error "profiles directory not found in $buildDir"; exit 1 } - # Add the slicer's runtime DLL directory to PATH so generate_system_cache.exe - # can resolve its dependencies (TKernel.dll etc.) without a full install step. - $dll_dir = Get-ChildItem -Recurse -Path $buildDir -Filter "TKernel.dll" | - Select-Object -First 1 | Select-Object -ExpandProperty DirectoryName - if ($dll_dir) { $env:PATH = "$dll_dir;$env:PATH" } - & $tool.FullName --path $profiles.FullName --log_level 2 - # NSIS is x86-only; it runs (and the installer it emits runs) under ARM64's # x86 emulation, packaging the native arm64 payload from build-arm64. - name: Create installer Win @@ -565,6 +554,20 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + # Both were packed from resources/ before the caches existed, so the + # AppImage is unpacked first and the caches shipped into it and into + # the package tree; the source tree keeps its JSONs for later steps. + appimage=$(find build -maxdepth 1 -name "OrcaSlicer_Linux_AppImage*.AppImage" | head -1) + chmod +x "$appimage" + "$appimage" --appimage-extract + ./scripts/build_preset_cache.sh -b build build/package/resources/profiles squashfs-root/resources/profiles + appimagetool=$(find build -name "appimagetool.AppImage" | head -1) + ARCH=$(uname -m) "$appimagetool" --appimage-extract-and-run squashfs-root "$appimage" + rm -rf squashfs-root # Ship the freshly-built validator so slice_check_linux (build_all.yml) # can slice-sweep the shipped profiles with this PR's engine. Taken from # the aarch64 leg so the sweep also exercises the arm build; x86_64 on @@ -579,22 +582,6 @@ jobs: retention-days: 5 if-no-files-found: error - - name: Generate system presets cache (Linux) - if: runner.os == 'Linux' - shell: bash - run: | - tool=$(find build -name generate_system_cache -type f | head -1) - if [ -z "$tool" ]; then echo "ERROR: generate_system_cache not found in build tree" >&2; exit 1; fi - "$tool" --path build/package/resources/profiles --log_level 2 - # Re-pack the AppImage so the per-vendor caches are included - appimage=$(find build -maxdepth 1 -name "OrcaSlicer_Linux_AppImage*.AppImage" | head -1) - chmod +x "$appimage" - "$appimage" --appimage-extract - cp build/package/resources/profiles/*.cache squashfs-root/resources/profiles/ - appimagetool=$(find build -name "appimagetool.AppImage" | head -1) - ARCH=$(uname -m) "$appimagetool" --appimage-extract-and-run squashfs-root "$appimage" - rm -rf squashfs-root - - name: Run external slicer regression tests if: runner.os == 'Linux' && inputs.arch != 'aarch64' timeout-minutes: 20 diff --git a/.gitignore b/.gitignore index 0b15bd55c9..4d3ccb5c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,4 @@ internal_docs/ # Python bytecode __pycache__/ *.pyc -*.cache \ No newline at end of file +*.opc diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md new file mode 100644 index 0000000000..7bb71919bc --- /dev/null +++ b/docs/HLSD/preset-cache.md @@ -0,0 +1,288 @@ +# System Preset Cache — High Level Design + +## Why it exists + +OrcaSlicer ships tens of thousands of system preset JSON files. Every launch used to +parse all of them: read each vendor profile, walk its machine, process and filament +sub-files, resolve inheritance, and build the preset collections from scratch. That +parse dominated startup, and it produced the same result every time, because system +presets only change when the app is updated or a profile update is installed. + +The preset cache replaces that parse with a read. Each vendor's presets are serialized +once — at build time, in CI — into a single binary file the app reads in one pass. The +read replaces the file walk and the JSON parsing, which is where the time went; +resolving inheritance and registering the presets still runs at load, through the same +code the JSON path uses, so the result is the parse's result without the parse. + +The cache is **only ever an optimization**. Every rule below exists to guarantee that a +cache is either provably equivalent to parsing the JSONs, or rejected. There is no +"mostly right" cache. + +## The unit is one vendor + +A cache covers exactly one vendor. `BBL.opc` sits beside `BBL.json` and holds +everything `BBL.json` and the `BBL/` sub-file tree would have produced. + +Per-vendor granularity is what makes the system practical: + +- A vendor whose profile is bumped invalidates only its own cache. The other 60-odd + vendors keep theirs — even when the bumped vendor is the shared Orca filament + library everyone else inherits from. +- The setup wizard, which loads vendors one at a time, gets the same speedup as + startup without a second code path. +- A vendor with no cache, or a broken one, costs only that vendor a parse. + +A cache holds *system* presets only. User presets, project settings and modified +presets are never serialized — they have their own storage and their own lifecycle. + +## Where the files live + +| Location | Contents on a shipped build | Role | +|---|---|---| +| `resources/profiles/` | `.opc` alone — the profile and its preset JSONs both pruned | What the app ships with; the fallback everything falls back to | +| `/system/` | `.opc` alone, or `.json` + `/` after an update | What the user has installed | +| `/system/` (dev build) | `.json` + `/` + `.opc` written at runtime | A developer tree caches as it parses | + +Two forms of the same vendor therefore exist, and the system's central rule is that +**a vendor's cache is the whole of it**. Where a cache ships or is installed, no profile +and no preset JSONs sit beside it: the cache carries the presets, the vendor profile, +and the version stamp that says which release it came from. A vendor is "installed" if +either form is present, and its installed version is read from whichever form is there. + +What stays beside the caches in `resources/profiles/` is everything that is not a +preset: each vendor's directory of printer thumbnails, cover images, bed models and +hotend meshes, which are read from disk by path and were never part of the cache. Files +that are not vendors at all, `blacklist.json` chief among them, are untouched. + +The alternative — shipping both and treating the cache as a sidecar — was rejected. It +doubles the installed size, and it creates a class of bug where the two disagree and +the app's behavior depends on which one a given code path happened to read. + +## What a cache file is + +A fixed-size header followed by one binary stream. + +The header carries a magic number, the cache format version, the payload size and a +CRC32 of the payload. It exists so that a truncated download, a half-written file or a +file from an entirely different program is rejected in microseconds, before anything +tries to interpret it. + +The payload opens with the stamps that decide whether the cache may be used at all — +format version, schema fingerprint, vendor name, vendor version — and then the +vendor's data: its vendor profile, three lists of preset entries (process, filament, +machine), and the count of errors the original parse hit. + +Each entry is one preset **in source form**: what its JSON sub-file states and nothing +that resolving it derives — the preset's own config diff, the name of the preset it +inherits, and the parse metadata (name, sub-path, description, instantiation, setting +and filament ids, renames). Non-instantiated base presets are stored too; the children +that inherit from them cannot resolve without them. + +Three deliberate choices in the layout: + +- **Stamps come first**, so the question "what version is this vendor installed at?" + can be answered by reading the first kilobyte. The updater asks that question for + every vendor on every launch; reading tens of megabytes to answer it would give back + the startup time the cache saved. +- **Nothing inherited is baked in.** A filament preset that inherits from the shared + library is stored as its own diff plus its parent's name, and the parent is looked up + when the entry is installed, against whatever library is loaded then. A cache + therefore carries no other vendor's values, and no other vendor's update — the + library's included — can make it stale. +- **Nothing derived is stored.** Default presets, flattened configs, aliases and + lookup maps are all reconstructed at load by the same code the JSON path runs, and + state that path never fills (obsolete-preset lists) is not stored either. This keeps + the cache a record of the vendor's data, not a memory image of the program's state. + +## When a cache may be used + +A cache is accepted only if every gate below passes. Any failure means "parse the +JSONs instead" — never a hard error, never a partial load. + +**1. Integrity.** Magic number, plausible size, CRC32 over the payload. + +**2. Cache format version.** A single integer bumped by hand whenever the binary layout +changes in a way nothing else would catch: reordering or retyping a hand-written +serialized field, or changing what the cache's own stamps mean. + +**3. Schema fingerprint.** A checksum over the app version and the entire print-config +option schema — every option's key, type, wire ordinal and enum values. This is the +gate that makes the cache safe across development: adding a config option, changing its +type, or reordering the enum values of an existing one all change the fingerprint, so +caches from before the change are rejected without anyone having to remember to bump +anything. It also means a cache never crosses app versions. + +**4. Vendor identity and version.** The cache names the vendor it holds and the profile +version it was built from. It is accepted only if that version is at least as new as +the profile now on disk. Where no profile sits beside the cache — the shipped, +cache-only form — the comparison is skipped, because nothing on disk can be newer than +a cache that is the installation. + +**5. Every entry installs.** Entries are installed as they are read, and an entry that +cannot be — typically one that inherits a parent the currently loaded filament library +no longer provides — rejects the whole cache, never just the entry. A partial vendor is +not a vendor. + +There is deliberately no stamp for the shared filament library. A cache stores its +filaments' inheritance by name and resolves it at load, so a library update changes +what a cache load *produces*, never whether the cache is *valid* — the same file yields +the updated result. This matters most on a shipped build, where a vendor is its cache +and nothing else: a profile update that delivered only the library would otherwise have +stranded every other vendor with a cache it invalidated and no JSONs to fall back on. + +A vendor profile with no parsable version is never cached and never served from a +cache. There would be no way to tell later whether the cache had gone stale, and a +cache nothing can invalidate is worse than no cache. + +## How a vendor is loaded + +Vendors load in a fixed order, because filament inheritance crosses exactly one +boundary: any vendor's filament may inherit from the shared Orca filament library, +and nothing else reaches across vendors. The library therefore goes first, alone; +every other vendor follows in parallel, resolving against it; and the results are +merged in a stable order: + +```mermaid +flowchart LR + lib["1 · OrcaFilamentLibrary
loaded first, synchronously"] --> par["2 · every other vendor in parallel,
each into its own bundle, filaments
resolving against the loaded library"] --> merge["3 · bundles merged into one,
sequentially, in stable vendor order"] +``` + +Whether a vendor comes from its cache or from a parse changes nothing in that +order — both produce the same bundle, so cached and parsed vendors mix freely in +one startup. Each vendor load — startup's and the setup wizard's alike — tries, +in order: + +1. The cache in the directory it was asked to load from — normally `/system/`. +2. The shipped cache in `resources/profiles/` — judged against the same installed + profile, so it cannot resurrect a version an update has superseded. +3. Parsing the JSONs — from the data directory if the profile is installed there, and + from `resources/profiles/` otherwise, which on a shipped build only has JSONs for a + vendor that has no cache. + +The same decision drawn out — "the gates" are the five acceptance checks above: + +```mermaid +flowchart TB + start["load vendor V from a directory dir
— normally <data_dir>/system/"] + start --> stamp["installed version = version of dir/V.json
— or ∞ with no profile there,
the cache then being the installation"] + stamp --> g1{"dir/V.opc
passes all five gates?"} + g1 -- "yes" --> hit(["served from the
installed cache"]) + g1 -- "no" --> g2{"resources/profiles/V.opc passes them,
against the same installed version?"} + g2 -- "yes" --> hit2(["served from the
shipped cache"]) + g2 -- "no" --> src{"dir/V.json
present?"} + src -- "yes" --> pd["parse the JSONs in dir"] + src -- "no" --> pr["parse the JSONs in
resources/profiles/"] + pd --> ver{"profile version
parsable?"} + pr --> ver + ver -- "yes" --> save(["loaded; dir/V.opc written back —
the next load takes the top path"]) + ver -- "no" --> raw(["loaded, never cached"]) +``` + +Serving from a cache is not a memory-image restore. The entries are deserialized and +then installed one by one — inheritance resolved against the presets installed before +them and the currently loaded filament library, configs flattened onto the collection +defaults, validated and registered — by the same function the JSON path calls straight +after parsing a sub-file. The two paths share everything below the parse, which is what +makes a cache-loaded bundle indistinguishable from a JSON-loaded one by construction +rather than by test coverage. Installation also rebuilds each preset's file path from +the local data directory, so a shipped cache never carries the generating machine's +paths. + +The second tier is what makes app upgrades work. After an upgrade, a cache the previous +version installed fails the fingerprint gate; the new build's own shipped cache answers +instead, and the user never sees a parse. The stale installed file is simply ignored +until the next profile update overwrites it. + +If a parse does happen and the vendor's profile carries a version, the app writes the +cache back beside where it looked for the vendor. That is how a developer build warms +itself up on second launch, and how a vendor delivered by a profile update becomes +cached without waiting for the next release. + +## How a vendor is installed + +Installing copies from `resources/profiles/` into `/system/`. A shipped build +offers only a cache and a source tree only JSONs, but a partially-generated tree can +have both, at different versions, so the installer picks the form that ships at the +**newer version** and installs only that one: + +- Cache newer or equal, and readable → copy the `.opc`, and delete any profile and + vendor directory a previous install left behind, so nothing can shadow it. +- Profile newer, or the cache unreadable or absent → copy the profile and the vendor's + preset JSONs exactly as the app did before caches existed, and delete any stale `.opc`. + +The result is that only one form of a vendor is ever present, and it is the newest one +the build has. This matters most for the update check, which compares what is installed +against what installing *would* lay down: if those two disagreed about which form +counts, a vendor could reinstall on every launch forever, or silently never update. + +Profile updates delivered over the air always arrive as JSONs, and they win — an +updated vendor's real profile lands in the data directory, the shipped cache is older +and gets rejected, and the vendor is parsed and re-cached. An update that touches only +the filament library needs nothing more: every other vendor's cache stays valid and +simply resolves against the new library on its next load. + +## How the caches are produced + +Cache generation is a build step, not something a user ever runs. + +One script per platform does the whole job, and CI calls it once on each. It builds a +small dev-utility that loads a profiles directory exactly as the app would, with cache +writing enabled, dropping a `.opc` beside every vendor profile it parses; then +it copies those caches into each packaged application it was pointed at and deletes +every preset JSON they replace — the vendor's own profile included. Only a vendor that +actually has a cache is pruned, so a vendor the generator skipped keeps its JSONs and is +simply parsed at startup. + +Because the schema fingerprint includes the app version, caches must be generated by +the same build that ships them. Generation runs after the build, in the same job. + +## Behavior when things go wrong + +The system is designed so that no cache problem is fatal: + +- **Corrupt, truncated or foreign file** — rejected at the header, vendor parsed. +- **Cache from another app version or schema** — rejected at the fingerprint, vendor + parsed or served from the shipped cache. +- **Stale cache** — rejected on the vendor version stamp, vendor parsed and re-cached. +- **Failure part-way through loading** — a deserialization error, or any entry that + fails to install — rejects the whole cache, and the bundle is reset to a clean state + before falling back, so a half-loaded cache can never leak into the parsed result. +- **A vendor that can be neither read nor parsed** — logged, and left out. The setup + wizard drops that vendor from its list and opens with the rest; startup records the + error alongside the vendors that did load. One broken vendor never takes the app down. + +The one genuine limit: on a shipped build a vendor is its cache and nothing else, so a +rejected cache has nothing to fall back to for that vendor. This is by design — the +alternative is shipping every preset twice — and it is why the acceptance gates are +conservative and why CI generates the caches with the same build that ships them. The +recovery path is a profile update, which delivers real JSONs. + +It also means nothing may quietly assume a `.json` exists. Discovery, version +checks and the update decision all read whichever form is present, and a code path that +enumerates only `*.json` will find no vendors at all in a packaged build. + +## Maintenance rules + +- **Adding or changing a config option** needs nothing. The fingerprint covers it. +- **Changing a hand-written `serialize()`** — `PresetBundle::CachedPreset`, + `VendorProfile` or its nested types — or the cache's own layout or stamps requires + bumping the cache format version by hand. Nothing else is serialized by hand; the + config payload is covered by the fingerprint. +- **Bumping a vendor profile's version** invalidates that vendor's cache and nothing + else — the filament library's included. Other vendors' caches resolve against the + new library the next time they load. +- **Caches are never committed.** They are build artifacts, generated per build, + ignored by git. + +## Where this lives in the tree + +| Area | Files | +|---|---| +| Cache format, entry serialization, read/write, load and save | `src/libslic3r/PresetBundle.{hpp,cpp}` | +| Vendor profile serialization | `src/libslic3r/Preset.hpp` | +| Vendor discovery, installed/shipped versions, installation | `src/libslic3r/PresetBundle.cpp` | +| Update and reinstall decisions | `src/slic3r/Utils/PresetUpdater.cpp` | +| Setup wizard and printer-selection dialog | `src/slic3r/GUI/ConfigWizard.cpp`, `src/slic3r/GUI/WebGuideDialog.cpp` | +| Generator tool | `src/dev-utils/generate_system_cache.cpp` | +| Build and packaging script | `scripts/build_preset_cache.{sh,bat}` | +| Tests | `tests/libslic3r/test_vendor_cache.cpp` | diff --git a/scripts/build_preset_cache.bat b/scripts/build_preset_cache.bat new file mode 100644 index 0000000000..ac839b7c49 --- /dev/null +++ b/scripts/build_preset_cache.bat @@ -0,0 +1,122 @@ +@echo off +rem Build the per-vendor system preset caches (one .opc per vendor) by +rem running the generate_system_cache.exe dev tool against a profiles directory, +rem and make every profiles directory named on the command line ship-ready: +rem install the caches into it and delete the preset JSONs they replace, so a +rem build ships one copy of its presets instead of two. +rem +rem scripts\build_preset_cache.bat [build_dir] [target_dir ...] +rem +rem build_dir defaults to "build" +rem target_dir profiles directories to ship into. Caches are generated into +rem the source tree's resources\profiles, which is what every +rem packaging step copies from; a target may be that same +rem directory, which then only gets pruned. +rem +rem Shipping deletes, so it is a CI packaging step. A vendor's own .json +rem goes along with its preset JSONs: the cache carries the vendor profile and +rem the version it was built at, so discovery, version checks and installing all +rem read it there. Only a vendor that has a cache is pruned, so non-vendor JSONs +rem (blacklist.json) are left alone, as are the vendor directories themselves - +rem thumbnails, covers and bed models still live there. +rem +rem set CONFIG= to pin the build config for multi-config generators +rem (default: the config of the tool already in the build tree, else Release) +setlocal enabledelayedexpansion + +set "REPO_ROOT=%~dp0.." +set "BUILD_DIR=%~1" +if "%BUILD_DIR%"=="" set "BUILD_DIR=build" +if not exist "%BUILD_DIR%\" ( + echo ERROR: build tree not found: %BUILD_DIR% 1>&2 + exit /b 1 +) +if not "%~1"=="" shift + +rem Newest match wins: a stale binary silently produces a stale cache layout. +call :find_tool +if not defined CONFIG ( + for %%c in (Debug Release RelWithDebInfo MinSizeRel) do ( + echo !TOOL! | findstr /i "\\%%c\\" >nul && set "CONFIG=%%c" + ) +) +if not defined CONFIG set "CONFIG=Release" + +echo Building generate_system_cache in %BUILD_DIR% (%CONFIG%) +cmake --build "%BUILD_DIR%" --config %CONFIG% --target generate_system_cache +if errorlevel 1 ( + echo ERROR: could not build generate_system_cache - configure the build tree with -DORCA_TOOLS=ON: 1>&2 + echo cmake -S "%REPO_ROOT%" -B "%BUILD_DIR%" -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) +call :find_tool +if not defined TOOL ( + echo ERROR: generate_system_cache.exe not found under %BUILD_DIR% - build with -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) + +set "PROFILES=%REPO_ROOT%\resources\profiles" +if not exist "%PROFILES%\" ( + echo ERROR: profiles directory not found: %PROFILES% 1>&2 + exit /b 1 +) +for %%d in ("%PROFILES%") do set "PROFILES=%%~fd" + +rem Add the slicer's runtime DLL directory to PATH so generate_system_cache.exe +rem can resolve its dependencies (TKernel.dll etc.) without a full install step. +set "DLL_DIR=" +for /f "delims=" %%f in ('dir /s /b "%BUILD_DIR%\TKernel.dll" 2^>nul') do ( + if not defined DLL_DIR set "DLL_DIR=%%~dpf" +) +if defined DLL_DIR set "PATH=%DLL_DIR%;%PATH%" + +echo Generating per-vendor preset caches in %PROFILES% +rem Start clean so vendors that went away - and caches written by older tool +rem versions - don't linger next to the freshly generated ones. +del /q "%PROFILES%\*.opc" 2>nul +del /q "%PROFILES%\*.cache" 2>nul +"%TOOL%" --path "%PROFILES%" --log_level 2 +if errorlevel 1 exit /b %errorlevel% + +:next_target +if "%~1"=="" exit /b 0 +call :ship "%~1" +if errorlevel 1 exit /b 1 +shift +goto :next_target + +:ship +set "TARGET=%~1" +if not exist "%TARGET%\" ( + echo ERROR: profiles directory not found: %TARGET% 1>&2 + exit /b 1 +) +for %%d in ("%TARGET%") do set "TARGET=%%~fd" +if /i not "%TARGET%"=="%PROFILES%" copy /y "%PROFILES%\*.opc" "%TARGET%\" >nul + +set /a SHIPPED=0 +set /a PRUNED=0 +for %%c in ("%PROFILES%\*.opc") do ( + set /a SHIPPED+=1 + set "VENDOR=%%~nc" + if exist "%TARGET%\!VENDOR!.json" ( + del /q "%TARGET%\!VENDOR!.json" + set /a PRUNED+=1 + ) + if exist "%TARGET%\!VENDOR!\" ( + for /f %%n in ('dir /s /b "%TARGET%\!VENDOR!\*.json" 2^>nul ^| find /c /v ""') do set /a PRUNED+=%%n + del /s /q "%TARGET%\!VENDOR!\*.json" >nul 2>&1 + rem Deepest first, so a directory the delete above emptied goes too; rd + rem refuses the ones still holding covers or meshes. + for /f "delims=" %%d in ('dir /s /b /ad "%TARGET%\!VENDOR!" 2^>nul ^| sort /r') do rd "%%d" 2>nul + ) +) +echo %TARGET%: !SHIPPED! caches, dropped !PRUNED! preset JSONs +exit /b 0 + +:find_tool +set "TOOL=" +for /f "delims=" %%f in ('dir /s /b /o-d "%BUILD_DIR%\generate_system_cache.exe" 2^>nul') do ( + if not defined TOOL set "TOOL=%%f" +) +exit /b 0 diff --git a/scripts/build_preset_cache.sh b/scripts/build_preset_cache.sh new file mode 100755 index 0000000000..33a436f151 --- /dev/null +++ b/scripts/build_preset_cache.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Build the per-vendor system preset caches (one .opc per vendor) by +# running the generate_system_cache dev tool against a profiles directory, and +# make every profiles directory named on the command line ship-ready: install +# the caches into it and delete the preset JSONs they replace, so a build ships +# one copy of its presets instead of two. +# +# ./scripts/build_preset_cache.sh # caches into resources/profiles +# ./scripts/build_preset_cache.sh -b build/arm64 # search this build tree for the tool +# ./scripts/build_preset_cache.sh [ ...] # and ship into these profiles dirs +# +# Caches are generated into the source tree's resources/profiles, which is what +# every packaging step copies from. Shipping deletes, so it is a CI packaging +# step: pass packaged output directories, or the checkout of a build that is +# about to be packaged from it. +# +# A vendor's own .json goes along with its preset JSONs: the cache +# carries the vendor profile and the version it was built at, so discovery, +# version checks and installing all read it there. A shipped vendor is its cache +# and nothing else. Only a vendor that has a cache is pruned, so an ungenerated +# vendor keeps its JSONs and is simply parsed at startup; non-vendor JSONs +# (blacklist.json) are left alone, as are the vendor directories themselves — +# thumbnails, covers and bed models still live there. +# +# -b build tree holding the tool +# (default: build/arm64, build/x86_64, or build — first that exists) +# -p profiles directory to generate caches into +# (default: /resources/profiles) +# -c build config for multi-config generators +# (default: the config of the tool already in the build tree, else +# the build tree's CMAKE_BUILD_TYPE) +# -n skip the rebuild and run the tool already in the build tree +# -l tool log level (default: 2) +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd -P)" +build_dir="" +profiles_dir="" +config="" +build_tool=1 +log_level=2 + +while getopts "b:p:c:l:nh" opt; do + case $opt in + b) build_dir="$OPTARG" ;; + p) profiles_dir="$OPTARG" ;; + c) config="$OPTARG" ;; + n) build_tool=0 ;; + l) log_level="$OPTARG" ;; + h) sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "$build_dir" ]; then + for candidate in "$repo_root/build/arm64" "$repo_root/build/x86_64" "$repo_root/build"; do + if [ -d "$candidate" ]; then build_dir="$candidate"; break; fi + done +fi +if [ -z "$build_dir" ] || [ ! -d "$build_dir" ]; then + echo "ERROR: build tree not found (pass -b )" >&2 + exit 1 +fi + +# Newest match wins: multi-config trees keep one binary per config, and a stale +# one silently produces a stale cache layout. +find_tool() { + local best="" f + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ -z "$best" ] || [ "$f" -nt "$best" ]; then best="$f"; fi + done < <(find "$build_dir" -name generate_system_cache -type f 2>/dev/null) + printf '%s' "$best" +} + +tool=$(find_tool) +if [ -z "$config" ]; then + case "$tool" in + */Debug/*) config=Debug ;; + */Release/*) config=Release ;; + */RelWithDebInfo/*) config=RelWithDebInfo ;; + */MinSizeRel/*) config=MinSizeRel ;; + *) config=$(sed -n 's/^CMAKE_BUILD_TYPE:[A-Z]*=\(.\+\)$/\1/p' "$build_dir/CMakeCache.txt" 2>/dev/null | head -1 || true) ;; + esac +fi + +if [ "$build_tool" = 1 ]; then + echo "Building generate_system_cache in $build_dir${config:+ ($config)}" + build_args=(--build "$build_dir" --target generate_system_cache) + if [ -n "$config" ]; then build_args+=(--config "$config"); fi + if ! cmake "${build_args[@]}"; then + echo "ERROR: could not build generate_system_cache — configure the build tree with -DORCA_TOOLS=ON:" >&2 + echo " cmake -S \"$repo_root\" -B \"$build_dir\" -DORCA_TOOLS=ON" >&2 + exit 1 + fi + tool=$(find_tool) +fi + +if [ -z "$tool" ]; then + echo "ERROR: generate_system_cache not found under $build_dir — build with -DORCA_TOOLS=ON" >&2 + exit 1 +fi + +if [ -z "$profiles_dir" ]; then profiles_dir="$repo_root/resources/profiles"; fi +if [ ! -d "$profiles_dir" ]; then + echo "ERROR: profiles directory not found: $profiles_dir" >&2 + exit 1 +fi +profiles_dir=$(cd "$profiles_dir" && pwd -P) + +# Start clean so vendors that went away — and caches written by older tool +# versions — don't linger next to the freshly generated ones. +echo "Generating per-vendor preset caches in $profiles_dir" +rm -f "$profiles_dir"/*.opc "$profiles_dir"/*.cache +"$tool" --path "$profiles_dir" --log_level "$log_level" + +for target in "$@"; do + resolved=$(cd "$target" 2>/dev/null && pwd -P) || { + echo "ERROR: profiles directory not found: $target" >&2 + exit 1 + } + if [ "$resolved" != "$profiles_dir" ]; then + cp "$profiles_dir"/*.opc "$resolved"/ + fi + + pruned=0 + shipped=0 + for cache in "$profiles_dir"/*.opc; do + vendor=$(basename "$cache" .opc) + shipped=$(( shipped + 1 )) + if [ -f "$resolved/$vendor.json" ]; then + rm -f "$resolved/$vendor.json" + pruned=$(( pruned + 1 )) + fi + [ -d "$resolved/$vendor" ] || continue + n=$(find "$resolved/$vendor" -name '*.json' | wc -l) + find "$resolved/$vendor" -name '*.json' -delete + find "$resolved/$vendor" -type d -empty -delete + pruned=$(( pruned + n )) + done + echo "$resolved: $shipped caches, dropped $pruned preset JSONs" +done diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b174144c5b..79b49cfd16 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -115,8 +115,6 @@ if(ORCA_TOOLS) endif() target_link_libraries(OrcaSlicer_profile_validator libslic3r boost_headeronly libcurl OpenSSL::SSL OpenSSL::Crypto) target_compile_definitions(OrcaSlicer_profile_validator PRIVATE -DBOOST_ALL_NO_LIB -DBOOST_USE_WINAPI_VERSION=0x602 -DBOOST_SYSTEM_USE_UTF8) - - endif() # Create a slic3r executable diff --git a/src/dev-utils/CMakeLists.txt b/src/dev-utils/CMakeLists.txt index 6b43cff319..2cfce6a7c5 100644 --- a/src/dev-utils/CMakeLists.txt +++ b/src/dev-utils/CMakeLists.txt @@ -23,7 +23,7 @@ endif() if (ORCA_TOOLS) set(_DEV_DEFS -DBOOST_ALL_NO_LIB -DBOOST_USE_WINAPI_VERSION=0x602 -DBOOST_SYSTEM_USE_UTF8) - # generate_system_cache: pre-generates per-vendor resources/profiles/.cache files for CI bundling. + # generate_system_cache: pre-generates per-vendor .opc files under resources/profiles for CI bundling. add_executable(generate_system_cache generate_system_cache.cpp) target_link_libraries(generate_system_cache libslic3r boost_headeronly) target_compile_definitions(generate_system_cache PRIVATE ${_DEV_DEFS}) diff --git a/src/dev-utils/generate_system_cache.cpp b/src/dev-utils/generate_system_cache.cpp index e86a920c7b..426ccee997 100644 --- a/src/dev-utils/generate_system_cache.cpp +++ b/src/dev-utils/generate_system_cache.cpp @@ -2,10 +2,10 @@ #include "libslic3r/Preset.hpp" #include "libslic3r/Utils.hpp" +#include #include #include #include -#include #include using namespace Slic3r; @@ -58,31 +58,27 @@ int main(int argc, char* argv[]) auto preset_bundle = std::make_unique(); preset_bundle->set_is_validation_mode(true); preset_bundle->set_default_suppressed(true); + preset_bundle->set_generate_vendor_caches(true); std::cout << "Loading system presets from: " << profiles_path << "\n"; try { + // In validation mode data_dir() is the profiles directory set above, so the + // loader writes each .opc next to its .json as it parses it. preset_bundle->load_presets(app_config, ForwardCompatibilitySubstitutionRule::EnableSilent); } catch (const std::exception& ex) { std::cerr << "Failed to load presets: " << ex.what() << "\n"; return 1; } - const std::string output_path = - (fs::path(profiles_path) / "system_presets.cache").make_preferred().string(); - - std::cout << "Saving single-bundle cache to: " << output_path << "\n"; - - const auto stats = preset_bundle->save_system_presets_cache(profiles_path, output_path); - - if (!stats.ok) { - std::cerr << "ERROR: verification failed\n"; + size_t cache_count = 0; + for (auto& entry : fs::directory_iterator(profiles_path)) + if (boost::iends_with(entry.path().string(), ".opc")) + ++ cache_count; + if (cache_count == 0) { + std::cerr << "No vendor cache files were generated under " << profiles_path << "\n"; return 1; } - - std::cout << "[ok] system_presets.cache\n" - << " Total print presets: " << stats.print_presets << "\n" - << " Total filament presets: " << stats.filament_presets << "\n" - << " Total printer presets: " << stats.printer_presets << "\n"; + std::cout << "Generated " << cache_count << " vendor cache file(s) under " << profiles_path << "\n"; return 0; } diff --git a/src/libslic3r/Config.hpp b/src/libslic3r/Config.hpp index 509095cbfc..6e25a4d931 100644 --- a/src/libslic3r/Config.hpp +++ b/src/libslic3r/Config.hpp @@ -28,6 +28,9 @@ #include #include +// The serialize() members below archive ConfigOption hierarchies through +// cereal::base_class, whose registration machinery lives in polymorphic.hpp. +#include namespace Slic3r { struct FloatOrPercent diff --git a/src/libslic3r/Preset.cpp b/src/libslic3r/Preset.cpp index 393f80e0ee..b07c5f0d5d 100644 --- a/src/libslic3r/Preset.cpp +++ b/src/libslic3r/Preset.cpp @@ -152,17 +152,6 @@ Semver get_version_from_json(std::string file_path) } } -std::string get_vendor_cache_key(const std::string& json_path) -{ - const Semver ver = get_version_from_json(json_path); - if (ver.valid()) - return ver.to_string(); - // No version field — use mtime as change fingerprint so edits invalidate the cache. - boost::system::error_code ec; - const std::time_t mtime = boost::filesystem::last_write_time(json_path, ec); - return ec ? std::string{} : ("mtime:" + std::to_string(mtime)); -} - //BBS: add a function to load the key-values from xxx.json int get_values_from_json(std::string file_path, std::vector& keys, std::map& key_values) { @@ -275,18 +264,28 @@ void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil } }; + // The four variant sets are immutable after static init and probed for every + // key of every preset loaded; one merged map makes that a single lookup. + // emplace keeps the first insertion, preserving the first-set-wins priority + // of the else-if chain this replaces. + static const std::unordered_map variant_class = [] { + std::unordered_map m; + for (const std::string& k : print_options_with_variant) m.emplace(k, 0); + for (const std::string& k : filament_options_with_variant) m.emplace(k, 1); + for (const std::string& k : printer_options_with_variant_1) m.emplace(k, 2); + for (const std::string& k : printer_options_with_variant_2) m.emplace(k, 3); + return m; + }(); + for(auto& key :config.keys()){ - if(auto iter = print_options_with_variant.find(key); iter != print_options_with_variant.end()){ - replace_nil_and_resize(key, process_variant_length); - } - else if(auto iter = filament_options_with_variant.find(key); iter != filament_options_with_variant.end()){ - replace_nil_and_resize(key, filament_variant_length); - } - else if(auto iter = printer_options_with_variant_1.find(key); iter != printer_options_with_variant_1.end()){ - replace_nil_and_resize(key, machine_variant_length); - } - else if(auto iter = printer_options_with_variant_2.find(key); iter != printer_options_with_variant_2.end()){ - replace_nil_and_resize(key, machine_variant_length * 2); + auto iter = variant_class.find(key); + if (iter == variant_class.end()) + continue; + switch (iter->second) { + case 0: replace_nil_and_resize(key, process_variant_length); break; + case 1: replace_nil_and_resize(key, filament_variant_length); break; + case 2: replace_nil_and_resize(key, machine_variant_length); break; + case 3: replace_nil_and_resize(key, machine_variant_length * 2); break; } } } @@ -758,7 +757,6 @@ void Preset::save(DynamicPrintConfig* parent_config) idx_file.replace_extension(".info"); this->save_info(idx_file.string()); } - } void Preset::reload(Preset const &parent) diff --git a/src/libslic3r/Preset.hpp b/src/libslic3r/Preset.hpp index 53604b3fea..c9b3197a6f 100644 --- a/src/libslic3r/Preset.hpp +++ b/src/libslic3r/Preset.hpp @@ -16,14 +16,6 @@ #include "Semver.hpp" #include "ProjectTask.hpp" -#include -#include -#include -#include -#include -#include -#include - //BBS: change system directories #define PRESET_SYSTEM_DIR "system" #define PRESET_USER_DIR "user" @@ -122,10 +114,6 @@ extern Semver get_version_from_json(std::string file_path); //BBS: add a function to load the key-values from xxx.json extern int get_values_from_json(std::string file_path, std::vector& keys, std::map& key_values); -// Returns the cache key for a vendor JSON: the Semver string for versioned -// vendors, or "mtime:" for vendors without a version field. -extern std::string get_vendor_cache_key(const std::string& json_path); - extern ConfigFileType guess_config_file_type(const boost::property_tree::ptree &tree); extern void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil_to_default, const DynamicPrintConfig& defaults); @@ -144,8 +132,9 @@ public: PrinterVariant(const std::string &name) : name(name) {} std::string name; + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. template - void serialize(Archive& ar) { ar(name); } + void serialize(Archive& ar) { ar(name); } // PrinterVariant }; struct PrinterModel { @@ -154,7 +143,7 @@ public: std::string name; //BBS: this is internal id for the printer. Currently only used for searching in database std::string model_id; - PrinterTechnology technology; + PrinterTechnology technology = ptFFF; std::string family; std::vector variants; std::vector default_materials; @@ -178,13 +167,15 @@ public: const PrinterVariant* variant(const std::string &name) const { return const_cast(this)->variant(name); } + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. template - void serialize(Archive& ar) + void serialize(Archive& ar) // PrinterModel { - ar(id, name, model_id, family, technology, variants, default_materials, + ar(id, name, model_id, technology, family, variants, default_materials, not_support_bed_types, bed_model, bed_texture, image_bed_type, bottom_texture_end_name, use_double_extruder_default_texture, - bottom_texture_rect, middle_texture_rect, hotend_model); + bottom_texture_rect, bottom_texture_rect_longer, middle_texture_rect, + hotend_model); } }; std::vector models; @@ -197,10 +188,11 @@ public: bool valid() const { return ! name.empty() && ! id.empty() && config_version.valid(); } + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. template - void serialize(Archive& ar) + void serialize(Archive& ar) // VendorProfile { - ar(id, name, config_version, config_update_url, changelog_url, + ar(name, id, config_version, config_update_url, changelog_url, models, default_filaments, default_sla_materials); } @@ -456,31 +448,12 @@ public: // BBS: move constructor to public Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {} - // Default constructor is public so cereal can default-construct elements when - // deserializing std::vector (std::allocator is not a cereal::access friend). - Preset() = default; protected: friend class PresetCollection; friend class PresetBundle; - friend class cereal::access; - // Serializes all value fields of Preset for the binary vendor cache. - // Raw pointers (vendor, loading_substitutions) are excluded — vendor is reconstructed - // by apply_vendor_cache() from the VendorProfile stored alongside the presets. - template - void serialize(Archive& ar) - { - ar(type, name, alias, file, version, - filament_id, setting_id, description, - renamed_from, is_system, is_visible, - is_default, is_external, is_dirty, is_compatible, - is_project_embedded, loaded, - m_from_orca_filament_lib, m_excluded_from, - bundle_id, user_id, base_id, sync_info, - updated_time, key_values, ini_str, - config); - } + Preset() = default; }; bool is_compatible_with_print (const PresetWithVendorProfile &preset, const PresetWithVendorProfile &active_print, const PresetWithVendorProfile &active_printer); diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index ab6e44e676..073627e745 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -6,8 +6,11 @@ #include "PresetBundle.hpp" #include +#include +#include #include #include +#include #include #include #include "PrintConfig.hpp" @@ -1233,13 +1236,10 @@ bool PresetBundle::apply_vendor_config( : std::map(); // Find vendors that need installation - const auto vendor_dir = (fs::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - std::vector install_bundles; for (const auto &it : new_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir / (it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -2248,6 +2248,166 @@ void PresetBundle::remove_users_preset(AppConfig &config, std::map vendor_names_in(const boost::filesystem::path& dir) +{ + std::set names; + for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { + const auto& path = dir_entry.path(); + if (Slic3r::is_json_file(path.string()) || path.extension() == ".opc") + names.insert(path.stem().string()); + } + return names; +} + +// A vendor's preset cache is the whole of its installation: it carries the presets, +// the vendor profile and the version they were built at, so where one ships nothing +// else needs copying. Unless the profile beside it claims a newer version — a cache +// generated before that profile was bumped is out of date, and a cache that cannot +// be read is no installation at all — and the vendor is installed the way it was +// before caches existed, as its profile and the preset JSONs it points at. Returns +// the version the cache is stamped with, invalid when it is not the form to install. +static Semver installable_cache_version(const boost::filesystem::path& dir, const std::string& vendor) +{ + const auto cache_ver = Semver::parse(PresetBundle::peek_vendor_cache_version((dir / (vendor + ".opc")).string(), vendor)); + if (! cache_ver) + return Semver::invalid(); + const Semver profile_ver = get_version_from_json((dir / (vendor + ".json")).string()); + return profile_ver.valid() && *cache_ver < profile_ver ? Semver::invalid() : *cache_ver; +} + +Semver resource_vendor_version(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(resources_dir()) / "profiles"; + const Semver ver = installable_cache_version(dir, vendor); + return ver.valid() ? ver : get_version_from_json((dir / (vendor + ".json")).string()); +} + +bool install_vendor_bundles_from_resources( + const std::vector& bundle_names, + const std::string& resource_subdir, + const std::string& data_subdir) +{ + namespace fs = boost::filesystem; + + fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; + fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; + + BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; + + for (const auto &bundle : bundle_names) { + try { + // Install the JSON file + auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); + auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); + auto cache_in_rsrc = (rsrc_path / bundle).replace_extension(".opc"); + auto cache_in_vendors = (vendor_path / bundle).replace_extension(".opc"); + + // Either form of the vendor will do: a build may ship it as a cache alone. + if (!fs::exists(path_in_rsrc) && !fs::exists(cache_in_rsrc)) { + BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; + return false; + } + + // Create target directory if needed + if (!fs::exists(vendor_path)) + fs::create_directories(vendor_path); + + std::string error_message; + bool installed_cache = false; + if (installable_cache_version(rsrc_path, bundle).valid()) { + installed_cache = copy_file(cache_in_rsrc.string(), cache_in_vendors.string(), error_message, false) == CopyFileResult::SUCCESS; + if (! installed_cache) + BOOST_LOG_TRIVIAL(warning) << "Failed to copy " << bundle << ".opc: " << error_message; + } else { + boost::system::error_code ec; + fs::remove(cache_in_vendors, ec); + } + + if (! installed_cache) { + CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); + if (cfr != CopyFileResult::SUCCESS) { + BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; + return false; + } + } else { + // Left in place, an earlier install's profile would shadow the cache. + boost::system::error_code ec; + fs::remove(path_in_vendors, ec); + } + + // Copy the vendor directory (if it exists) + auto dir_in_rsrc = rsrc_path / bundle; + auto dir_in_vendors = vendor_path / bundle; + + // Whatever is installed came from an earlier version of this vendor and + // would be parsed in place of the one being installed now. + if (fs::exists(dir_in_vendors)) + fs::remove_all(dir_in_vendors); + + if (! installed_cache && fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { + fs::create_directories(dir_in_vendors); + + // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) + // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf + auto file_filter = [](const std::string name) -> bool { + return boost::iends_with(name, ".stl") || + boost::iends_with(name, ".png") || + boost::iends_with(name, ".svg") || + boost::iends_with(name, ".jpeg") || + boost::iends_with(name, ".jpg") || + boost::iends_with(name, ".3mf"); + }; + + copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); + } + + BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; + + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); + return false; + } + } + + return true; +} + +// m_printer_hold_alias survives reset() (and a cache body that failed partway +// in), so every full-bundle rebuild clears all five collections' maps by hand. +void PresetBundle::clear_printer_hold_aliases() +{ + this->prints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); +} + //BBS: add json related logic, load system presets from json std::pair PresetBundle::load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule) { @@ -2266,39 +2426,19 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); - // Single-bundle cache: try user cache, then bundled cache, then JSON parse. - bool loaded_from_cache = false; - if (!validation_mode) { - const auto t0 = std::chrono::steady_clock::now(); - const std::string expected_key = compute_system_presets_cache_key(dir.string()); - if (try_load_system_presets_from_cache(expected_key)) { - update_system_maps(); - const auto ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t0).count(); - BOOST_LOG_TRIVIAL(info) << "PresetBundle: system presets loaded from single-bundle cache in " << ms << " ms"; - return {PresetsConfigSubstitutions{}, ""}; - } - loaded_from_cache = false; // cache miss — fall through to JSON parse - } + const auto load_t0 = std::chrono::steady_clock::now(); + + // The vendors below are loaded whole and against each other — the filament + // library first, then every other vendor with it as the base — so each parse + // is complete enough to be worth caching. + m_generate_vendor_caches = m_generate_vendor_caches || ! validation_mode; - const auto json_load_t0 = std::chrono::steady_clock::now(); PresetsConfigSubstitutions substitutions; std::string errors_cummulative; - std::set errored_vendors; // vendors whose JSON parse failed — skip their cache save bool first = true; - std::vector vendor_names; - // store all vendor names in vendor_names - for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { - std::string vendor_file = dir_entry.path().string(); - if (!Slic3r::is_json_file(vendor_file)) - continue; - - std::string vendor_name = dir_entry.path().filename().string(); - - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - vendor_names.push_back(vendor_name); - } + // Sorted, so any duplicate-preset warning below comes out in the same order on + // every run. + const std::set vendor_names = vendor_names_in(dir); // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded // first because other vendors' filaments may inherit from it via the // `base_bundle` lookup in parse_subfile. The remaining vendors are @@ -2314,8 +2454,13 @@ std::pair PresetBundle::load_system_pre } // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (!orca_lib_vendor.empty()) { + if (! orca_lib_vendor.empty()) { try { + // Match a fresh launch before parsing: hold aliases and the error + // counter survive reset(), and would otherwise carry prior-cycle + // state into this load. + this->clear_printer_hold_aliases(); + this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json(dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule).first); first = false; } catch (const std::runtime_error &err) { @@ -2323,7 +2468,6 @@ std::pair PresetBundle::load_system_pre throw err; errors_cummulative += err.what(); errors_cummulative += "\n"; - errored_vendors.insert(orca_lib_vendor); } } @@ -2339,10 +2483,10 @@ std::pair PresetBundle::load_system_pre for (size_t i = range.begin(); i < range.end(); ++i) { auto bundle = std::make_unique(); bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); try { auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, - compatibility_rule, this); + dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this); parallel_substitutions[i] = std::move(result.first); parallel_bundles[i] = std::move(bundle); } catch (const std::runtime_error &err) { @@ -2360,7 +2504,6 @@ std::pair PresetBundle::load_system_pre throw std::runtime_error(parallel_errors[i]); errors_cummulative += parallel_errors[i]; errors_cummulative += "\n"; - errored_vendors.insert(other_vendors[i]); continue; } if (!parallel_bundles[i]) @@ -2389,26 +2532,9 @@ std::pair PresetBundle::load_system_pre this->update_system_maps(); - { - const auto json_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - json_load_t0).count(); - BOOST_LOG_TRIVIAL(info) << "PresetBundle: system presets loaded from JSON in " << json_ms << " ms"; - } - - // Save single-bundle cache after successful JSON parse. - if (!validation_mode && errored_vendors.empty()) { - const auto save_t0 = std::chrono::steady_clock::now(); - const auto stats = save_system_presets_cache(dir.string(), user_system_presets_cache_path()); - const auto save_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - save_t0).count(); - if (stats.ok) - BOOST_LOG_TRIVIAL(info) << "PresetBundle: single-bundle cache saved in " << save_ms << " ms" - << " (print=" << stats.print_presets - << " filament=" << stats.filament_presets - << " printer=" << stats.printer_presets << ")"; - else - BOOST_LOG_TRIVIAL(warning) << "PresetBundle: single-bundle cache save failed"; - } + const auto load_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; @@ -4824,20 +4950,282 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(": finished"); } +// Orca: install one source-form preset entry — parsed from its JSON subfile just +// now, or deserialized from the vendor's cache; the code is shared so a +// cache-loaded bundle cannot come out different from a JSON-loaded one. +// Resolves `inherits` against the presets installed before this one +// (config_maps) or against base_bundle's filament library, flattens, validates +// and registers the preset. Returns the reason installation failed, empty on +// success. +std::string PresetBundle::install_vendor_preset( + const CachedPreset& entry, + const std::string& path, const std::string& vendor_name, + const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, + ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, + std::map& config_maps, std::map& filament_id_maps, + PresetCollection* presets_collection, size_t& count, bool is_from_lib, + const std::set* retain_configs) +{ + const VendorProfile* current_vendor_profile = &this->vendors.at(vendor_name); + const std::string subfile = path + "/" + vendor_name + "/" + entry.sub_path; + const std::string& preset_name = entry.name; + std::string alias_name, filament_id = entry.filament_id; + std::vector renamed_from = entry.renamed_from; + DynamicPrintConfig config; + const DynamicPrintConfig* default_config = nullptr; + std::string reason; + + //check whether it inherits other preset or not + if (! entry.inherits.empty()) { + auto it2 = config_maps.find(entry.inherits); + if (it2 != config_maps.end()) + default_config = &(it2->second); + if (default_config == nullptr && base_bundle != nullptr) { + auto base_it2 = base_bundle->m_config_maps.find(entry.inherits); + if (base_it2 != base_bundle->m_config_maps.end()) + default_config = &(base_it2->second); + } + if (default_config != nullptr) { + if (filament_id.empty() && (presets_collection->type() == Preset::TYPE_FILAMENT)) { + auto filament_id_map_iter = filament_id_maps.find(entry.inherits); + if (filament_id_map_iter != filament_id_maps.end()) { + filament_id = filament_id_map_iter->second; + } + if (filament_id.empty() && base_bundle != nullptr) { + auto base_filament_id_map_iter = base_bundle->m_filament_id_maps.find(entry.inherits); + if (base_filament_id_map_iter != base_bundle->m_filament_id_maps.end()) { + filament_id = base_filament_id_map_iter->second; + } + } + } + } + else { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << preset_name; + // throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); + reason = "Can not find inherits: " + entry.inherits; + return reason; + } + } + else { + if (presets_collection->type() == Preset::TYPE_PRINTER) + default_config = &presets_collection->default_preset_for(entry.config_src).config; + else + default_config = &presets_collection->default_preset().config; + } + config = *default_config; + config.apply(entry.config_src); + extend_default_config_length(config, true, *default_config); + if (entry.instantiation == "false" && "Template" != vendor_name) { + // Report configuration fields, which are misplaced into a wrong group. + std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); + if (!incorrect_keys.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys + << ", which were removed"; + } + + if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) + config_maps.emplace(preset_name, std::move(config)); + if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!filament_id.empty())) + filament_id_maps.emplace(preset_name, filament_id); + return reason; + } + if (config.has("alias")) + alias_name = (dynamic_cast(config.option("alias")))->value; + Preset::normalize(config); + + // Report configuration fields, which are misplaced into a wrong group. + std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); + if (!incorrect_keys.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys + << ", which were removed"; + } + + if (presets_collection->type() == Preset::TYPE_PRINTER) { + // Filter out printer presets, which are not mentioned in the vendor profile. + // These presets are considered not installed. + auto printer_model = config.opt_string("printer_model"); + if (printer_model.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines no printer model, it will be ignored."; + reason = std::string("can not find printer_model"); + return reason; + } + auto printer_variant = config.opt_string("printer_variant"); + if (printer_variant.empty()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines no printer variant, it will be ignored."; + reason = std::string("can not find printer_variant"); + return reason; + } + auto it_model = std::find_if(current_vendor_profile->models.cbegin(), current_vendor_profile->models.cend(), + [&](const VendorProfile::PrinterModel &m) { return m.id == printer_model; } + ); + if (it_model == current_vendor_profile->models.end()) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines invalid printer model \"" << printer_model << "\", it will be ignored."; + reason = std::string("can not find printer model in vendor profile"); + return reason; + } + auto it_variant = it_model->variant(printer_variant); + if (it_variant == nullptr) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" defines invalid printer variant \"" << printer_variant << "\", it will be ignored."; + reason = std::string("can not find printer_variant in vendor profile"); + return reason; + } + // An instantiation printer profile's nozzle_diameter must match the numeric (diameter) + // prefix of its printer_variant: "0.4" -> {0.4}, "0.8HF" -> {0.8} (a trailing + // non-numeric suffix such as "HF"/"HS" distinguishes a hardware sub-variant and is + // ignored here), and for multi-nozzle printers "0.4+0.6" -> {0.4, 0.6}. + // Note: a variant may legitimately repeat across presets of the same model (e.g. speed + // modes, IDEX copy/mirror, or different control boards), so only the diameter is + // validated, not variant uniqueness. Validation-only so the app keeps loading existing + // profiles unchanged. + if (validation_mode && entry.instantiation == "true") { + const auto *nd = config.option("nozzle_diameter"); + std::set nozzles, variant_nozzles; + if (nd != nullptr) + nozzles.insert(nd->values.begin(), nd->values.end()); + std::vector variant_tokens; + boost::algorithm::split(variant_tokens, printer_variant, boost::algorithm::is_any_of("+")); + bool variant_ok = true; // printer_variant is already guaranteed non-empty above + for (const std::string &tok : variant_tokens) { + size_t consumed = 0; + double d = string_to_double_decimal_point(tok, &consumed); + // Require a leading numeric diameter; a trailing suffix (e.g. "HF") is allowed. + if (consumed == 0) { variant_ok = false; break; } + variant_nozzles.insert(d); + } + if (!variant_ok || variant_nozzles != nozzles) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" has printer_variant \"" << printer_variant << + "\" that does not match its nozzle_diameter \"" << (nd ? nd->serialize() : std::string()) << "\". " + "printer_variant must begin with the nozzle diameter, optionally followed by a non-numeric suffix " + "(e.g. \"0.4\", \"0.8HF\"); for multi-nozzle printers, join the per-nozzle diameters with \"+\" in " + "nozzle order (e.g. \"0.4+0.6\")."; + } + } + } + const Preset *preset_existing = presets_collection->find_preset(preset_name, false); + if (preset_existing != nullptr) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << + preset_name << "\" has already been loaded from another Config Bundle."; + reason = std::string("duplicated defines"); + return reason; + } + + auto file_path = (boost::filesystem::path(data_dir()) /PRESET_SYSTEM_DIR/ vendor_name / entry.sub_path).make_preferred(); + if(validation_mode) + file_path = (boost::filesystem::path(data_dir()) / vendor_name / entry.sub_path).make_preferred(); + + // Load the preset into the list of presets, save it to disk. + Preset &loaded = presets_collection->load_preset(file_path.string(), preset_name, std::move(config), false); + if (flags.has(LoadConfigBundleAttribute::LoadSystem)) { + loaded.is_system = true; + loaded.vendor = current_vendor_profile; + loaded.version = current_vendor_profile->config_version; + loaded.description = entry.description; + loaded.setting_id = entry.setting_id; + // Derive the preset setting_id on the fly when a profile ships without one, + // matching scripts/assign_vendor_setting_ids.py. Only instantiated presets + // carry an id; non-instantiated base profiles return earlier above. This never + // touches the per-user cloud-sync setting_id written into user .info files. + if (loaded.setting_id.empty() && entry.instantiation == "true") + loaded.setting_id = generate_preset_setting_id( + vendor_name, Preset::get_type_string(presets_collection->type()), preset_name); + loaded.filament_id = filament_id; + loaded.m_from_orca_filament_lib = is_from_lib; + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << " " << __LINE__ << ", " << loaded.name << " load filament_id: " << filament_id; + if (presets_collection->type() == Preset::TYPE_FILAMENT) { + if (filament_id.empty() && "Template" != vendor_name) { + ++m_errors; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": can not find filament_id for " << preset_name; + //throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); + reason = "Can not find filament_id for " + preset_name; + return reason; + } + else { + filament_id_maps.emplace(preset_name, filament_id); + } + } + } + + // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. + if (alias_name.empty()) { + size_t end_pos = preset_name.find_first_of("@"); + if (end_pos != std::string::npos) { + alias_name = preset_name.substr(0, end_pos); + if (renamed_from.empty()) + // Add the preset name with the '@' character removed into the "renamed_from" list. + renamed_from.emplace_back(alias_name + preset_name.substr(end_pos + 1)); + boost::trim_right(alias_name); + } + } + if (alias_name.empty()) + loaded.alias = preset_name; + else { + loaded.alias = std::move(alias_name); + filaments.set_printer_hold_alias(loaded.alias, loaded); + } + loaded.renamed_from = std::move(renamed_from); + if (! substitution_context.empty()) + substitutions.push_back({ + preset_name, presets_collection->type(), PresetConfigSubstitutions::Source::ConfigBundle, + std::string(), std::move(substitution_context.substitutions) }); + if (retain_configs == nullptr || retain_configs->count(preset_name) != 0) + config_maps.emplace(preset_name, loaded.config); + ++count; + //BBS: add config related logs + BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", got preset %1%, from %2%")%loaded.name %subfile; + return reason; +} + //BBS: Load a config bundle file from json std::pair PresetBundle::load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) { // Enable substitutions for user config bundle, throw an exception when loading a system profile. ConfigSubstitutionContext substitution_context { compatibility_rule }; PresetsConfigSubstitutions substitutions; + // Errors already on this bundle when the load began; the cache stamp below + // counts only what this parse adds. + const int errors_at_entry = m_errors; //BBS: add config related logs - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%path.c_str()%compatibility_rule; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; if (flags.has(LoadConfigBundleAttribute::ResetUserProfile) || flags.has(LoadConfigBundleAttribute::LoadSystem)) // Reset this bundle, delete user profile files if SaveImported. this->reset(flags.has(LoadConfigBundleAttribute::SaveImported)); + // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only + // scans want a slice of one. Validation reads the JSONs whatever is cached. + const boost::filesystem::path dir_path(dir); + const bool cacheable = flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name, base_bundle)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + + // Orca: a build that ships preset caches installs them without the preset + // JSONs, so a vendor left to be parsed is parsed from the profiles in + // resources. An update does deliver JSONs into `dir`, and those win. + const std::string path = (validation_mode || boost::filesystem::exists(dir_path / (vendor_name + ".json"))) + ? dir : (boost::filesystem::path(resources_dir()) / "profiles").string(); + // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); std::string root_file = path + "/" + vendor_name + ".json"; @@ -5050,7 +5438,6 @@ std::pair PresetBundle::load_vendor_configs_ //insert the vendor profile this->vendors.emplace(vendor_name, vendor_profile); - const VendorProfile* current_vendor_profile = &this->vendors[vendor_name]; BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", loaded vendor profile, name %1%, id %2%, version %3%")%vendor_profile.name%vendor_profile.id%vendor_profile.config_version.to_string(); @@ -5061,123 +5448,65 @@ std::pair PresetBundle::load_vendor_configs_ PresetCollection *presets = nullptr; size_t presets_loaded = 0; - auto parse_subfile = [this, path, vendor_name, presets_loaded, current_vendor_profile, base_bundle]( + // Parse one subfile into a source-form entry — everything the JSON states, + // nothing resolved. Installing the entry (install_vendor_preset) is the + // same code whether the entry was parsed just now or deserialized from the + // vendor's cache. + auto parse_subfile = [this, path, vendor_name]( ConfigSubstitutionContext& substitution_context, - PresetsConfigSubstitutions& substitutions, - LoadConfigBundleAttributes& flags, - std::pair& subfile_iter, - std::map& config_maps, - std::map& filament_id_maps, - PresetCollection* presets_collection, - size_t& count, bool is_from_lib = false) -> std::string { + const std::pair& subfile_iter, + CachedPreset& entry) -> std::string { std::string subfile = path + "/" + vendor_name + "/" + subfile_iter.second; - // Load the print, filament or printer preset. - std::string preset_name; - DynamicPrintConfig config; - std::string alias_name, inherits, description, instantiation, setting_id, filament_id; - std::vector renamed_from; - const DynamicPrintConfig* default_config = nullptr; - std::string reason; + std::string reason; try { std::map key_values; substitution_context.substitutions.clear(); //parse the json elements - DynamicPrintConfig config_src; - std::string _renamed_from_str; - config_src.load_from_json(subfile, substitution_context, false, key_values, reason); + entry.sub_path = subfile_iter.second; + entry.config_src.load_from_json(subfile, substitution_context, false, key_values, reason); if (!reason.empty()) { ++m_errors; BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": load config file "<second; + entry.setting_id = setting_it->second; auto filament_it = key_values.find(BBL_JSON_KEY_FILAMENT_ID); if (filament_it != key_values.end()) - filament_id = filament_it->second; - //check whether it inherits other preset or not + entry.filament_id = filament_it->second; auto it1 = key_values.find(BBL_JSON_KEY_INHERITS); if (it1 != key_values.end()) { - inherits = it1->second; - auto it2 = config_maps.find(inherits); - default_config = nullptr; - if (it2 != config_maps.end()) - default_config = &(it2->second); - if(default_config == nullptr && base_bundle != nullptr) { - auto base_it2 = base_bundle->m_config_maps.find(inherits); - if (base_it2 != base_bundle->m_config_maps.end()) - default_config = &(base_it2->second); - } - if (default_config != nullptr) { - if (filament_id.empty() && (presets_collection->type() == Preset::TYPE_FILAMENT)) { - auto filament_id_map_iter = filament_id_maps.find(inherits); - if (filament_id_map_iter != filament_id_maps.end()) { - filament_id = filament_id_map_iter->second; - } - if (filament_id.empty() && base_bundle != nullptr) { - auto filament_id_map_iter = base_bundle->m_filament_id_maps.find(inherits); - if (filament_id_map_iter != base_bundle->m_filament_id_maps.end()) { - filament_id = filament_id_map_iter->second; - } - } - } - } - else { + entry.inherits = it1->second; + // An `inherits` key naming nothing can never resolve; fail it + // here so install can key off the empty string as "no inherits". + if (entry.inherits.empty()) { ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << inherits << " for " << preset_name; - // throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find inherits: " + inherits; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find inherits " << entry.inherits << " for " << entry.name; + reason = "Can not find inherits: " + entry.inherits; return reason; } } - else { - if (presets_collection->type() == Preset::TYPE_PRINTER) - default_config = &presets_collection->default_preset_for(config_src).config; - else - default_config = &presets_collection->default_preset().config; - } - config = *default_config; - config.apply(config_src); - extend_default_config_length(config, true, *default_config); - if (instantiation == "false" && "Template" != vendor_name) { - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } - - config_maps.emplace(preset_name, std::move(config)); - if ((presets_collection->type() == Preset::TYPE_FILAMENT) && (!filament_id.empty())) - filament_id_maps.emplace(preset_name, filament_id); - return reason; - } - if (config.has("alias")) - alias_name = (dynamic_cast(config.option("alias")))->value; - if (key_values.find(ORCA_JSON_KEY_RENAMED_FROM) != key_values.end()) { - if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], renamed_from)) { - BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << path << "\": The preset \"" << preset_name + if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], entry.renamed_from)) { + BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << path << "\": The preset \"" << entry.name << "\" contains invalid \"renamed_from\" key, which is being ignored."; } } - Preset::normalize(config); } catch(nlohmann::detail::parse_error &err) { ++m_errors; @@ -5185,195 +5514,60 @@ std::pair PresetBundle::load_vendor_configs_ reason = std::string("json parse error") + err.what(); return reason; } - - // Report configuration fields, which are misplaced into a wrong group. - std::string incorrect_keys = Preset::remove_invalid_keys(config, *default_config); - if (!incorrect_keys.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": The config " << subfile << " contains incorrect keys: " << incorrect_keys - << ", which were removed"; - } - - if (presets_collection->type() == Preset::TYPE_PRINTER) { - // Filter out printer presets, which are not mentioned in the vendor profile. - // These presets are considered not installed. - auto printer_model = config.opt_string("printer_model"); - if (printer_model.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer model, it will be ignored."; - reason = std::string("can not find printer_model"); - return reason; - } - auto printer_variant = config.opt_string("printer_variant"); - if (printer_variant.empty()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines no printer variant, it will be ignored."; - reason = std::string("can not find printer_variant"); - return reason; - } - auto it_model = std::find_if(current_vendor_profile->models.cbegin(), current_vendor_profile->models.cend(), - [&](const VendorProfile::PrinterModel &m) { return m.id == printer_model; } - ); - if (it_model == current_vendor_profile->models.end()) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer model \"" << printer_model << "\", it will be ignored."; - reason = std::string("can not find printer model in vendor profile"); - return reason; - } - auto it_variant = it_model->variant(printer_variant); - if (it_variant == nullptr) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" defines invalid printer variant \"" << printer_variant << "\", it will be ignored."; - reason = std::string("can not find printer_variant in vendor profile"); - return reason; - } - // An instantiation printer profile's nozzle_diameter must match the numeric (diameter) - // prefix of its printer_variant: "0.4" -> {0.4}, "0.8HF" -> {0.8} (a trailing - // non-numeric suffix such as "HF"/"HS" distinguishes a hardware sub-variant and is - // ignored here), and for multi-nozzle printers "0.4+0.6" -> {0.4, 0.6}. - // Note: a variant may legitimately repeat across presets of the same model (e.g. speed - // modes, IDEX copy/mirror, or different control boards), so only the diameter is - // validated, not variant uniqueness. Validation-only so the app keeps loading existing - // profiles unchanged. - if (validation_mode && instantiation == "true") { - const auto *nd = config.option("nozzle_diameter"); - std::set nozzles, variant_nozzles; - if (nd != nullptr) - nozzles.insert(nd->values.begin(), nd->values.end()); - std::vector variant_tokens; - boost::algorithm::split(variant_tokens, printer_variant, boost::algorithm::is_any_of("+")); - bool variant_ok = true; // printer_variant is already guaranteed non-empty above - for (const std::string &tok : variant_tokens) { - size_t consumed = 0; - double d = string_to_double_decimal_point(tok, &consumed); - // Require a leading numeric diameter; a trailing suffix (e.g. "HF") is allowed. - if (consumed == 0) { variant_ok = false; break; } - variant_nozzles.insert(d); - } - if (!variant_ok || variant_nozzles != nozzles) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" has printer_variant \"" << printer_variant << - "\" that does not match its nozzle_diameter \"" << (nd ? nd->serialize() : std::string()) << "\". " - "printer_variant must begin with the nozzle diameter, optionally followed by a non-numeric suffix " - "(e.g. \"0.4\", \"0.8HF\"); for multi-nozzle printers, join the per-nozzle diameters with \"+\" in " - "nozzle order (e.g. \"0.4+0.6\")."; - } - } - } - const Preset *preset_existing = presets_collection->find_preset(preset_name, false); - if (preset_existing != nullptr) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << "Error in a Vendor Config Bundle \"" << path << "\": The printer preset \"" << - preset_name << "\" has already been loaded from another Config Bundle."; - reason = std::string("duplicated defines"); - return reason; - } - - auto file_path = (boost::filesystem::path(data_dir()) /PRESET_SYSTEM_DIR/ vendor_name / subfile_iter.second).make_preferred(); - if(validation_mode) - file_path = (boost::filesystem::path(data_dir()) / vendor_name / subfile_iter.second).make_preferred(); - - // Load the preset into the list of presets, save it to disk. - Preset &loaded = presets_collection->load_preset(file_path.string(), preset_name, std::move(config), false); - if (flags.has(LoadConfigBundleAttribute::LoadSystem)) { - loaded.is_system = true; - loaded.vendor = current_vendor_profile; - loaded.version = current_vendor_profile->config_version; - loaded.description = description; - loaded.setting_id = setting_id; - // Derive the preset setting_id on the fly when a profile ships without one, - // matching scripts/assign_vendor_setting_ids.py. Only instantiated presets - // carry an id; non-instantiated base profiles return earlier above. This never - // touches the per-user cloud-sync setting_id written into user .info files. - if (loaded.setting_id.empty() && instantiation == "true") - loaded.setting_id = generate_preset_setting_id( - vendor_name, Preset::get_type_string(presets_collection->type()), preset_name); - loaded.filament_id = filament_id; - loaded.m_from_orca_filament_lib = is_from_lib; - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << " " << __LINE__ << ", " << loaded.name << " load filament_id: " << filament_id; - if (presets_collection->type() == Preset::TYPE_FILAMENT) { - if (filament_id.empty() && "Template" != vendor_name) { - ++m_errors; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__<< ": can not find filament_id for " << preset_name; - //throw ConfigurationError(format("can not find inherits %1% for %2%", inherits, preset_name)); - reason = "Can not find filament_id for " + preset_name; - return reason; - } - else { - filament_id_maps.emplace(preset_name, filament_id); - } - } - } - - // Derive the profile logical name aka alias from the preset name if the alias was not stated explicitely. - if (alias_name.empty()) { - size_t end_pos = preset_name.find_first_of("@"); - if (end_pos != std::string::npos) { - alias_name = preset_name.substr(0, end_pos); - if (renamed_from.empty()) - // Add the preset name with the '@' character removed into the "renamed_from" list. - renamed_from.emplace_back(alias_name + preset_name.substr(end_pos + 1)); - boost::trim_right(alias_name); - } - } - if (alias_name.empty()) - loaded.alias = preset_name; - else { - loaded.alias = std::move(alias_name); - filaments.set_printer_hold_alias(loaded.alias, loaded); - } - loaded.renamed_from = std::move(renamed_from); - if (! substitution_context.empty()) - substitutions.push_back({ - preset_name, presets_collection->type(), PresetConfigSubstitutions::Source::ConfigBundle, - std::string(), std::move(substitution_context.substitutions) }); - config_maps.emplace(preset_name, loaded.config); - ++count; - //BBS: add config related logs - BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", got preset %1%, from %2%")%loaded.name %subfile; return reason; }; std::map configs; std::map filament_id_maps; + // Orca: whether to (re)write the vendor's cache after this parse, leaving it + // in step with the profile so the next run reads it instead. It is written + // where the vendor was looked for, even when the profile came from resources, + // and stamped with the version that profile claims — a profile without one + // cannot be judged for staleness later, and a cache nothing can invalidate is + // worse than none. + const bool will_cache = cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid(); + std::vector process_entries, filament_entries, machine_entries; + // Errors added by install are counted apart: a cache load runs install again, + // so the parse_errors stamped into the cache must hold only what a cache load + // will not recount. + int install_errors = 0; + auto load_subfiles = [&](std::vector>& subfiles, + std::vector& entries, const char* kind, bool is_from_lib = false) { + configs.clear(); + filament_id_maps.clear(); + for (auto& subfile : subfiles) { + CachedPreset entry; + std::string reason = parse_subfile(substitution_context, subfile, entry); + if (reason.empty()) { + const int errors_before_install = m_errors; + reason = install_vendor_preset(entry, path, vendor_name, base_bundle, flags, + substitution_context, substitutions, configs, filament_id_maps, presets, + presets_loaded, is_from_lib); + install_errors += m_errors - errors_before_install; + } + if (!reason.empty()) { + ++m_errors; + //parse error + std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse %1% setting from %2%") % kind % subfile_path; + throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); + } + if (will_cache) + entries.emplace_back(std::move(entry)); + } + }; + + // The section order below — process, filaments (with the ORCA-lib map copy), + // printers — is mirrored by load_vendor_cache's install loops; keep the two + // in lockstep. //3.1) paste the process presets = &this->prints; - configs.clear(); - filament_id_maps.clear(); - for (auto& subfile : process_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, presets_loaded); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse process setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } - } + load_subfiles(process_subfiles, process_entries, "process"); //3.2) paste the filaments presets = &this->filaments; - configs.clear(); - filament_id_maps.clear(); const auto is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; - for (auto& subfile : filament_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, - presets_loaded, is_orca_lib); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse filament setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } - } + load_subfiles(filament_subfiles, filament_entries, "filament", is_orca_lib); if (is_orca_lib) { m_config_maps = configs; m_filament_id_maps = filament_id_maps; @@ -5381,18 +5575,14 @@ std::pair PresetBundle::load_vendor_configs_ //3.3) paste the printers presets = &this->printers; - configs.clear(); - filament_id_maps.clear(); - for (auto& subfile : machine_subfiles) - { - std::string reason = parse_subfile(substitution_context, substitutions, flags, subfile, configs, filament_id_maps, presets, presets_loaded); - if (!reason.empty()) { - ++m_errors; - //parse error - std::string subfile_path = path + "/" + vendor_name + "/" + subfile.second; - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << boost::format(", got error when parse printer setting from %1%") % subfile_path; - throw ConfigurationError((boost::format("Failed loading configuration file %1%\nSuggest cleaning the directory %2% firstly") % subfile_path % path).str()); - } + load_subfiles(machine_subfiles, machine_entries, "printer"); + + if (will_cache) { + const auto parse_errors = uint64_t(m_errors - errors_at_entry - install_errors); + if (! save_vendor_cache((dir_path / (vendor_name + ".opc")).string(), vendor_name, + vendor_profile.config_version.to_string(), this->vendors, + process_entries, filament_entries, machine_entries, parse_errors)) + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; } //BBS: add config related logs @@ -6017,7 +6207,7 @@ bool BundleMetadata::save_to_json(const std::string& path) const return false; } } -// ---- System presets single-bundle cache implementation ------------------ +// ---- Preset cache file format (shared by the per-vendor cache) ---------- namespace { @@ -6031,76 +6221,60 @@ struct CacheFileHeader { #pragma pack(pop) static_assert(sizeof(CacheFileHeader) == 20, "CacheFileHeader must be 20 bytes"); -// Presets for one vendor, grouped so vendor pointers can be re-wired on load. -struct VendorPresetGroup { - std::vector prints, filaments, printers, sla_prints, sla_materials; - template - void serialize(Archive& ar) { ar(prints, filaments, printers, sla_prints, sla_materials); } -}; +constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ" +// Bump when the wire format changes in a way the schema fingerprint cannot +// detect: reordering, removing, or retyping a field of a hand-written +// serialize() (PresetBundle::CachedPreset, VendorProfile and its nested types), +// or when the cache's own layout or the meaning of its stamps changes (e.g. +// the move from flattened presets to source-form entries). +constexpr uint32_t CACHE_VERSION = 5; -struct SystemPresetsCache { - static constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ" - static constexpr uint32_t CACHE_VERSION = 1; +// A cache stays usable as long as it was built from a vendor profile at least +// as new as the one now on disk. Profiles whose version is invalid cannot be +// judged this way and are never served from cache; where no profile sits +// beside the cache at all, nothing can be newer than it — that state is passed +// as Semver::inf(), which no real profile can carry (an invalid version could +// not say it apart from "profile there but unjudgeable", and zero would +// collide with a genuine "0.0.0"). +static bool cache_covers_version(const std::string& cached, const Semver& on_disk) +{ + if (on_disk == Semver::inf()) + return true; // before parsing `cached`: nothing exists that the stamp must cover + if (! on_disk.valid()) + return false; + const auto cached_ver = Semver::parse(cached); + return cached_ver && *cached_ver >= on_disk; +} - uint32_t cache_version = CACHE_VERSION; - uint32_t config_options_count = 0; - std::string bundle_key; - - VendorMap vendors; - std::map preset_groups; - std::map config_maps; - std::map filament_id_maps; - - template - void serialize(Archive& ar) - { - ar(cache_version, config_options_count, bundle_key, - vendors, preset_groups, config_maps, filament_id_maps); - } - - bool is_valid(const std::string& expected_key) const - { - return cache_version == CACHE_VERSION - && config_options_count == static_cast(print_config_def.options.size()) - && bundle_key == expected_key; - } -}; +// Fingerprint of everything that determines the cache wire format: the app +// version and the DynamicPrintConfig option schema (key/type/ordinal/enum +// values — serialization_key_ordinal IS the config wire format). Any mismatch +// means bytes written by another build could deserialize into the wrong +// fields, so the cache is rejected wholesale before its body is read. +const std::string& compute_cache_schema_fingerprint() +{ + // Constant for the lifetime of the process (print_config_def is immutable + // after static initialization), and asked for once per cache load and save. + static const std::string fingerprint = [] { + std::string schema; + schema += SLIC3R_VERSION; + schema += ';'; + for (const auto& [key, def] : print_config_def.options) { // std::map => stable order + schema += key; + schema += '#'; schema += std::to_string(int(def.type)); + schema += '@'; schema += std::to_string(def.serialization_key_ordinal); + for (const std::string& ev : def.enum_values) { schema += ','; schema += ev; } + schema += ';'; + } + boost::crc_32_type crc; + crc.process_bytes(schema.data(), schema.size()); + return std::to_string(crc.checksum()); + }(); + return fingerprint; +} } // anonymous namespace -// static -std::string PresetBundle::bundled_system_presets_cache_path() -{ - return (boost::filesystem::path(resources_dir()) / "profiles" / "system_presets.cache") - .make_preferred().string(); -} - -// static -std::string PresetBundle::user_system_presets_cache_path() -{ - return (boost::filesystem::path(data_dir()) / PRESET_SYSTEM_DIR / "system_presets.cache") - .make_preferred().string(); -} - -// static -std::string PresetBundle::compute_system_presets_cache_key(const std::string& system_dir) -{ - // Sorted map so key is stable regardless of directory iteration order. - std::map keys; - try { - for (const auto& e : boost::filesystem::directory_iterator(system_dir)) { - if (Slic3r::is_json_file(e.path().string())) - keys[e.path().stem().string()] = get_vendor_cache_key(e.path().string()); - } - } catch (const std::exception& ex) { - BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: cannot scan " << system_dir << ": " << ex.what(); - } - std::string combined; - for (const auto& [name, key] : keys) - combined += name + ":" + key + ";"; - return combined; -} - // static bool PresetBundle::read_cache_blob(const std::string& path, std::string& out_blob) { @@ -6111,7 +6285,7 @@ bool PresetBundle::read_cache_blob(const std::string& path, std::string& out_blo CacheFileHeader fhdr; if (!ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr))) return false; - if (fhdr.magic != SystemPresetsCache::CACHE_MAGIC) + if (fhdr.magic != CACHE_MAGIC) return false; if (fhdr.data_size == 0 || fhdr.data_size > 512u * 1024u * 1024u) return false; @@ -6132,7 +6306,7 @@ bool PresetBundle::read_cache_blob(const std::string& path, std::string& out_blo } // static -void PresetBundle::write_cache_blob(const std::string& path, const std::string& blob) +bool PresetBundle::write_cache_blob(const std::string& path, const std::string& blob) { boost::crc_32_type crc; crc.process_bytes(blob.data(), blob.size()); @@ -6141,179 +6315,186 @@ void PresetBundle::write_cache_blob(const std::string& path, const std::string& boost::nowide::ofstream ofs(path, std::ios::binary | std::ios::trunc); if (!ofs.is_open()) { BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: cannot open for writing: " << path; - return; + return false; } CacheFileHeader fhdr; - fhdr.magic = SystemPresetsCache::CACHE_MAGIC; - fhdr.version = SystemPresetsCache::CACHE_VERSION; + fhdr.magic = CACHE_MAGIC; + fhdr.version = CACHE_VERSION; fhdr.data_size = static_cast(blob.size()); fhdr.crc32 = crc.checksum(); ofs.write(reinterpret_cast(&fhdr), sizeof(fhdr)); ofs.write(blob.data(), static_cast(blob.size())); + ofs.close(); // flush; close() raises failbit on error + if (! ofs.good()) + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: write failed (" << path << ")"; + return ofs.good(); } catch (const std::exception& e) { BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: write failed (" << path << "): " << e.what(); + return false; } } -// Apply one VendorPresetGroup from a loaded cache into this PresetBundle's collections. -static void apply_vendor_preset_group(VendorPresetGroup& grp, - const VendorProfile* vp, - PresetCollection& prints_coll, - PresetCollection& filaments_coll, - PrinterPresetCollection& printers_coll, - PresetCollection& sla_prints_coll, - PresetCollection& sla_materials_coll) +// ---- Per-vendor preset cache implementation ------------------------------ + +// static +bool PresetBundle::save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const VendorMap& vendors, + const std::vector& process_entries, + const std::vector& filament_entries, + const std::vector& machine_entries, + uint64_t parse_errors) { - auto apply = [&](std::vector& cached, PresetCollection& coll, bool is_filaments) { - for (Preset& cp : cached) { - // Reserve a slot in the collection, then move the fully-deserialized - // preset into it so all serialized fields are transferred at once. - // Only vendor (a raw pointer, excluded from serialization) is patched after. - DynamicPrintConfig config = cp.config; - Preset& p = coll.load_preset(cp.file, cp.name, std::move(config), false, cp.version); - const std::string alias = cp.alias; - p = std::move(cp); - p.vendor = vp; - if (is_filaments) - coll.set_printer_hold_alias(alias, p); + try { + std::ostringstream body(std::ios::binary); + { + cereal::BinaryOutputArchive ar(body); + ar(CACHE_VERSION); + ar(compute_cache_schema_fingerprint()); + ar(vendor_name, vendor_version); + ar(vendors); + ar(process_entries, filament_entries, machine_entries); + ar(parse_errors); } - }; - apply(grp.prints, prints_coll, false); - apply(grp.filaments, filaments_coll, true); - apply(grp.printers, printers_coll, false); - apply(grp.sla_prints, sla_prints_coll, false); - apply(grp.sla_materials, sla_materials_coll, false); -} - -bool PresetBundle::try_load_system_presets_from_cache(const std::string& expected_key) -{ - auto try_path = [&](const std::string& path) -> bool { - std::string blob; - if (!read_cache_blob(path, blob)) - return false; - try { - std::istringstream iss(blob); - cereal::BinaryInputArchive ar(iss); - SystemPresetsCache cache; - ar(cache); - if (!cache.is_valid(expected_key)) - return false; - - this->reset(false); - vendors = std::move(cache.vendors); - - for (auto& [vendor_id, grp] : cache.preset_groups) { - auto it = vendors.find(vendor_id); - if (it == vendors.end()) continue; - apply_vendor_preset_group(grp, &it->second, - prints, filaments, printers, - sla_prints, sla_materials); - } - m_config_maps = std::move(cache.config_maps); - m_filament_id_maps = std::move(cache.filament_id_maps); - return true; - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: apply failed (" << path << "): " << e.what(); - return false; - } - }; - - if (try_path(user_system_presets_cache_path())) - return true; - if (try_path(bundled_system_presets_cache_path())) { - // Promote bundled cache to user path so future loads skip JSON parse. - const std::string user_path = user_system_presets_cache_path(); - const std::string bundled_path = bundled_system_presets_cache_path(); - try { - boost::filesystem::create_directories( - boost::filesystem::path(user_path).parent_path()); - boost::filesystem::copy_file(bundled_path, user_path, - boost::filesystem::copy_options::overwrite_existing); - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: promote failed: " << e.what(); - } - return true; + return write_cache_blob(cache_path, body.str()); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache " << cache_path << ": " << e.what(); + return false; } - return false; -} - -PresetBundle::SaveCacheResult PresetBundle::save_system_presets_cache( - const std::string& profiles_dir, const std::string& output_path) const -{ - SystemPresetsCache cache; - cache.config_options_count = static_cast(print_config_def.options.size()); - cache.bundle_key = compute_system_presets_cache_key(profiles_dir); - cache.vendors = vendors; - cache.config_maps = m_config_maps; - cache.filament_id_maps = m_filament_id_maps; - - auto fill = [&](const PresetCollection& coll, - std::vector VendorPresetGroup::*field) { - for (const Preset& p : coll()) - if (p.is_system && p.vendor) - (cache.preset_groups[p.vendor->id].*field).push_back(p); - }; - fill(prints, &VendorPresetGroup::prints); - fill(filaments, &VendorPresetGroup::filaments); - fill(sla_prints, &VendorPresetGroup::sla_prints); - fill(sla_materials, &VendorPresetGroup::sla_materials); - // PrinterPresetCollection is a PresetCollection subclass; iterate directly. - for (const Preset& p : printers()) - if (p.is_system && p.vendor) - cache.preset_groups[p.vendor->id].printers.push_back(p); - - std::ostringstream oss; - { cereal::BinaryOutputArchive ar(oss); ar(cache); } - const std::string blob = oss.str(); - write_cache_blob(output_path, blob); - - SaveCacheResult result; - // Verify: read back and check validity. - if (std::string vblob; read_cache_blob(output_path, vblob)) { - try { - std::istringstream iss(vblob); - cereal::BinaryInputArchive ar(iss); - SystemPresetsCache v; ar(v); - result.ok = v.is_valid(cache.bundle_key); - } catch (...) {} - } - for (const auto& [vid, grp] : cache.preset_groups) { - result.print_presets += grp.prints.size(); - result.filament_presets += grp.filaments.size(); - result.printer_presets += grp.printers.size(); - } - return result; } // static -bool PresetBundle::load_system_presets_cache_for_guide(const std::string& cache_path, - PresetBundle& out_bundle) +std::string PresetBundle::peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name) +{ + try { + boost::nowide::ifstream ifs(cache_path, std::ios::binary); + CacheFileHeader fhdr; + if (! ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr)) || fhdr.magic != CACHE_MAGIC) + return {}; + // Only the head of the body is read, and its CRC left unverified: the stamps + // sit at the front, this answers "what version is this vendor at?" once per + // vendor on every update check, and reading tens of megabytes to do so is not + // worth it. A stamp that comes out garbled fails to parse as a version, which + // is the same answer as none. + std::string head(static_cast(std::min(fhdr.data_size, 1024)), '\0'); + if (! ifs.read(&head[0], static_cast(head.size()))) + return {}; + std::istringstream body(head, std::ios::binary); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + std::string fingerprint, vendor_name, vendor_version; + ar(fingerprint); + ar(vendor_name, vendor_version); + // The fingerprint is deliberately not checked: the version a cache carries + // is what this build installed, whether or not this build can still read it. + if (cache_version != CACHE_VERSION || vendor_name != expected_vendor_name) + return {}; + return vendor_version; + } catch (const std::exception&) { + return {}; + } +} + +bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle) +{ + // Whichever cache answers is judged against the vendor as installed in `dir`: + // the profile there, or — with none, as when the cache is the whole of the + // installation — nothing, since nothing on disk can then be newer than it. + const boost::filesystem::path profile = dir / (vendor_name + ".json"); + const Semver version = boost::filesystem::exists(profile) ? get_version_from_json(profile.string()) + : Semver::inf(); + const boost::filesystem::path rsrc = boost::filesystem::path(resources_dir()) / "profiles"; + return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, base_bundle) + || (dir != rsrc && this->load_vendor_cache((rsrc / (vendor_name + ".opc")).string(), vendor_name, version, base_bundle)); +} + +bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, const PresetBundle* base_bundle) { std::string blob; - if (!read_cache_blob(cache_path, blob)) + if (! read_cache_blob(cache_path, blob)) return false; try { - std::istringstream iss(blob); - cereal::BinaryInputArchive ar(iss); - SystemPresetsCache cache; - ar(cache); - if (cache.cache_version != SystemPresetsCache::CACHE_VERSION) + // Read in place: an istringstream would copy the blob once more just to + // stream over it. + boost::iostreams::stream body(blob.data(), blob.size()); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + if (cache_version != CACHE_VERSION) return false; + std::string fingerprint; + ar(fingerprint); + if (fingerprint != compute_cache_schema_fingerprint()) + return false; + std::string vendor_name, vendor_version; + ar(vendor_name, vendor_version); + if (vendor_name != expected_vendor_name || + ! cache_covers_version(vendor_version, expected_vendor_version)) + return false; + ar(this->vendors); + std::vector process_entries, filament_entries, machine_entries; + ar(process_entries, filament_entries, machine_entries); + uint64_t parse_errors = 0; + ar(parse_errors); - out_bundle.vendors = std::move(cache.vendors); - for (auto& [vendor_id, grp] : cache.preset_groups) { - auto it = out_bundle.vendors.find(vendor_id); - if (it == out_bundle.vendors.end()) continue; - apply_vendor_preset_group(grp, &it->second, - out_bundle.prints, - out_bundle.filaments, - out_bundle.printers, - out_bundle.sla_prints, - out_bundle.sla_materials); + if (this->vendors.find(vendor_name) == this->vendors.end()) + throw std::runtime_error("vendor cache does not carry its own vendor profile"); + + // What the parse counted before install took over; install recounts its + // own below, so m_errors comes out as a JSON parse would leave it. + m_errors += int(parse_errors); + + // Install the entries exactly as load_vendor_configs_from_json installs + // them straight after parsing — same code, same order. The substitution + // context stays empty (the entries were substituted when they were + // parsed), so no substitutions are reported, as before. + ConfigSubstitutionContext substitution_context { ForwardCompatibilitySubstitutionRule::EnableSilent }; + PresetsConfigSubstitutions substitutions; + std::map configs; + std::map filament_id_maps; + const std::string path = boost::filesystem::path(cache_path).parent_path().string(); + size_t count = 0; + auto install_entries = [&](const std::vector& entries, PresetCollection* presets, bool is_from_lib) { + configs.clear(); + filament_id_maps.clear(); + // Only configs of presets that other entries inherit are ever looked + // up again; registering just those skips one full config copy for + // every leaf preset. The library's filaments are all retained — they + // become the m_config_maps other vendors resolve against. + std::set inherited; + for (const CachedPreset& entry : entries) + if (! entry.inherits.empty()) + inherited.insert(entry.inherits); + const std::set* retain_configs = is_from_lib ? nullptr : &inherited; + for (const CachedPreset& entry : entries) { + const std::string reason = install_vendor_preset(entry, path, vendor_name, + base_bundle, LoadConfigBundleAttribute::LoadSystem, substitution_context, substitutions, + configs, filament_id_maps, presets, count, is_from_lib, retain_configs); + if (! reason.empty()) + throw std::runtime_error("entry " + entry.name + " failed to install: " + reason); + } + }; + install_entries(process_entries, &this->prints, false); + const bool is_orca_lib = vendor_name == ORCA_FILAMENT_LIBRARY; + install_entries(filament_entries, &this->filaments, is_orca_lib); + if (is_orca_lib) { + m_config_maps = configs; + m_filament_id_maps = filament_id_maps; } - return !out_bundle.vendors.empty(); + install_entries(machine_entries, &this->printers, false); + return true; } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: guide load failed (" << cache_path << "): " << e.what(); + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); + // Restore a clean state so the caller can fall back to the JSON parse. + this->reset(false); + this->vendors.clear(); + this->m_config_maps.clear(); + this->m_filament_id_maps.clear(); + this->m_errors = 0; + // A failure partway through installing may have left presets in some + // collections with hold aliases already registered. + this->clear_printer_hold_aliases(); return false; } } diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 28ed2ed7df..2a134016a3 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -6,6 +6,7 @@ #include "enum_bitmask.hpp" #include +#include #include #include #include @@ -13,7 +14,6 @@ #include #include - #define DEFAULT_USER_FOLDER_NAME "default" #define BUNDLE_STRUCTURE_JSON_NAME "bundle_structure.json" @@ -171,31 +171,72 @@ struct PresetBundleMetadata class PresetBundle { public: - // ---- System presets single-bundle cache -------------------------------- - // All vendor profiles + all system presets in one file. - // Bundled (CI): resources/profiles/system_presets.cache - // User runtime: data_dir/system/system_presets.cache + // ---- Per-vendor preset cache -------------------------------------------- + // One cache file per vendor (plus the Orca filament library), stamped with + // the vendor's own profile version rather than a directory scan. - struct SaveCacheResult { - bool ok = false; - size_t print_presets = 0; - size_t filament_presets = 0; - size_t printer_presets = 0; + // The cache is not something a caller loads from: a vendor is loaded with + // load_vendor_configs_from_json, which comes from the cache whenever one covers + // it. What is public here is what the cache's own tests drive directly. + + // One preset as its JSON subfile states it: the config diff, the name of the + // preset it inherits, and the parse metadata — everything the parse phase of + // load_vendor_configs_from_json extracts and nothing it derives. Inheritance + // is resolved when the entry is installed, against whatever filament library + // is loaded then, so a cache carries no other vendor's values and no other + // vendor's update can make it stale. + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + struct CachedPreset + { + std::string name; + std::string sub_path; // path under the vendor's directory + DynamicPrintConfig config_src; // the preset's own diff, nothing inherited + std::string inherits; + std::string description; + std::string instantiation; // "true"/"false" as stated; anything else was already counted as a parse error + std::string setting_id; + std::string filament_id; + std::vector renamed_from; + + template + void serialize(Archive& ar) + { + ar(name, sub_path, config_src, inherits, description, instantiation, + setting_id, filament_id, renamed_from); + } }; - static std::string bundled_system_presets_cache_path(); - static std::string user_system_presets_cache_path(); + // Save one vendor (vendor_name at vendor_version): its vendor profile and its + // presets in source form, plus how many errors their parse counted. + static bool save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const VendorMap& vendors, + const std::vector& process_entries, + const std::vector& filament_entries, + const std::vector& machine_entries, + uint64_t parse_errors); - // Capture the currently-loaded PresetBundle and write a single-bundle cache - // to output_path. profiles_dir contains the vendor JSON files (for the key). - // Used by generate_system_cache and tests. - SaveCacheResult save_system_presets_cache(const std::string& profiles_dir, - const std::string& output_path) const; + // Load a validated per-vendor cache into this bundle by installing its + // entries, with base_bundle's filament library as the inheritance base. + // Rejects (returns false, with this bundle left clean) unless the cache + // version, schema fingerprint and vendor name match, the cache was built + // from a vendor profile at least as new as the expected one, and every + // entry installs. An invalid expected version (a profile whose version + // cannot be judged) is never served from cache; Semver::inf() (no profile + // beside the cache at all) accepts whatever is cached. + bool load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const Semver& expected_vendor_version, const PresetBundle* base_bundle = nullptr); - // Load a single-bundle cache into out_bundle for the guide wizard. - // Returns false if the cache is missing, stale, or corrupt. - static bool load_system_presets_cache_for_guide(const std::string& cache_path, - PresetBundle& out_bundle); + // Read the profile version a cache was stamped with, without deserializing its + // presets. Empty if the file is unreadable, not a cache this build understands, + // or not this vendor's. This is how an installed vendor's version is known when + // only its cache is installed. + static std::string peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name); + + // Enable writing a per-vendor cache after a JSON parse (off by default). Cache + // content is pure parse output, so the guard is policy, not correctness: only + // the deliberate generators (load_system_presets_from_json, the cache build + // tool) write files, not every incidental load a dialog performs. + void set_generate_vendor_caches(bool enable) { m_generate_vendor_caches = enable; } static DynamicPrintConfig construct_full_config(Preset &in_printer_preset, Preset &in_print_preset, @@ -471,8 +512,12 @@ public: /*std::pair load_configbundle( const std::string &path, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule);*/ //Orca: load config bundle from json, pass the base bundle to support cross vendor inheritance + // Orca: `dir` is where the vendor is looked for — its own directory, whether or + // not the profile JSONs are still there. A whole-vendor load comes from the + // vendor's preset cache whenever one covers the profile on disk, and is parsed + // from the JSONs (falling back to the ones in resources) only when none does. std::pair load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); // Export a config bundle file containing all the presets and the names of the active presets. //void export_configbundle(const std::string &path, bool export_system_settings = false, bool export_physical_printers = false); @@ -544,19 +589,55 @@ public: // Orca: for validation only. bool has_errors(bool check_duplicate_filament_subtypes = false) const; + // Errors the last load recorded. What the cache's error accounting promises — + // a cache-served vendor reports what its parse would — is pinned against this. + int error_count() const { return m_errors; } + // Orca: for validation only. Flag any system preset whose inherits / compatible_printers / // compatible_prints references a deleted (unknown) or renamed (old) preset name. bool check_preset_references() const; + // Merge one vendor's presets with the other vendor's presets, report duplicates. + // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a + // bundle out of several per-vendor caches loaded into separate PresetBundle instances. + std::vector merge_presets(PresetBundle &&other); + private: - // Compute combined invalidation key from all vendor JSON files in system_dir. - static std::string compute_system_presets_cache_key(const std::string& system_dir); - // Try user cache then bundled cache; apply all presets on hit. - bool try_load_system_presets_from_cache(const std::string& expected_key); + // Load one vendor from its preset cache: the one in `dir`, or — when that is + // missing or stale — the one shipped in resources/profiles, both judged against + // the vendor as installed in `dir`. False, with this bundle left clean, when + // neither is usable and the vendor has to be parsed. This is how + // load_vendor_configs_from_json reads a cache. + bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle); + // Read raw cache blob: verify magic, size, CRC. static bool read_cache_blob(const std::string& path, std::string& out_blob); - // Write a cache blob with the standard 20-byte file header. - static void write_cache_blob(const std::string& path, const std::string& blob); + // Write a cache blob with the standard 20-byte file header. False when the + // file could not be opened or written whole. + static bool write_cache_blob(const std::string& path, const std::string& blob); + + // Install one source-form preset entry into this bundle: resolve `inherits`, + // flatten, validate and register the preset. Returns the reason installation + // failed, empty on success. See the definition for the sharing contract + // between the JSON parse and the cache load. + // retain_configs, when non-null, names the only presets registered into + // config_maps (a full config copy each). The cache load passes the names its + // entries inherit — the only ones ever looked up again; the JSON parse + // retains all, not knowing what later subfiles inherit. + std::string install_vendor_preset(const CachedPreset& entry, + const std::string& path, const std::string& vendor_name, + const PresetBundle* base_bundle, + LoadConfigBundleAttributes flags, + ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions, + std::map& config_maps, std::map& filament_id_maps, + PresetCollection* presets_collection, size_t& count, bool is_from_lib, + const std::set* retain_configs = nullptr); + + // Clear every collection's m_printer_hold_alias, which reset() leaves alone. + void clear_printer_hold_aliases(); + + // Whether to (re)write a per-vendor cache after a JSON parse. + bool m_generate_vendor_caches { false }; // Orca: validation only - flag any printer with two or more compatible // filament presets sharing one filament_id (ambiguous AMS subtype match). @@ -565,8 +646,6 @@ private: //std::pair load_system_presets(ForwardCompatibilitySubstitutionRule compatibility_rule); //BBS: add json related logic std::pair load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule); - // Merge one vendor's presets with the other vendor's presets, report duplicates. - std::vector merge_presets(PresetBundle &&other); // Update the multicolor information for filaments. void update_filament_multi_color(); // Update renamed_from and alias maps of system profiles. @@ -601,6 +680,38 @@ private: ENABLE_ENUM_BITMASK_OPERATORS(PresetBundle::LoadConfigBundleAttribute) +// True if `vendor` is installed in data_dir()/system. A build that ships preset +// caches installs the cache alone, so it — not the profile — marks a vendor +// installed, and either one on its own counts. +extern bool is_vendor_installed(const std::string& vendor); + +// The version of the installed vendor: what its profile claims, or what its cache +// was stamped with where only the cache is installed. Invalid Semver if neither is. +extern Semver installed_vendor_version(const std::string& vendor); + +// Remove every form `vendor` can be installed as from data_dir()/system: its +// profile, its preset cache, and its preset directory. +extern void remove_installed_vendor(const std::string& vendor); + +// The vendors `dir` holds, sorted: one is named by its profile or, in a build that +// ships preset caches instead of the raw profile JSONs, by its cache alone. +extern std::set vendor_names_in(const boost::filesystem::path& dir); + +// The version a build ships `vendor` at: whichever of its preset cache and its +// profile is newer, that being the one installing lays down. Invalid Semver if the +// build ships neither. +extern Semver resource_vendor_version(const std::string& vendor); + +// Install vendors from the resources directory into the data directory, each as +// its preset cache or as its profile and preset JSONs — whichever of the two the +// build ships at the newer version. Anything the previous install of that vendor +// left behind goes, so only the form just installed is there to be loaded. +// bundle_names: vendor names, without extension. +// Returns false on the first vendor that cannot be installed. +extern bool install_vendor_bundles_from_resources(const std::vector& bundle_names, + const std::string& resource_subdir = "profiles", + const std::string& data_subdir = "system"); + } // namespace Slic3r #endif /* slic3r_PresetBundle_hpp_ */ diff --git a/src/libslic3r/Semver.hpp b/src/libslic3r/Semver.hpp index 1e75c57ad0..d3683b4eb8 100644 --- a/src/libslic3r/Semver.hpp +++ b/src/libslic3r/Semver.hpp @@ -197,7 +197,10 @@ public: std::string save_minimal(const Archive&) const { return to_string_sf(); } template void load_minimal(const Archive&, const std::string& s) { - if (auto v = Semver::parse(s)) *this = std::move(*v); + auto v = Semver::parse(s); + if (! v) + throw std::runtime_error("Semver: cannot parse serialized version: " + s); + *this = std::move(*v); } private: diff --git a/src/libslic3r/Utils.hpp b/src/libslic3r/Utils.hpp index 62b2eeb78e..c52a120eff 100644 --- a/src/libslic3r/Utils.hpp +++ b/src/libslic3r/Utils.hpp @@ -722,15 +722,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, std::function filter = nullptr, bool merge_mode = false); -// Install vendor bundles from resources directory to data directory -// bundle_names: vector of vendor bundle names (without .json extension) -// resource_subdir: subdirectory under resources_dir() (default: "profiles") -// data_subdir: subdirectory under data_dir() (default: "system") -// Returns: true if all bundles installed successfully, false otherwise -bool install_vendor_bundles_from_resources(const std::vector& bundle_names, - const std::string& resource_subdir = "profiles", - const std::string& data_subdir = "system"); - // Orca: Since 1.7.9 Boost deprecated save_string_file and load_string_file, copy and modified from boost 1.7.8 void save_string_file(const boost::filesystem::path& p, const std::string& str); void load_string_file(const boost::filesystem::path& p, std::string& str); diff --git a/src/libslic3r/utils.cpp b/src/libslic3r/utils.cpp index 5f429f076a..4f4cc3450d 100644 --- a/src/libslic3r/utils.cpp +++ b/src/libslic3r/utils.cpp @@ -1724,76 +1724,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, return; } -bool install_vendor_bundles_from_resources( - const std::vector& bundle_names, - const std::string& resource_subdir, - const std::string& data_subdir) -{ - namespace fs = boost::filesystem; - - fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; - fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; - - BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; - - for (const auto &bundle : bundle_names) { - try { - // Install the JSON file - auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); - auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); - - if (!fs::exists(path_in_rsrc)) { - BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; - return false; - } - - // Create target directory if needed - if (!fs::exists(vendor_path)) - fs::create_directories(vendor_path); - - // Copy JSON file - std::string error_message; - CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); - if (cfr != CopyFileResult::SUCCESS) { - BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; - return false; - } - - // Copy the vendor directory (if it exists) - auto dir_in_rsrc = rsrc_path / bundle; - auto dir_in_vendors = vendor_path / bundle; - - if (fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { - // Remove existing directory - if (fs::exists(dir_in_vendors)) - fs::remove_all(dir_in_vendors); - fs::create_directories(dir_in_vendors); - - // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) - // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf - auto file_filter = [](const std::string name) -> bool { - return boost::iends_with(name, ".stl") || - boost::iends_with(name, ".png") || - boost::iends_with(name, ".svg") || - boost::iends_with(name, ".jpeg") || - boost::iends_with(name, ".jpg") || - boost::iends_with(name, ".3mf"); - }; - - copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); - } - - BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; - - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); - return false; - } - } - - return true; -} - void save_string_file(const boost::filesystem::path& p, const std::string& str) { boost::nowide::ofstream file; diff --git a/src/slic3r/Config/Snapshot.cpp b/src/slic3r/Config/Snapshot.cpp index 4b071994fc..a7135eac6f 100644 --- a/src/slic3r/Config/Snapshot.cpp +++ b/src/slic3r/Config/Snapshot.cpp @@ -432,14 +432,9 @@ const Snapshot& SnapshotDB::take_snapshot(const AppConfig &app_config, Snapshot: cfg.models_variants_installed.erase(it ++); else ++ it; - // Read the active config bundle, parse the config version. - PresetBundle bundle; - //BBS: change directoties by design - //bundle.load_configbundle((data_dir / PRESET_SYSTEM_DIR / (cfg.name + ".ini")).string(), PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - bundle.load_vendor_configs_from_json((data_dir/PRESET_SYSTEM_DIR).string(), cfg.name, PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - for (const auto &vp : bundle.vendors) - if (vp.second.id == cfg.name) - cfg.version.config_version = vp.second.config_version; + // Orca: the version the vendor is installed at, read from its profile or — + // where the cache is the whole installation — from the cache's own stamp. + cfg.version.config_version = installed_vendor_version(cfg.name); snapshot.vendor_configs.emplace_back(std::move(cfg)); } diff --git a/src/slic3r/GUI/ConfigWizard.cpp b/src/slic3r/GUI/ConfigWizard.cpp index 0bbbc15f87..dba8699105 100644 --- a/src/slic3r/GUI/ConfigWizard.cpp +++ b/src/slic3r/GUI/ConfigWizard.cpp @@ -66,41 +66,41 @@ using Config::SnapshotDB; // Configuration data structures extensions needed for the wizard //BBS: set BBL as default -bool Bundle::load(fs::path source_path, bool ais_in_resources, bool ais_bbl_bundle) +bool Bundle::load(fs::path dir, const std::string &vendor_name, bool ais_in_resources, bool ais_bbl_bundle) { this->preset_bundle = std::make_unique(); this->is_in_resources = ais_in_resources; this->is_bbl_bundle = ais_bbl_bundle; - std::string path_string = source_path.string(); - std::string parent_path = source_path.parent_path().string(); //BBS: add json logic for vendor bundles - std::string vendor_name = source_path.filename().string(); - if (Slic3r::is_json_file(path_string)) { - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - } - else + // Orca: served from the vendor's preset cache where one covers it — which is + // how a shipped build carries its vendors — and parsed from the JSONs otherwise. + // A vendor that can be neither read nor parsed — a cache the build cannot use + // with the preset JSONs behind it pruned, say — is one the wizard cannot offer. + // Every other vendor still can be, so it is left out rather than thrown over. + size_t presets_loaded = 0; + try { + auto [config_substitutions, loaded] = preset_bundle->load_vendor_configs_from_json( + dir.string(), vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + UNUSED(config_substitutions); + // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. + assert(config_substitutions.empty()); + presets_loaded = loaded; + } catch (const std::exception &e) { + BOOST_LOG_TRIVIAL(fatal) << boost::format("Vendor bundle: `%1%`: cannot be loaded, leaving it out: %2%") % vendor_name % e.what(); return false; - - // Throw when parsing invalid configuration. Only valid configuration is supposed to be provided over the air. - //BBS: add json logic for vendor bundles - auto [config_substitutions, presets_loaded] = preset_bundle->load_vendor_configs_from_json( - parent_path, vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); - UNUSED(config_substitutions); - // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. - assert(config_substitutions.empty()); + } auto first_vendor = preset_bundle->vendors.begin(); if (first_vendor == preset_bundle->vendors.end()) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % vendor_name; return false; } if (presets_loaded == 0) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % vendor_name; return false; - } + } - BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % path_string % presets_loaded; + BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % vendor_name % presets_loaded; this->vendor_profile = &first_vendor->second; return true; } @@ -125,15 +125,10 @@ BundleMap BundleMap::load() //Orca: add custom as default //Orca: add json logic for vendor bundle - auto orca_bundle_path = (vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - auto orca_bundle_rsrc = false; - if (!boost::filesystem::exists(orca_bundle_path)) { - orca_bundle_path = (rsrc_vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - orca_bundle_rsrc = true; - } { + const bool from_rsrc = ! is_vendor_installed(PresetBundle::ORCA_DEFAULT_BUNDLE); Bundle bbl_bundle; - if (bbl_bundle.load(std::move(orca_bundle_path), orca_bundle_rsrc, true)) + if (bbl_bundle.load(from_rsrc ? rsrc_vendor_dir : vendor_dir, PresetBundle::ORCA_DEFAULT_BUNDLE, from_rsrc, true)) res.emplace(PresetBundle::ORCA_DEFAULT_BUNDLE, std::move(bbl_bundle)); } @@ -141,18 +136,13 @@ BundleMap BundleMap::load() // and then additionally from resources/profiles. bool is_in_resources = false; for (auto dir : { &vendor_dir, &rsrc_vendor_dir }) { - for (const auto &dir_entry : boost::filesystem::directory_iterator(*dir)) { - //BBS: add json logic for vendor bundle - if (Slic3r::is_json_file(dir_entry.path().string())) { - std::string id = dir_entry.path().stem().string(); // stem() = filename() without the trailing ".json" part + for (const std::string &id : vendor_names_in(*dir)) { + // Don't load this bundle if we've already loaded it. + if (res.find(id) != res.end()) { continue; } - // Don't load this bundle if we've already loaded it. - if (res.find(id) != res.end()) { continue; } - - Bundle bundle; - if (bundle.load(dir_entry.path(), is_in_resources)) - res.emplace(std::move(id), std::move(bundle)); - } + Bundle bundle; + if (bundle.load(*dir, id, is_in_resources)) + res.emplace(id, std::move(bundle)); } is_in_resources = true; diff --git a/src/slic3r/GUI/ConfigWizard_private.hpp b/src/slic3r/GUI/ConfigWizard_private.hpp index 364d378b42..7b9674b216 100644 --- a/src/slic3r/GUI/ConfigWizard_private.hpp +++ b/src/slic3r/GUI/ConfigWizard_private.hpp @@ -71,9 +71,11 @@ struct Bundle Bundle() = default; Bundle(Bundle&& other); + // Load the vendor `vendor_name` as it is installed in `dir`, from its preset + // cache or its profile JSONs, whichever is usable. // Returns false if not loaded. Reason for that is logged as boost::log error. //BBS: set BBL as default - bool load(fs::path source_path, bool is_in_resources, bool is_bbl_bundle = false); + bool load(fs::path dir, const std::string &vendor_name, bool is_in_resources, bool is_bbl_bundle = false); const std::string& vendor_id() const { return vendor_profile->id; } }; diff --git a/src/slic3r/GUI/CreatePresetsDialog.cpp b/src/slic3r/GUI/CreatePresetsDialog.cpp index 1bd80d5f00..33f49c2a38 100644 --- a/src/slic3r/GUI/CreatePresetsDialog.cpp +++ b/src/slic3r/GUI/CreatePresetsDialog.cpp @@ -2201,25 +2201,14 @@ bool CreatePrinterPresetDialog::load_system_and_user_presets_with_curr_model(Pre } else { selected_vendor_id = m_printer_preset_vendor_selected.id; - if (boost::filesystem::exists(boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(); - } else if (boost::filesystem::exists(boost::filesystem::path(Slic3r::resources_dir()) / "profiles" / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::resources_dir()) / "profiles").string(); - } - - if (preset_path.empty()) { - BOOST_LOG_TRIVIAL(info) << "Preset path was not found"; - MessageDialog dlg(this, _L("Preset path was not found; please reselect vendor."), wxString(SLIC3R_APP_FULL_NAME) + " - " + _L("Info"), - wxYES_NO | wxYES_DEFAULT | wxCENTRE); - dlg.ShowModal(); - return false; - } - try { // Pass the app's preset bundle (which already holds OrcaFilamentLibrary) as the base // bundle so vendor filaments that inherit OFL bases resolve via the existing // cross-vendor inheritance path. - temp_preset_bundle.load_vendor_configs_from_json(preset_path, selected_vendor_id, + // Orca: served from the vendor's preset cache where one covers it — a shipped + // build carries that instead of the raw preset JSONs — and parsed otherwise. + temp_preset_bundle.load_vendor_configs_from_json((boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(), + selected_vendor_id, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent, wxGetApp().preset_bundle); diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index 7aa52a8b1c..ea91571001 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -1,6 +1,7 @@ #include "WebGuideDialog.hpp" #include "ConfigWizard.hpp" +#include #include #include #include @@ -43,33 +44,6 @@ using namespace nlohmann; namespace Slic3r { namespace GUI { -static std::string guide_json_cache_version_key(const boost::filesystem::path& rsrc_dir, - const boost::filesystem::path& user_dir) -{ - std::vector parts; - auto collect = [&](const boost::filesystem::path& dir) { - boost::system::error_code ec; - if (!boost::filesystem::exists(dir, ec)) return; - for (const auto& e : boost::filesystem::directory_iterator(dir, ec)) { - if (e.path().extension().string() != ".json") continue; - const std::string k = get_vendor_cache_key(e.path().string()); - if (!k.empty()) - parts.push_back(e.path().filename().string() + "=" + k); - } - }; - collect(rsrc_dir); - if (user_dir != rsrc_dir) collect(user_dir); - std::sort(parts.begin(), parts.end()); - std::string result; - for (const auto& p : parts) { result += p; result += ';'; } - return result; -} - -static boost::filesystem::path guide_json_cache_path() -{ - return boost::filesystem::path(data_dir()) / "guide_profile_cache.json"; -} - static wxString update_custom_filaments() { json m_Res = json::object(); @@ -217,8 +191,7 @@ GuideFrame::GuideFrame(GUI_App *pGUI, long style) GuideFrame::~GuideFrame() { - m_destroy = true; - *m_cancel_token = true; // signal any queued CallAfter lambdas before join + *m_cancel_token = true; // stop the loading thread and any queued CallAfter lambdas before join if (m_load_task && m_load_task->joinable()) m_load_task->join(); m_load_task.reset(); @@ -327,13 +300,19 @@ void GuideFrame::OnNavigationRequest(wxWebViewEvent &evt) /** * Callback invoked when a navigation request was accepted */ -void GuideFrame::init_guide_paths() +// The empty shape every profile-loading path starts from or falls back to. +void GuideFrame::reset_profile_json() { - m_ProfileJson = json::parse("{}"); m_ProfileJson["model"] = json::array(); m_ProfileJson["machine"] = json::object(); m_ProfileJson["filament"] = json::object(); m_ProfileJson["process"] = json::array(); +} + +void GuideFrame::init_guide_paths() +{ + m_ProfileJson = json::parse("{}"); + reset_profile_json(); vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); @@ -376,27 +355,7 @@ void GuideFrame::OnNavigationComplete(wxWebViewEvent &evt) try { init_guide_paths(); if (BuildProfileDataFromPresetBundle()) { - // Persist so future opens that start before preset_bundle is ready - // can skip the slower loading paths. Capture by value so the - // thread is safe even if the dialog closes before it finishes. - boost::thread([data = m_ProfileJson, - rsrc = rsrc_vendor_dir, - user = vendor_dir] { - try { - json cache; - cache["version"] = guide_json_cache_version_key(rsrc, user); - if (cache["version"].get().empty()) return; - json base = data; - for (auto& entry : base["model"]) entry["nozzle_selected"] = ""; - cache["data"] = std::move(base); - boost::nowide::ofstream ofs(guide_json_cache_path().string()); - ofs << cache.dump(-1, ' ', false, json::error_handler_t::ignore); - BOOST_LOG_TRIVIAL(info) << "GuideFrame: guide JSON cache saved"; - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "GuideFrame: failed to save guide JSON cache: " << e.what(); - } - }).detach(); - if (!m_destroy) + if (!*m_cancel_token) on_profile_loaded(); } else { // Presets not yet in memory — delegate to background thread. @@ -858,11 +817,9 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle bool check_unsaved_preset_changes = false; std::vector install_bundles; std::vector remove_bundles; - const auto vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); for (const auto &it : enabled_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir/(it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -873,8 +830,7 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle if (it.second.size() > 0) { if (enabled_vendors.find(it.first) != enabled_vendors.end()) continue; - auto vendor_file = vendor_dir/(it.first + ".json"); - if (fs::exists(vendor_file)) { + if (is_vendor_installed(it.first)) { remove_bundles.emplace_back(it.first); } } @@ -1223,72 +1179,20 @@ int GuideFrame::GetFilamentInfo( std::string VendorDirectory, json & pFilaList, return status; } -bool GuideFrame::TryLoadGuideJsonCache() +bool GuideFrame::BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors) { - const auto path = guide_json_cache_path(); - boost::system::error_code ec; - if (!boost::filesystem::exists(path, ec)) return false; - try { - boost::nowide::ifstream ifs(path.string()); - json cache; - ifs >> cache; - if (!cache.contains("version") || !cache.contains("data")) return false; - const std::string expected = guide_json_cache_version_key(rsrc_vendor_dir, vendor_dir); - if (expected.empty() || cache["version"].get() != expected) return false; - m_ProfileJson = cache["data"]; - if (m_ProfileJson["machine"].empty()) return false; - BOOST_LOG_TRIVIAL(info) << "GuideFrame: loaded profile data from guide JSON cache (" - << m_ProfileJson["model"].size() << " models, " - << m_ProfileJson["machine"].size() << " machines, " - << m_ProfileJson["filament"].size() << " filaments)"; - return true; - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "GuideFrame: guide JSON cache load failed: " << e.what(); - return false; - } -} - -void GuideFrame::SaveGuideJsonCache() -{ - try { - json cache; - cache["version"] = guide_json_cache_version_key(rsrc_vendor_dir, vendor_dir); - if (cache["version"].get().empty()) return; - json base = m_ProfileJson; - // Strip user-specific state — SaveProfileData() re-applies it from AppConfig. - for (auto& entry : base["model"]) - entry["nozzle_selected"] = ""; - cache["data"] = std::move(base); - boost::nowide::ofstream ofs(guide_json_cache_path().string()); - ofs << cache.dump(-1, ' ', false, json::error_handler_t::ignore); - BOOST_LOG_TRIVIAL(info) << "GuideFrame: guide JSON cache saved"; - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "GuideFrame: failed to save guide JSON cache: " << e.what(); - } -} - -bool GuideFrame::BuildProfileDataFromPresetBundle() -{ - PresetBundle* pb = wxGetApp().preset_bundle; - if (!pb || pb->vendors.empty()) - return false; - try { // Models from vendor profiles - for (const auto& [vendor_id, vp] : pb->vendors) { + for (const auto& [vendor_id, vp] : bundle.vendors) { for (const auto& model : vp.models) { std::string nozzle_str; for (const auto& v : model.variants) { if (!nozzle_str.empty()) nozzle_str += ";"; nozzle_str += v.name; } - std::string materials_str; - for (const auto& m : model.default_materials) { - if (!materials_str.empty()) materials_str += ";"; - materials_str += m; - } + const std::string materials_str = boost::algorithm::join(model.default_materials, ";"); boost::filesystem::path cover_path = - (boost::filesystem::path(resources_dir()) / "profiles" / vendor_id / (model.id + "_cover.png")) + (boost::filesystem::path(resources_dir()) / "profiles" / vp.id / (model.id + "_cover.png")) .make_preferred(); if (!boost::filesystem::exists(cover_path)) cover_path = @@ -1298,7 +1202,7 @@ bool GuideFrame::BuildProfileDataFromPresetBundle() json entry; entry["model"] = model.id; entry["name"] = model.name; - entry["vendor"] = vendor_id; + entry["vendor"] = vp.id; entry["nozzle_diameter"] = nozzle_str; entry["materials"] = materials_str; entry["cover"] = cover_path.string(); @@ -1309,8 +1213,8 @@ bool GuideFrame::BuildProfileDataFromPresetBundle() } // Machine map: preset name -> {model, nozzle variant} - for (const Preset& p : pb->printers()) { - if (!p.is_system) continue; + for (const Preset& p : bundle.printers()) { + if (!p.is_system || !p.vendor) continue; const auto* printer_model = p.config.option("printer_model"); const auto* printer_variant = p.config.option("printer_variant"); if (!printer_model || printer_model->value.empty() || !printer_variant) continue; @@ -1322,8 +1226,9 @@ bool GuideFrame::BuildProfileDataFromPresetBundle() } // Filament map from system filament presets (vendor/type already resolved in config) - for (const Preset& p : pb->filaments()) { - if (!p.is_system) continue; + const json& machines = m_ProfileJson["machine"]; + for (const Preset& p : bundle.filaments()) { + if (!p.is_system || !p.vendor) continue; const auto* fila_vendor = p.config.option("filament_vendor"); const auto* fila_type = p.config.option("filament_type"); const auto* compat_printers = p.config.option("compatible_printers"); @@ -1334,9 +1239,10 @@ bool GuideFrame::BuildProfileDataFromPresetBundle() std::string model_list; if (compat_printers) { for (const std::string& pname : compat_printers->values) { - if (m_ProfileJson["machine"].contains(pname)) { - std::string m = m_ProfileJson["machine"][pname]["model"]; - std::string n = m_ProfileJson["machine"][pname]["nozzle"]; + auto it = machines.find(pname); + if (it != machines.end()) { + const std::string m = (*it)["model"]; + const std::string n = (*it)["nozzle"]; model_list += "[" + m + "++" + n + "]"; } } @@ -1353,157 +1259,103 @@ bool GuideFrame::BuildProfileDataFromPresetBundle() } // Process list from visible system print presets - for (const Preset& p : pb->prints()) { - if (!p.is_system || !p.is_visible) continue; + for (const Preset& p : bundle.prints()) { + if (!p.is_system || !p.vendor || !p.is_visible) continue; json entry; entry["name"] = p.name; entry["sub_path"] = p.file; m_ProfileJson["process"].push_back(entry); } - // If rsrc_vendor_dir has vendor JSONs not covered by the current bundle, the - // bundle is incomplete (e.g. dev env where data_dir/system only has - // OrcaFilamentLibrary+Custom). Fall back so LoadProfileFamily reads both dirs. - try { - for (const auto& e : boost::filesystem::directory_iterator(rsrc_vendor_dir)) { - if (e.path().extension().string() != ".json") continue; - const std::string stem = e.path().stem().string(); - if (pb->vendors.find(stem) == pb->vendors.end()) { - BOOST_LOG_TRIVIAL(info) << "GuideFrame: vendor '" << stem - << "' in resources but not in preset_bundle — falling back to JSON loading"; - m_ProfileJson["model"] = json::array(); - m_ProfileJson["machine"] = json::object(); - m_ProfileJson["filament"] = json::object(); - m_ProfileJson["process"] = json::array(); - return false; + if (require_all_resource_vendors) { + // If rsrc_vendor_dir has vendors (profile JSONs, or the preset caches a + // packaged build ships instead) not covered by the current bundle, the + // bundle is incomplete (e.g. dev env where data_dir/system only has + // OrcaFilamentLibrary+Custom). Fall back so the slow path reads both dirs. + try { + for (const std::string& name : vendor_names_in(rsrc_vendor_dir)) { + if (bundle.vendors.find(name) == bundle.vendors.end()) { + BOOST_LOG_TRIVIAL(info) << "GuideFrame: vendor '" << name + << "' in resources but not in preset_bundle — falling back to JSON loading"; + reset_profile_json(); + return false; + } } - } - } catch (const std::exception&) {} + } catch (const std::exception&) {} + } - BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data from preset_bundle (" + BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data (" << m_ProfileJson["model"].size() << " models, " << m_ProfileJson["machine"].size() << " machines, " << m_ProfileJson["filament"].size() << " filaments)"; return !m_ProfileJson["machine"].empty(); } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileDataFromPresetBundle failed: " << e.what() + BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileJson failed: " << e.what() << " — falling back to JSON loading"; - m_ProfileJson["model"] = json::array(); - m_ProfileJson["machine"] = json::object(); - m_ProfileJson["filament"] = json::object(); - m_ProfileJson["process"] = json::array(); + reset_profile_json(); return false; } } -// Builds guide profile JSON from the per-vendor bundled caches -// (resources/profiles/system_presets.cache, generated by CI). -// This avoids the 90-second LoadProfileFamily fallback on first launch. -bool GuideFrame::BuildProfileDataFromBundledCache() +bool GuideFrame::BuildProfileDataFromPresetBundle() { - const std::string cache_path = PresetBundle::bundled_system_presets_cache_path(); - if (!boost::filesystem::exists(cache_path)) + PresetBundle* pb = wxGetApp().preset_bundle; + if (!pb || pb->vendors.empty()) return false; + return BuildProfileJson(*pb, /*require_all_resource_vendors=*/true); +} +bool GuideFrame::BuildProfileDataFromVendors() +{ try { + // Same vendor set and precedence as the JSON scan in LoadProfileData: a + // vendor in the user's system dir shadows the bundled one of that name. + // A vendor is named by its profile or, where a build ships preset caches + // instead, by its cache alone — so both forms name one here. + std::map vendor_files; + auto collect = [&vendor_files](const boost::filesystem::path& dir) { + boost::system::error_code ec; + if (!boost::filesystem::exists(dir, ec)) + return; + for (const auto& e : boost::filesystem::directory_iterator(dir, ec)) + if (Slic3r::is_json_file(e.path().string()) || e.path().extension() == ".opc") + vendor_files.emplace(e.path().stem().string(), e.path()); // first wins + }; + collect(vendor_dir); + collect(rsrc_vendor_dir); + + // Each vendor comes from its preset cache where one covers it, which is what + // makes this worth doing instead of the scan below; the filament library goes + // first because the others' filaments inherit from it, and resolving those on + // the vendors the cache does not cover needs it already loaded. PresetBundle bundle; - if (!PresetBundle::load_system_presets_cache_for_guide(cache_path, bundle)) + auto load_vendor = [this](PresetBundle& into, const std::string& vendor, const PresetBundle* base) { + into.load_vendor_configs_from_json(vendor_dir.string(), vendor, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, base); + }; + const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); + if (vendor_files.count(filament_library)) + load_vendor(bundle, filament_library, nullptr); + for (const auto& entry : vendor_files) { + if (*m_cancel_token) + return false; // as in the scan below: a vendor without a cache is parsed, and that takes time + const std::string& vendor = entry.first; + // A cache is only ever written for a versioned vendor; a JSON has to be + // asked, so that an unversioned one (blacklist.json) carrying no presets + // is passed over. + if (vendor == filament_library || + (entry.second.extension() != ".opc" && ! get_version_from_json(entry.second.string()).valid())) + continue; + PresetBundle tmp; + load_vendor(tmp, vendor, &bundle); + bundle.merge_presets(std::move(tmp)); + } + if (bundle.vendors.empty()) return false; - - for (const auto& [vendor_id, vp] : bundle.vendors) { - for (const auto& cm : vp.models) { - std::string nozzle_str; - for (const auto& v : cm.variants) { - if (!nozzle_str.empty()) nozzle_str += ";"; - nozzle_str += v.name; - } - std::string materials_str; - for (const auto& m : cm.default_materials) { - if (!materials_str.empty()) materials_str += ";"; - materials_str += m; - } - boost::filesystem::path cover_path = - (boost::filesystem::path(resources_dir()) / "profiles" / vp.id / (cm.id + "_cover.png")) - .make_preferred(); - if (!boost::filesystem::exists(cover_path)) - cover_path = - (boost::filesystem::path(resources_dir()) / "web/image/printer" / (cm.id + "_cover.png")) - .make_preferred(); - - json entry; - entry["model"] = cm.id; - entry["name"] = cm.name; - entry["vendor"] = vp.id; - entry["nozzle_diameter"] = nozzle_str; - entry["materials"] = materials_str; - entry["cover"] = cover_path.string(); - entry["nozzle_selected"] = ""; - entry["sub_path"] = ""; - m_ProfileJson["model"].push_back(entry); - } - } - - for (const Preset& cp : bundle.printers()) { - if (!cp.is_system || !cp.vendor) continue; - const auto* pm = cp.config.option("printer_model"); - const auto* pv = cp.config.option("printer_variant"); - if (!pm || pm->value.empty() || !pv) continue; - - json mach; - mach["model"] = pm->value; - mach["nozzle"] = pv->value; - m_ProfileJson["machine"][cp.name] = mach; - } - - for (const Preset& cp : bundle.filaments()) { - if (!cp.is_system || !cp.vendor) continue; - const auto* fv = cp.config.option("filament_vendor"); - const auto* ft = cp.config.option("filament_type"); - const auto* compat = cp.config.option("compatible_printers"); - - std::string vendor = (fv && !fv->values.empty()) ? fv->values[0] : ""; - std::string type = (ft && !ft->values.empty()) ? ft->values[0] : ""; - - std::string model_list; - if (compat) { - for (const std::string& pname : compat->values) { - if (m_ProfileJson["machine"].contains(pname)) { - std::string m = m_ProfileJson["machine"][pname]["model"]; - std::string n = m_ProfileJson["machine"][pname]["nozzle"]; - model_list += "[" + m + "++" + n + "]"; - } - } - } - - json ff; - ff["name"] = cp.name; - ff["sub_path"] = cp.file; - ff["vendor"] = vendor; - ff["type"] = type; - ff["models"] = model_list; - ff["selected"] = 0; - m_ProfileJson["filament"][cp.name] = ff; - } - - for (const Preset& cp : bundle.prints()) { - if (!cp.is_system || !cp.vendor || !cp.is_visible) continue; - json entry; - entry["name"] = cp.name; - entry["sub_path"] = cp.file; - m_ProfileJson["process"].push_back(entry); - } - - BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data from single-bundle cache (" - << m_ProfileJson["model"].size() << " models, " - << m_ProfileJson["machine"].size() << " machines, " - << m_ProfileJson["filament"].size() << " filaments)"; - return !m_ProfileJson["machine"].empty(); - } catch (const std::exception& ex) { - BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileDataFromBundledCache failed: " << ex.what(); - m_ProfileJson["model"] = json::array(); - m_ProfileJson["machine"] = json::object(); - m_ProfileJson["filament"] = json::object(); - m_ProfileJson["process"] = json::array(); + return BuildProfileJson(bundle, /*require_all_resource_vendors=*/false); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << " failed: " << e.what(); + reset_profile_json(); return false; } } @@ -1512,55 +1364,48 @@ int GuideFrame::LoadProfileData() { // Background thread: the fast path in OnNavigationComplete failed (presets not yet loaded). // Loading order (fastest to slowest): - // 1. Guide JSON cache (data_dir/guide_profile_cache.json, sub-second) - // 2. Bundled per-vendor binary caches (CI-generated, ~1-2s) - // 3. Read all vendor JSONs (~90s) - // After paths 2 or 3 the guide JSON cache is written so next open uses path 1. + // 1. Load every vendor, from its preset cache wherever one covers it + // 2. Read all vendor JSONs by hand try { - if (!TryLoadGuideJsonCache()) { - if (!BuildProfileDataFromBundledCache()) { - // Last resort — read all vendor JSONs (~90s) - std::set loaded_vendors; - auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); - if (boost::filesystem::exists(vendor_dir / filament_library_name)) - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); - else - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); - loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); + if (!BuildProfileDataFromVendors()) { + // Last resort — read all vendor JSONs + std::set loaded_vendors; + auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + if (boost::filesystem::exists(vendor_dir / filament_library_name)) + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); + else + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); + loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - boost::filesystem::directory_iterator endIter; - for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } - if (m_destroy) return 0; - } - - boost::filesystem::directory_iterator others_endIter; - for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } - if (m_destroy) return 0; + boost::filesystem::directory_iterator endIter; + for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); } + if (*m_cancel_token) return 0; + } + + boost::filesystem::directory_iterator others_endIter; + for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (*m_cancel_token) return 0; } - // Persist the result so subsequent opens skip both the bundled cache and - // the slow JSON loading path entirely. - SaveGuideJsonCache(); } // Capture the cancel token by value (shared_ptr) so the lambda doesn't diff --git a/src/slic3r/GUI/WebGuideDialog.hpp b/src/slic3r/GUI/WebGuideDialog.hpp index a61e619e71..b9592d03fe 100644 --- a/src/slic3r/GUI/WebGuideDialog.hpp +++ b/src/slic3r/GUI/WebGuideDialog.hpp @@ -84,10 +84,10 @@ public: int LoadProfileFamily(std::string strVendor, std::string strFilePath); void init_guide_paths(); void on_profile_loaded(); + bool BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors); bool BuildProfileDataFromPresetBundle(); - bool BuildProfileDataFromBundledCache(); - bool TryLoadGuideJsonCache(); - void SaveGuideJsonCache(); + bool BuildProfileDataFromVendors(); + void reset_profile_json(); int SaveProfile(); int GetFilamentInfo( std::string VendorDirectory,json & pFilaList, std::string filepath, std::string &sVendor, std::string &sType); @@ -122,9 +122,9 @@ private: //First Load bool bFirstComplete{false}; - std::atomic m_destroy{false}; - // Shared cancel token captured by CallAfter lambdas so they don't touch - // `this` after the destructor has run and the object is freed. + // Set once in the destructor. Read through `this` by the loading thread + // (joined before `this` dies) and captured as the shared_ptr by CallAfter + // lambdas so they don't touch `this` after the object is freed. std::shared_ptr> m_cancel_token{std::make_shared>(false)}; std::unique_ptr m_load_task; diff --git a/src/slic3r/Utils/PresetUpdater.cpp b/src/slic3r/Utils/PresetUpdater.cpp index 18a9db4e26..06808e253d 100644 --- a/src/slic3r/Utils/PresetUpdater.cpp +++ b/src/slic3r/Utils/PresetUpdater.cpp @@ -1044,46 +1044,42 @@ void PresetUpdater::priv::check_installed_vendor_profiles() const std::set bundles; // Orca: always install filament library bundles.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - for (auto &dir_entry : boost::filesystem::directory_iterator(rsrc_path)) { - const auto &path = dir_entry.path(); - std::string file_path = path.string(); - if (is_json_file(file_path)) { - const auto path_in_vendor = vendor_path / path.filename(); - std::string vendor_name = path.filename().string(); - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - if (bundles.find(vendor_name) != bundles.end())continue; + // A vendor is named by its profile or, where the build ships preset caches + // instead of the raw profile JSONs, by its cache alone. + for (const std::string &vendor_name : vendor_names_in(rsrc_path)) { + if (bundles.find(vendor_name) != bundles.end())continue; - const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE - || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); - if (enabled_config_update) { - if ( fs::exists(path_in_vendor)) { - if (is_vendor_enabled) { - Semver resource_ver = get_version_from_json(file_path); - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE + || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); + if (enabled_config_update) { + if (is_vendor_installed(vendor_name)) { + if (is_vendor_enabled) { + // Orca: whichever form of the vendor resources ships at the newer + // version is the one installing lays down, and the one to judge + // what is installed against. + Semver resource_ver = resource_vendor_version(vendor_name); + // Orca: a vendor installed as a preset cache has no profile + // beside it; the version it was installed at is in the cache. + Semver vendor_ver = installed_vendor_version(vendor_name); - if (vendor_ver < resource_ver) { - BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor " << vendor_name << " newer version " - << resource_ver.to_string() << " from resource, old version " << vendor_ver.to_string(); - bundles.insert(vendor_name); - } - } - else { - //need to be removed because not installed - fs::remove(path_in_vendor); - const auto path_of_vendor = vendor_path / vendor_name; - if (fs::exists(path_of_vendor)) - fs::remove_all(path_of_vendor); + if (vendor_ver < resource_ver) { + BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor " << vendor_name << " newer version " + << resource_ver.to_string() << " from resource, old version " << vendor_ver.to_string(); + bundles.insert(vendor_name); } } - else if (is_vendor_enabled) { - bundles.insert(vendor_name); + else { + //need to be removed because not installed + remove_installed_vendor(vendor_name); } } else if (is_vendor_enabled) { bundles.insert(vendor_name); } } + else if (is_vendor_enabled) { + bundles.insert(vendor_name); + } } if (bundles.size() > 0) { @@ -1163,11 +1159,12 @@ Updates PresetUpdater::priv::get_config_updates(const Semver &old_slic3r_version auto filament_in_cache = (cache_profile_path / vendor_name / PRESET_FILAMENT_NAME); auto machine_in_cache = (cache_profile_path / vendor_name / PRESET_PRINTER_NAME); - if (( fs::exists(path_in_vendor)) + if (is_vendor_installed(vendor_name) || fs::exists(print_in_cache) || fs::exists(filament_in_cache) || fs::exists(machine_in_cache)) { - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + // Orca: a vendor installed as a preset cache carries its version there. + Semver vendor_ver = installed_vendor_version(vendor_name); std::map key_values; std::vector keys(3); diff --git a/tests/libslic3r/test_preset_bundle_loading.cpp b/tests/libslic3r/test_preset_bundle_loading.cpp index c697c4461c..0266473b36 100644 --- a/tests/libslic3r/test_preset_bundle_loading.cpp +++ b/tests/libslic3r/test_preset_bundle_loading.cpp @@ -146,7 +146,7 @@ TEST_CASE("Current vendor type tolerates missing printer model", "[Preset][Bundl { PresetBundle bundle; - VendorProfile orca_vendor("ORCA"); + VendorProfile orca_vendor; orca_vendor.id = "ORCA"; VendorProfile::PrinterModel model; model.name = "Orca Test"; orca_vendor.models.emplace_back(model); diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp index ff93fe1b3c..2e2bd3ac13 100644 --- a/tests/libslic3r/test_vendor_cache.cpp +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -8,8 +8,10 @@ #include "libslic3r/PresetBundle.hpp" #include "libslic3r/Preset.hpp" #include "libslic3r/PrintConfig.hpp" +#include "libslic3r/Utils.hpp" using namespace Slic3r; +using Catch::Matchers::WithinAbs; namespace fs = boost::filesystem; namespace { @@ -32,6 +34,85 @@ std::string write_vendor_json(const fs::path& dir, const std::string& vendor_id, return p.string(); } +// One vendor profile with a single process preset beside it, as an install or an +// update lays it down: /.json plus //process/standard.json. +void write_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor + << R"(","process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}]})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; +} + +// A small but complete vendor: one machine model, one process, a non-instantiated +// base filament with an instantiated child inheriting it, a second standalone +// filament carrying explicit metadata, and one machine preset with a rename — so +// the equivalence test below sees every CachedPreset field populated. +void write_full_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + fs::create_directories(dir / vendor / "filament"); + fs::create_directories(dir / vendor / "machine"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor << R"(",)" + << R"("machine_model_list":[{"name":"Test Model","sub_path":"machine/model.json"}],)" + << R"("process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}],)" + << R"("filament_list":[)" + << R"({"name":")" << vendor << R"( Base PLA","sub_path":"filament/base.json"},)" + << R"({"name":")" << vendor << R"( PLA @0.4","sub_path":"filament/pla.json"},)" + << R"({"name":")" << vendor << R"( Silk PLA @0.4","sub_path":"filament/silk.json"}],)" + << R"("machine_list":[{"name":")" << vendor << R"( 0.4 nozzle","sub_path":"machine/printer.json"}]})"; + std::ofstream((dir / vendor / "machine" / "model.json").string()) + << R"({"type":"machine_model","name":"Test Model","nozzle_diameter":"0.4"})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; + std::ofstream((dir / vendor / "filament" / "base.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( Base PLA","from":"system","instantiation":"false","filament_id":"GFA_base","filament_cost":"42"})"; + std::ofstream((dir / vendor / "filament" / "pla.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( PLA @0.4","from":"system","instantiation":"true","filament_id":"GFA00","filament_cost":"20",)" + << R"("setting_id":"GFSA04","description":"Test PLA description"})"; + std::ofstream((dir / vendor / "filament" / "silk.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( Silk PLA @0.4","from":"system","instantiation":"true","inherits":")" << vendor << R"( Base PLA"})"; + std::ofstream((dir / vendor / "machine" / "printer.json").string()) + << R"({"type":"machine","name":")" << vendor + << R"( 0.4 nozzle","from":"system","instantiation":"true","printer_model":"Test Model","printer_variant":"0.4",)" + << R"("renamed_from":")" << vendor << R"( old 0.4 nozzle"})"; +} + +// The filament library: one non-instantiated base filament other vendors inherit +// from. `cost` lets a test bump the library and watch the change flow through. +void write_lib_tree(const fs::path& dir, const std::string& version, const std::string& cost) +{ + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + fs::create_directories(dir / lib / "filament"); + std::ofstream((dir / (lib + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << lib << R"(",)" + << R"("filament_list":[{"name":"Generic PLA","sub_path":"filament/generic_pla.json"}]})"; + std::ofstream((dir / lib / "filament" / "generic_pla.json").string()) + << R"({"type":"filament","name":"Generic PLA","from":"system","instantiation":"false",)" + << R"("filament_id":"GFL99","filament_cost":")" << cost << R"("})"; +} + +// A vendor whose one filament inherits the library's base and states nothing of +// its own — everything it shows comes from the library it is resolved against. +void write_vendor_with_lib_filament(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "filament"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor << R"(",)" + << R"("filament_list":[{"name":")" << vendor << R"( PLA @0.4","sub_path":"filament/pla.json"}]})"; + std::ofstream((dir / vendor / "filament" / "pla.json").string()) + << R"({"type":"filament","name":")" << vendor + << R"( PLA @0.4","from":"system","instantiation":"true","inherits":"Generic PLA"})"; +} + std::string write_versionless_vendor_json(const fs::path& dir, const std::string& vendor_id) { const fs::path p = dir / (vendor_id + ".json"); @@ -40,6 +121,14 @@ std::string write_versionless_vendor_json(const fs::path& dir, const std::string return p.string(); } +// Whole file as bytes, for the byte-identity comparisons below. +std::string slurp(const fs::path& p) +{ + std::string s; + load_string_file(p, s); + return s; +} + void corrupt_blob_byte(const std::string& path) { std::fstream f(path, std::ios::in | std::ios::out | std::ios::binary); @@ -50,8 +139,8 @@ void corrupt_blob_byte(const std::string& path) f.write(&b, 1); } -// Patch cache_version (blob[0..3]) and recompute CRC so the file passes -// the CRC check but fails the cache_version check in load_system_presets_cache_for_guide. +// Patch cache_version (blob[0..3], i.e. file offset 20) and recompute CRC so the +// file passes the CRC check but fails the cache_version check in load_vendor_cache. void patch_cache_version(const std::string& path, uint32_t wrong_version) { std::ifstream in(path, std::ios::binary); @@ -67,31 +156,94 @@ void patch_cache_version(const std::string& path, uint32_t wrong_version) out.write(data.data(), static_cast(data.size())); } -void add_vendor(PresetBundle& bundle, const std::string& vendor_id, - const std::string& name = "", Semver ver = Semver(1, 0, 0)) +// Truncates the cache's PAYLOAD (everything after the 20-byte header) by +// `truncate_by` bytes and recomputes data_size/crc32 in the header, exactly +// as write_cache_blob computes them, so read_cache_blob's size and CRC checks +// still pass but cereal runs out of bytes partway through deserializing the +// body — exercising load_vendor_cache's catch block instead of its early +// (pre-body) rejection paths. +void truncate_payload_and_fix_header(const std::string& path, size_t truncate_by) { + constexpr size_t header_size = 20; // magic(4) + version(4) + data_size(8) + crc32(4) + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + REQUIRE(data.size() > header_size + truncate_by); + const size_t new_payload_size = data.size() - header_size - truncate_by; + const uint64_t data_size_field = static_cast(new_payload_size); + boost::crc_32_type crc; + crc.process_bytes(&data[header_size], new_payload_size); + const uint32_t crc_field = crc.checksum(); + std::memcpy(&data[8], &data_size_field, sizeof(data_size_field)); // data_size offset + std::memcpy(&data[16], &crc_field, sizeof(crc_field)); // crc32 offset + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(header_size + new_payload_size)); +} + +// One vendor as a cache's VendorMap. It carries one printer model ("Test Model", +// variant "0.4") so machine entries can pass install's model/variant validation. +VendorMap one_vendor(const std::string& vendor_id, const std::string& name = "", + Semver ver = Semver(1, 0, 0)) +{ + VendorMap vendors; VendorProfile vp(vendor_id); vp.name = name.empty() ? vendor_id + " Corp" : name; vp.config_version = ver; - bundle.vendors.emplace(vendor_id, vp); + VendorProfile::PrinterModel model; + model.id = "Test Model"; + model.variants.emplace_back(VendorProfile::PrinterVariant("0.4")); + vp.models.push_back(model); + vendors.emplace(vendor_id, vp); + return vendors; } -Preset& add_system_preset(PresetCollection& coll, const std::string& name, - const VendorProfile* vp) +// Source-form entries as parse_subfile would emit them. The alias is derived by +// install from the '@' in the name, exactly as it is for the JSON parse. +PresetBundle::CachedPreset filament_entry(const std::string& name, const std::string& filament_id = "GFA00", + const std::string& inherits = "") { - Preset& p = coll.load_preset("", name, DynamicPrintConfig(coll.default_preset().config), false); - p.is_system = true; - p.vendor = vp; - return p; + PresetBundle::CachedPreset e; + e.name = name; + e.sub_path = "filament/" + name + ".json"; + e.instantiation = "true"; + e.filament_id = filament_id; + e.inherits = inherits; + return e; } -PresetBundle::SaveCacheResult save_one_vendor(PresetBundle& src, - const fs::path& profiles_dir, - const fs::path& cache_path) +PresetBundle::CachedPreset printer_entry(const std::string& name) { - return src.save_system_presets_cache(profiles_dir.string(), cache_path.string()); + PresetBundle::CachedPreset e; + e.name = name; + e.sub_path = "machine/" + name + ".json"; + e.instantiation = "true"; + e.config_src.set_key_value("printer_model", new ConfigOptionString("Test Model")); + e.config_src.set_key_value("printer_variant", new ConfigOptionString("0.4")); + return e; } +static bool save_one_vendor(const std::string& path, const VendorMap& vendors, + const std::string& vendor, const std::string& vendor_version, + const std::vector& filament_entries = {}, + const std::vector& machine_entries = {}, + const std::vector& process_entries = {}) +{ + return PresetBundle::save_vendor_cache(path, vendor, vendor_version, vendors, + process_entries, filament_entries, machine_entries, 0); +} + +// resources_dir()/data_dir() are process-wide, so restore them however the test +// leaves — including through a failed REQUIRE — to stay green under --order rand. +struct ScopedDirs { + std::string prev_data{data_dir()}, prev_rsrc{resources_dir()}; + ScopedDirs(const fs::path& data, const fs::path& rsrc) + { + set_data_dir(data.string()); + set_resources_dir(rsrc.string()); + } + ~ScopedDirs() { set_data_dir(prev_data); set_resources_dir(prev_rsrc); } +}; + // Helper: filter a collection by vendor_id. std::vector presets_for(const PresetCollection& coll, const std::string& vendor_id) { @@ -104,30 +256,57 @@ std::vector presets_for(const PresetCollection& coll, const std:: } // namespace -TEST_CASE("SystemPresetsCache: save and load via guide API", "[VendorCache]") +namespace Slic3r { +inline bool operator==(const VendorProfile::PrinterVariant& a, const VendorProfile::PrinterVariant& b) { return a.name == b.name; } +inline bool operator==(const VendorProfile::PrinterModel& a, const VendorProfile::PrinterModel& b) +{ + return a.id == b.id && a.name == b.name && a.model_id == b.model_id && a.technology == b.technology + && a.family == b.family && a.variants == b.variants && a.default_materials == b.default_materials + && a.not_support_bed_types == b.not_support_bed_types && a.bed_model == b.bed_model + && a.bed_texture == b.bed_texture && a.image_bed_type == b.image_bed_type + && a.bottom_texture_end_name == b.bottom_texture_end_name + && a.use_double_extruder_default_texture == b.use_double_extruder_default_texture + && a.bottom_texture_rect == b.bottom_texture_rect + && a.bottom_texture_rect_longer == b.bottom_texture_rect_longer + && a.middle_texture_rect == b.middle_texture_rect && a.hotend_model == b.hotend_model; +} +} // namespace Slic3r + +static bool vendor_deep_equal(const VendorProfile& a, const VendorProfile& b) +{ + return a.name == b.name && a.id == b.id && a.config_version == b.config_version + && a.config_update_url == b.config_update_url && a.changelog_url == b.changelog_url + && a.models == b.models && a.default_filaments == b.default_filaments + && a.default_sla_materials == b.default_sla_materials; +} + +static bool preset_deep_equal(const Preset& a, const Preset& b) +{ + return a.type == b.type && a.is_default == b.is_default && a.is_external == b.is_external + && a.is_system == b.is_system && a.is_visible == b.is_visible && a.is_dirty == b.is_dirty + && a.is_compatible == b.is_compatible && a.is_project_embedded == b.is_project_embedded + && a.name == b.name && a.file == b.file && a.loaded == b.loaded + && a.config.equals(b.config) + && a.alias == b.alias && a.renamed_from == b.renamed_from + && a.m_excluded_from == b.m_excluded_from && a.m_from_orca_filament_lib == b.m_from_orca_filament_lib + && a.bundle_id == b.bundle_id && a.version == b.version && a.ini_str == b.ini_str + && a.setting_id == b.setting_id && a.filament_id == b.filament_id && a.user_id == b.user_id + && a.base_id == b.base_id && a.sync_info == b.sync_info && a.description == b.description + && a.updated_time == b.updated_time && a.key_values == b.key_values; +} + +TEST_CASE("a saved cache loads back with names, aliases and filament ids intact", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - const VendorProfile* vp = &src.vendors.at(vid); - - Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", vp); - fp.alias = "Acme PLA"; - fp.filament_id = "GFL_acme_pla"; - add_system_preset(src.printers, vid + " Printer 0.4", vp); - - const auto stats = save_one_vendor(src, tmp.path, cache); - REQUIRE(stats.ok); - CHECK(stats.filament_presets == 1); - CHECK(stats.printer_presets == 1); - CHECK(stats.print_presets == 0); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA @0.4", "GFL_acme_pla")}, + {printer_entry(vid + " Printer 0.4")})); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); REQUIRE(out.vendors.count(vid) == 1); auto fi = presets_for(out.filaments, vid); @@ -140,73 +319,58 @@ TEST_CASE("SystemPresetsCache: save and load via guide API", "[VendorCache]") CHECK(pr[0]->name == vid + " Printer 0.4"); } -TEST_CASE("SystemPresetsCache: missing file returns false", "[VendorCache]") +TEST_CASE("loading a missing cache file returns false", "[VendorCache]") { TempDir tmp; PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide( - (tmp.path / "nonexistent.cache").string(), out)); + REQUIRE(!out.load_vendor_cache((tmp.path / "nonexistent.opc").string(), "Acme", Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: corrupt data rejected by CRC check", "[VendorCache]") +TEST_CASE("a cache with a corrupted byte is rejected by the CRC check", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); - save_one_vendor(src, tmp.path, cache); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); corrupt_blob_byte(cache.string()); PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: multiple vendors in one bundle are isolated", "[VendorCache]") +TEST_CASE("two vendors produce two independent cache files", "[VendorCache]") { TempDir tmp; - const std::string vid1 = "VendorA"; - const std::string vid2 = "VendorB"; - write_vendor_json(tmp.path, vid1); - write_vendor_json(tmp.path, vid2); - const fs::path cache = tmp.path / "system_presets.cache"; + const fs::path cacheA = tmp.path / "vendorA.opc"; + const fs::path cacheB = tmp.path / "vendorB.opc"; - PresetBundle src; - for (const auto& vid : {vid1, vid2}) { - add_vendor(src, vid); - const VendorProfile* vp = &src.vendors.at(vid); - add_system_preset(src.filaments, vid + " PLA", vp); - add_system_preset(src.printers, vid + " Printer", vp); - } + REQUIRE(save_one_vendor(cacheA.string(), one_vendor("VendorA"), "VendorA", "1.0.0", + {filament_entry("VendorA PLA")})); + REQUIRE(save_one_vendor(cacheB.string(), one_vendor("VendorB"), "VendorB", "1.0.0", + {filament_entry("VendorB PLA")})); - REQUIRE(save_one_vendor(src, tmp.path, cache).ok); + // Corrupt only vendor B's file; vendor A's must be unaffected. + corrupt_blob_byte(cacheB.string()); - PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); - REQUIRE(out.vendors.count(vid1) == 1); - REQUIRE(out.vendors.count(vid2) == 1); + PresetBundle outA; + REQUIRE(outA.load_vendor_cache(cacheA.string(), "VendorA", Semver("1.0.0"))); + REQUIRE(outA.vendors.count("VendorA") == 1); + REQUIRE(presets_for(outA.filaments, "VendorA").size() == 1); - for (const auto& vid : {vid1, vid2}) { - auto fi = presets_for(out.filaments, vid); - auto pr = presets_for(out.printers, vid); - REQUIRE(fi.size() == 1); - CHECK(fi[0]->name == vid + " PLA"); - REQUIRE(pr.size() == 1); - CHECK(pr[0]->name == vid + " Printer"); - } + PresetBundle outB; + REQUIRE(!outB.load_vendor_cache(cacheB.string(), "VendorB", Semver("1.0.0"))); + REQUIRE(outB.vendors.empty()); } -TEST_CASE("SystemPresetsCache: vendor profile fields are preserved", "[VendorCache]") +TEST_CASE("vendor profile fields survive a cache round-trip", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid, "2.5.1"); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; + VendorMap vendors; VendorProfile vp(vid); vp.name = "Acme Corporation"; vp.config_version = Semver(2, 5, 1); @@ -216,13 +380,16 @@ TEST_CASE("SystemPresetsCache: vendor profile fields are preserved", "[VendorCac VendorProfile::PrinterVariant v0_4; v0_4.name = "0.4"; model.variants.push_back(v0_4); vp.models.push_back(model); - src.vendors.emplace(vid, vp); - save_one_vendor(src, tmp.path, cache); + vendors.emplace(vid, vp); + REQUIRE(save_one_vendor(cache.string(), vendors, vid, "2.5.1")); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("2.5.1"))); REQUIRE(out.vendors.count(vid) == 1); const VendorProfile& gvp = out.vendors.at(vid); + REQUIRE(vendor_deep_equal(gvp, vendors.at(vid))); + // Spot-check the fields the old test asserted directly, so a + // vendor_deep_equal regression still points at what actually broke. CHECK(gvp.id == vid); CHECK(gvp.name == "Acme Corporation"); REQUIRE(gvp.models.size() == 1); @@ -232,21 +399,18 @@ TEST_CASE("SystemPresetsCache: vendor profile fields are preserved", "[VendorCac CHECK(gvp.models[0].variants[0].name == "0.4"); } -TEST_CASE("SystemPresetsCache: config option values are preserved", "[VendorCache]") +TEST_CASE("config option values survive a cache round-trip", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - Preset& fp = add_system_preset(src.filaments, vid + " PETG @0.4", &src.vendors.at(vid)); - fp.config.set_key_value("filament_type", new ConfigOptionStrings({"PETG"})); - save_one_vendor(src, tmp.path, cache); + auto entry = filament_entry(vid + " PETG @0.4"); + entry.config_src.set_key_value("filament_type", new ConfigOptionStrings({"PETG"})); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", {entry})); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); auto fi = presets_for(out.filaments, vid); REQUIRE(fi.size() == 1); @@ -256,29 +420,23 @@ TEST_CASE("SystemPresetsCache: config option values are preserved", "[VendorCach CHECK(ft->values[0] == "PETG"); } -TEST_CASE("SystemPresetsCache: multiple presets per collection round-trip", "[VendorCache]") +TEST_CASE("multiple presets in one collection all round-trip", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; - - PresetBundle src; - add_vendor(src, vid); - const VendorProfile* vp = &src.vendors.at(vid); + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; const std::vector fi_names = {vid + " PLA", vid + " PETG", vid + " ABS"}; const std::vector pr_names = {vid + " Printer 0.4", vid + " Printer 0.6"}; - for (const auto& n : fi_names) add_system_preset(src.filaments, n, vp); - for (const auto& n : pr_names) add_system_preset(src.printers, n, vp); + std::vector filament_entries, machine_entries; + for (const auto& n : fi_names) filament_entries.push_back(filament_entry(n)); + for (const auto& n : pr_names) machine_entries.push_back(printer_entry(n)); - const auto stats = save_one_vendor(src, tmp.path, cache); - REQUIRE(stats.ok); - CHECK(stats.filament_presets == 3); - CHECK(stats.printer_presets == 2); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + filament_entries, machine_entries)); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); auto fi = presets_for(out.filaments, vid); auto pr = presets_for(out.printers, vid); @@ -292,30 +450,27 @@ TEST_CASE("SystemPresetsCache: multiple presets per collection round-trip", "[Ve for (const auto& n : pr_names) CHECK(pr_got.count(n) == 1); } -TEST_CASE("SystemPresetsCache: truncated file is rejected", "[VendorCache]") +TEST_CASE("a truncated cache file is rejected", "[VendorCache]") { TempDir tmp; - const fs::path cache = tmp.path / "truncated.cache"; + const fs::path cache = tmp.path / "truncated.opc"; { std::ofstream f(cache.string(), std::ios::binary); const char data[] = {0x4F, 0x52, 0x43}; f.write(data, sizeof(data)); } PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: wrong magic is rejected", "[VendorCache]") +TEST_CASE("a cache with the wrong magic number is rejected", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); - save_one_vendor(src, tmp.path, cache); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); { std::fstream f(cache.string(), std::ios::in | std::ios::out | std::ios::binary); @@ -324,30 +479,19 @@ TEST_CASE("SystemPresetsCache: wrong magic is rejected", "[VendorCache]") } PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: vendor with no presets saves and loads cleanly", "[VendorCache]") +TEST_CASE("a vendor with no presets saves and loads cleanly", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - VendorProfile vp(vid); - vp.name = "Acme Corporation"; - vp.config_version = Semver(1, 0, 0); - src.vendors.emplace(vid, vp); - - const auto stats = save_one_vendor(src, tmp.path, cache); - REQUIRE(stats.ok); - CHECK(stats.print_presets == 0); - CHECK(stats.filament_presets == 0); - CHECK(stats.printer_presets == 0); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid, "Acme Corporation"), vid, "1.0.0")); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); REQUIRE(out.vendors.count(vid) == 1); CHECK(out.vendors.at(vid).id == vid); CHECK(out.vendors.at(vid).name == "Acme Corporation"); @@ -356,74 +500,151 @@ TEST_CASE("SystemPresetsCache: vendor with no presets saves and loads cleanly", CHECK(presets_for(out.prints, vid).empty()); } -TEST_CASE("SystemPresetsCache: all Preset metadata fields are preserved", "[VendorCache]") +TEST_CASE("a cache-loaded vendor is indistinguishable from a JSON-loaded one", "[VendorCache]") { - TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_full_vendor_tree(user, "Acme", "1.0.0"); - PresetBundle src; - add_vendor(src, vid); - Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", &src.vendors.at(vid)); - fp.setting_id = "sid-test-001"; - fp.description = "A test filament preset"; - fp.bundle_id = "bundle-xyz"; - fp.user_id = "user-abc"; - fp.base_id = "base-123"; - fp.sync_info = "update"; - fp.updated_time = 1700000000LL; - fp.key_values = {{"color", "red"}, {"diameter", "1.75"}}; - fp.ini_str = "[filament]\nnozzle_temperature = 230\n"; - save_one_vendor(src, tmp.path, cache); + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); - PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); - auto fi = presets_for(out.filaments, vid); - REQUIRE(fi.size() == 1); - CHECK(fi[0]->setting_id == "sid-test-001"); - CHECK(fi[0]->description == "A test filament preset"); - CHECK(fi[0]->bundle_id == "bundle-xyz"); - CHECK(fi[0]->user_id == "user-abc"); - CHECK(fi[0]->base_id == "base-123"); - CHECK(fi[0]->sync_info == "update"); - CHECK(fi[0]->updated_time == 1700000000LL); - REQUIRE(fi[0]->key_values.count("color") == 1); - CHECK(fi[0]->key_values.at("color") == "red"); - REQUIRE(fi[0]->key_values.count("diameter") == 1); - CHECK(fi[0]->key_values.at("diameter") == "1.75"); - CHECK(fi[0]->ini_str == "[filament]\nnozzle_temperature = 230\n"); + // Take the preset JSONs away: were the cache rejected, the load below would + // have nothing to parse — so its success proves the cache answered. + fs::remove_all(user / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + // Both paths run the same install code over the same entries, so everything + // observable must come out identical — the vendor profile and every preset, + // field by field. + REQUIRE(from_cache.vendors.count("Acme") == 1); + REQUIRE(vendor_deep_equal(from_cache.vendors.at("Acme"), from_json.vendors.at("Acme"))); + const std::pair colls[] = { + {&from_json.prints, &from_cache.prints}, + {&from_json.filaments, &from_cache.filaments}, + {&from_json.printers, &from_cache.printers}, + }; + for (const auto& [jc, cc] : colls) { + auto a = presets_for(*jc, "Acme"); + auto b = presets_for(*cc, "Acme"); + REQUIRE(a.size() == b.size()); + REQUIRE(!a.empty()); + for (size_t i = 0; i < a.size(); ++i) { + CHECK(a[i]->name == b[i]->name); + CHECK(preset_deep_equal(*a[i], *b[i])); + } + } + + // Pin the explicit metadata against symmetric loss: dropping a field from + // CachedPreset::serialize keeps the two bundles equal to each other, but not + // to the fixture. + const Preset* pla = from_cache.filaments.find_preset("Acme PLA @0.4", false); + REQUIRE(pla != nullptr); + CHECK(pla->setting_id == "GFSA04"); + CHECK(pla->description == "Test PLA description"); + const Preset* silk = from_cache.filaments.find_preset("Acme Silk PLA @0.4", false); + REQUIRE(silk != nullptr); + CHECK(silk->filament_id == "GFA_base"); // inherited from the non-instantiated base + const auto* cost = silk->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(42., 1e-9)); + const Preset* pr = from_cache.printers.find_preset("Acme 0.4 nozzle", false); + REQUIRE(pr != nullptr); + CHECK(pr->renamed_from == std::vector{"Acme old 0.4 nozzle"}); } -TEST_CASE("SystemPresetsCache: wrong cache_version is rejected", "[VendorCache]") +TEST_CASE("a cache-served vendor reports the errors its parse counted", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + // One process preset without the required "instantiation" key — a parse-phase + // error the load survives, so it must reach the cache's parse_errors stamp. + fs::create_directories(user / "Acme" / "process"); + std::ofstream((user / "Acme.json").string()) + << R"({"version":"1.0.0","name":"Acme","process_list":[{"name":"0.20mm Standard @Acme","sub_path":"process/standard.json"}]})"; + std::ofstream((user / "Acme" / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @Acme","from":"system","layer_height":"0.2"})"; + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle from_json; + from_json.set_generate_vendor_caches(true); + from_json.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); + CHECK(from_json.error_count() > 0); + + fs::remove_all(user / "Acme"); + PresetBundle from_cache; + from_cache.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(from_cache.error_count() == from_json.error_count()); + CHECK(presets_for(from_cache.prints, "Acme").size() == 1); +} + +TEST_CASE("a non-instantiated base in a regular vendor's cache resolves its children and stays out of the library maps", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + + // Entry order is the resolution order: the base must install (into the local + // config maps) before the child that inherits it. + auto base = filament_entry("Acme Base PLA", "GFA_base"); + base.instantiation = "false"; + base.config_src.set_key_value("filament_cost", new ConfigOptionFloats({42.})); + auto child = filament_entry("Acme Silk PLA @0.4", "", "Acme Base PLA"); + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0", {base, child})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + auto fi = presets_for(out.filaments, "Acme"); + REQUIRE(fi.size() == 1); // the base never becomes a preset + CHECK(fi[0]->name == "Acme Silk PLA @0.4"); + CHECK(fi[0]->filament_id == "GFA_base"); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(42., 1e-9)); + // Only the filament library's bases persist as the cross-vendor inheritance + // maps; a regular vendor's stay local to its own load. + CHECK(out.m_config_maps.empty()); + CHECK(out.m_filament_id_maps.empty()); +} + +TEST_CASE("a cache with the wrong cache version is rejected", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); - save_one_vendor(src, tmp.path, cache); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); patch_cache_version(cache.string(), 0xFFFFFFFFu); PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: mid-blob truncation is rejected", "[VendorCache]") +TEST_CASE("a cache truncated mid-blob is rejected", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); - save_one_vendor(src, tmp.path, cache); + REQUIRE(save_one_vendor(cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA")})); { std::ifstream in(cache.string(), std::ios::binary); @@ -435,24 +656,559 @@ TEST_CASE("SystemPresetsCache: mid-blob truncation is rejected", "[VendorCache]" } PresetBundle out; - CHECK_FALSE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); + REQUIRE(!out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); } -TEST_CASE("SystemPresetsCache: versionless vendor uses mtime key in bundle", "[VendorCache]") +TEST_CASE("printer model bed texture fields survive a cache round-trip", "[VendorCache]") { TempDir tmp; - const std::string vid = "Acme"; - write_versionless_vendor_json(tmp.path, vid); - const fs::path cache = tmp.path / "system_presets.cache"; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; - PresetBundle src; - add_vendor(src, vid); - add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); - REQUIRE(save_one_vendor(src, tmp.path, cache).ok); + VendorMap vendors = one_vendor(vid); + VendorProfile::PrinterModel model; + model.id = "N1"; + model.name = "Neat One"; + model.bottom_texture_rect_longer = "5,5,50,10"; + vendors.at(vid).models.push_back(model); + REQUIRE(save_one_vendor(cache.string(), vendors, vid, "1.0.0")); PresetBundle out; - REQUIRE(PresetBundle::load_system_presets_cache_for_guide(cache.string(), out)); - auto fi = presets_for(out.filaments, vid); - REQUIRE(fi.size() == 1); - CHECK(fi[0]->name == vid + " PLA"); + REQUIRE(out.load_vendor_cache(cache.string(), vid, Semver("1.0.0"))); + REQUIRE(out.vendors.at(vid).models.size() == 2); + REQUIRE(vendor_deep_equal(out.vendors.at(vid), vendors.at(vid))); + CHECK(out.vendors.at(vid).models[1].bottom_texture_rect_longer == "5,5,50,10"); +} + +TEST_CASE("a cache older than the vendor profile on disk is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.1"))); +} + +TEST_CASE("a cache newer than the vendor profile on disk is used", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.2.0", + {filament_entry("Acme PLA")})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + CHECK(presets_for(out.filaments, "Acme").size() == 1); +} + +TEST_CASE("a vendor cache outlives a filament library update and resolves against the new library", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(user, "1.0.0", "20"); + write_vendor_with_lib_filament(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // First launch: the library parses first, then the vendor against it, and + // both caches are written. + PresetBundle base1; + base1.set_generate_vendor_caches(true); + base1.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme1; + acme1.set_generate_vendor_caches(true); + acme1.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base1); + REQUIRE(fs::exists(user / "Acme.opc")); + { + auto fi = presets_for(acme1.filaments, "Acme"); + REQUIRE(fi.size() == 1); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(20., 1e-9)); + } + + // An update delivers a new library only; the vendor stays as it was. + write_lib_tree(user, "2.0.0", "30"); + PresetBundle base2; + base2.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + + // Take the vendor's preset JSONs away: were its cache rejected, the load + // below would have nothing to parse — so its success proves the cache + // survived the library bump. + fs::remove_all(user / "Acme"); + PresetBundle acme2; + auto [substitutions, presets_loaded] = acme2.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base2); + CHECK(presets_loaded == 1); + auto fi = presets_for(acme2.filaments, "Acme"); + REQUIRE(fi.size() == 1); + // The cache holds only the vendor's own diff; the library values come from + // the library loaded now, not the one in effect when the cache was written. + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); + CHECK(fi[0]->filament_id == "GFL99"); +} + +TEST_CASE("a vendor installed as its cache alone still loads after a library update", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + write_lib_tree(user, "1.0.0", "20"); + write_vendor_with_lib_filament(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Generate the vendor's cache, then strip the vendor to the cache alone — + // the shape of a packaged install, which ships each vendor as its .opc and + // nothing else. + PresetBundle base1; + base1.set_generate_vendor_caches(true); + base1.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme1; + acme1.set_generate_vendor_caches(true); + acme1.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base1); + fs::remove(user / "Acme.json"); + fs::remove_all(user / "Acme"); + + // An OTA update then delivers a new library only. With no JSONs anywhere to + // fall back on, the vendor must keep loading from its cache. + write_lib_tree(user, "2.0.0", "30"); + PresetBundle base2; + base2.load_vendor_configs_from_json(user.string(), lib, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + PresetBundle acme2; + auto [substitutions, presets_loaded] = acme2.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, &base2); + CHECK(presets_loaded == 1); + REQUIRE(acme2.vendors.count("Acme") == 1); + auto fi = presets_for(acme2.filaments, "Acme"); + REQUIRE(fi.size() == 1); + const auto* cost = fi[0]->config.option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(30., 1e-9)); +} + +TEST_CASE("a cache entry whose parent is missing falls back to the vendor's JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_tree(user, "Acme", "1.0.0"); + + // A cache claiming the installed version, but whose entry inherits a preset + // no loaded library provides. + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4", "GFA00", "No Such Base")})); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Directly: the load fails and leaves the bundle clean. + PresetBundle direct; + REQUIRE(!direct.load_vendor_cache((user / "Acme.opc").string(), "Acme", Semver("1.0.0"))); + CHECK(direct.vendors.empty()); + + // Through the vendor load: the JSONs answer instead, as if no cache existed. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").name == "Acme"); // the profile's name, not the cache's +} + +TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + + PresetBundle out; + // An unversioned vendor profile has no version to compare against. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver::invalid())); + // And a cache carrying no version of its own cannot cover a profile that has one. + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "")); + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver("1.0.0"))); + REQUIRE(out.vendors.empty()); +} + +TEST_CASE("a versionless profile beside a cache keeps the cache from being served", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4")})); + // The profile beside the cache parses to no usable version, which can no + // more judge the cache's staleness than it could be cached itself. + write_versionless_vendor_json(user, "Acme"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + // Nothing came from the cache: the versionless profile was parsed instead, + // and it carries no presets. + CHECK(presets_loaded == 0); +} + +TEST_CASE("a vendor's cache is its whole installation", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources"; + const fs::path data = tmp.path / "data"; + fs::create_directories(rsrc / "profiles" / "Acme" / "machine"); + write_vendor_json(rsrc / "profiles", "Acme"); + std::ofstream((rsrc / "profiles" / "Acme" / "machine" / "printer.json").string()) << "{}"; + + REQUIRE(save_one_vendor((rsrc / "profiles" / "Acme.opc").string(), one_vendor("Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(data, rsrc); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + // The cache carries the presets, the vendor profile and the version they were + // built at, so it is installed on its own. + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); + CHECK(is_vendor_installed("Acme")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // A vendor with no cache is installed as its profile and preset JSONs instead, + // parsing them being the only way left to load it — and the cache the previous + // install left behind has to go, or it would shadow the profile just installed. + fs::remove(rsrc / "profiles" / "Acme.opc"); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(!fs::exists(data / "system" / "Acme.opc")); + CHECK(fs::exists(data / "system" / "Acme" / "machine" / "printer.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // Installing the cache again takes the profile and its preset JSONs back out. + REQUIRE(save_one_vendor((rsrc / "profiles" / "Acme.opc").string(), one_vendor("Acme"), "Acme", "1.0.0")); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); +} + +TEST_CASE("a vendor shipped as a cache alone is installed and loaded from it", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // A packaged build: every vendor is its cache, with no profile of any kind + // beside it — not even the filament library's. + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + REQUIRE(save_one_vendor((rsrc / (lib + ".opc")).string(), one_vendor(lib, "Shipped Library"), lib, "1.0.0")); + REQUIRE(save_one_vendor((rsrc / "Acme.opc").string(), one_vendor("Acme", "Shipped Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The version the build ships the vendor at comes from the cache, there being + // no profile to read it from. + CHECK(resource_vendor_version("Acme") == Semver(1, 0, 0)); + + // Nothing installed yet, so the shipped cache answers. + PresetBundle before; + before.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(before.vendors.at("Acme").name == "Shipped Acme"); + + REQUIRE(install_vendor_bundles_from_resources({lib, "Acme"})); + CHECK(fs::exists(user / "Acme.opc")); + CHECK(!fs::exists(user / "Acme.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + PresetBundle after; + after.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(after.vendors.at("Acme").name == "Shipped Acme"); +} + +TEST_CASE("a vendor cache installed in the data dir shadows the shipped one", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + write_vendor_tree(rsrc, "Acme", "1.0.0"); + REQUIRE(save_one_vendor((rsrc / "Acme.opc").string(), one_vendor("Acme", "Shipped Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The vendor profiles name the vendor "Acme"; the caches name it after where + // they came from, so the loaded name says which source answered. + auto loaded_name = [&user](PresetBundle& bundle) { + bundle.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + return bundle.vendors.at("Acme").name; + }; + + // Nothing installed yet: the shipped cache answers. + PresetBundle from_rsrc; + CHECK(loaded_name(from_rsrc) == "Shipped Acme"); + + // Installing a newer vendor profile makes the shipped cache too old for it, even + // though that cache still matches the profile sitting beside it in resources. + write_vendor_tree(user, "Acme", "2.0.0"); + PresetBundle stale; + CHECK(loaded_name(stale) == "Acme"); + + // The cache installed alongside it does answer, and wins over the shipped one. + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Installed Acme"), "Acme", "2.0.0")); + PresetBundle from_user; + CHECK(loaded_name(from_user) == "Installed Acme"); +} + +TEST_CASE("a cache installed with no profile beside it is used whatever its version", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, "Acme"); + + // Installed at an older version than the one now shipped in resources. Nothing + // sits beside it claiming to be newer, so the cache is what the vendor is. + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Installed Acme"), "Acme", "0.9.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "0.9.0"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Other").empty()); + CHECK(installed_vendor_version("Acme") == Semver(0, 9, 0)); + + // Loading the vendor takes the installed cache, not the newer shipped profile. + PresetBundle out; + out.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(out.vendors.at("Acme").name == "Installed Acme"); +} + +TEST_CASE("a vendor whose cache covers it is loaded without parsing any JSON", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0", + {filament_entry("Acme PLA @0.4")}, + {printer_entry("Acme Printer 0.4")})); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // The cache is the whole installation — no profile, no preset JSONs — and the + // caller asks for the vendor exactly as it would for a JSON install. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + CHECK(substitutions.empty()); + CHECK(presets_loaded == 2); + CHECK(out.vendors.at("Acme").name == "Cached Acme"); + + // Nothing was written back: the presets never came from a parse. + CHECK(!fs::exists(user / "Acme.json")); +} + +TEST_CASE("a vendor whose cache is stale falls back to parsing its JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // An update installed the vendor at 2.0.0; the cache next to it was built from + // the profile before that, so it no longer covers what is on disk. + write_vendor_tree(user, "Acme", "2.0.0"); + REQUIRE(save_one_vendor((user / "Acme.opc").string(), one_vendor("Acme", "Cached Acme"), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(2, 0, 0)); + + // A one-off parse like this one leaves the stale cache alone: only a bundle + // told its parses are complete writes one. + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "1.0.0"); + + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_version_from_json((user / "Acme.json").string()).to_string()); +} + +TEST_CASE("a vendor with nothing installed is parsed from the shipped profiles", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(user); + fs::create_directories(rsrc); + write_vendor_tree(rsrc, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Neither a cache nor a profile in the directory asked for, so the vendor comes + // out of resources — which is where a build that ships caches keeps the JSONs a + // rejected cache has to be re-parsed from. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(1, 0, 0)); + + // The cache such a parse writes lands in the directory asked for, stamped with + // the version of the profile it was actually built from. + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_version_from_json((rsrc / "Acme.json").string()).to_string()); +} + +TEST_CASE("a cache with a mismatched vendor name is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("VendorA"), "VendorA", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "VendorB", Semver("1.0.0"))); +} + +TEST_CASE("a cache is rejected against an unparsable version", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + REQUIRE(save_one_vendor(cache.string(), one_vendor("Acme"), "Acme", "1.0.0")); + PresetBundle out; + // A profile version that does not parse comes out of get_version_from_json + // as zero, which cannot be judged any more than Semver::invalid() can. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", Semver())); + REQUIRE(out.vendors.empty()); // rejection happens before the body is touched +} + +TEST_CASE("the filament library's inheritance maps are rebuilt on cache load", "[VendorCache]") +{ + // m_config_maps/m_filament_id_maps are the inheritance base other vendors + // resolve against. The cache no longer stores them: they are rebuilt by + // installing the library's entries — including the non-instantiated bases, + // which exist for exactly this and never become presets. + TempDir tmp; + const fs::path cache = tmp.path / "lib.opc"; + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + + auto base = filament_entry("Generic PLA", "GFL99"); + base.instantiation = "false"; + base.config_src.set_key_value("filament_cost", new ConfigOptionFloats({20.})); + REQUIRE(save_one_vendor(cache.string(), one_vendor(lib), lib, "1.0.0", {base})); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), lib, Semver("1.0.0"))); + REQUIRE(out.m_config_maps.count("Generic PLA") == 1); + const auto* cost = out.m_config_maps.at("Generic PLA").option("filament_cost"); + REQUIRE(cost != nullptr); + CHECK_THAT(cost->values.front(), WithinAbs(20., 1e-9)); + CHECK(out.m_filament_id_maps.at("Generic PLA") == "GFL99"); + CHECK(presets_for(out.filaments, lib).empty()); // not instantiated, not a preset +} + +TEST_CASE("the same fixture parsed twice serializes byte-identically", "[VendorCache]") +{ + // Shipped caches must be reproducible: the same profiles must produce the + // same bytes on every machine that generates them. + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_full_vendor_tree(user, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle first; + first.set_generate_vendor_caches(true); + first.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(fs::exists(user / "Acme.opc")); + const std::string bytes1 = slurp(user / "Acme.opc"); + fs::remove(user / "Acme.opc"); + + PresetBundle second; + second.set_generate_vendor_caches(true); + second.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + REQUIRE(slurp(user / "Acme.opc") == bytes1); +} + +TEST_CASE("a cache that fails mid-body deserialization is rejected and leaves the bundle clean", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path valid_cache = tmp.path / "valid.opc"; + const fs::path corrupt_cache = tmp.path / "corrupt.opc"; + + REQUIRE(save_one_vendor(valid_cache.string(), one_vendor(vid), vid, "1.0.0", + {filament_entry(vid + " PLA @0.4")}, + {printer_entry(vid + " Printer 0.4")})); + // Truncate the tail (machine entries + parse_errors, per save_vendor_cache's + // field order) so the header's size/CRC still validate but cereal runs out of + // bytes partway through the body. Grow the cut if a given size ever stops + // throwing (e.g. after an unrelated field-order change to the cache format). + size_t truncate_by = 40; + bool throws = false; + for (; truncate_by <= 200; truncate_by += 8) { + fs::copy_file(valid_cache, corrupt_cache, fs::copy_option::overwrite_if_exists); + truncate_payload_and_fix_header(corrupt_cache.string(), truncate_by); + PresetBundle probe_bundle; + if (!probe_bundle.load_vendor_cache(corrupt_cache.string(), vid, Semver("1.0.0"))) { + throws = true; + break; + } + } + REQUIRE(throws); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(corrupt_cache.string(), vid, Semver("1.0.0"))); + // The catch block put the bundle back the way a failed parse would leave it. + CHECK(out.vendors.empty()); + CHECK(out.m_config_maps.empty()); + CHECK(presets_for(out.filaments, vid).empty()); + + // The recovery must leave a bundle a caller can still load a good cache into. + REQUIRE(out.load_vendor_cache(valid_cache.string(), vid, Semver("1.0.0"))); + CHECK(out.vendors.count(vid) == 1); + CHECK(presets_for(out.filaments, vid).size() == 1); }