From e5aa9a8cfcce3c01ff5a3cc322321446a8c38e0a Mon Sep 17 00:00:00 2001 From: SoftFever Date: Wed, 29 Jul 2026 21:56:17 +0800 Subject: [PATCH] Speed up profile loading with per-vendor preset caches Each vendor's system presets are serialized into a .opc cache that the app loads instead of parsing the profile JSONs, falling back to the parse whenever no cache covers what is installed. Shipped builds carry the caches instead of the raw profiles, installing and updating treat a vendor's cache as its installation, and the setup wizard loads through the same path. Per-platform scripts and CI generate the caches at build time; tests and a design doc cover the format and its validation. --- .gitattributes | 5 + .github/workflows/build_orca.yml | 29 + .gitignore | 1 + build_linux.sh | 2 + docs/HLSD/preset-cache.md | 218 ++++ scripts/build_preset_cache.bat | 122 ++ scripts/build_preset_cache.sh | 143 +++ src/dev-utils/CMakeLists.txt | 10 + src/dev-utils/generate_system_cache.cpp | 85 ++ src/libslic3r/Preset.cpp | 11 + src/libslic3r/Preset.hpp | 59 +- src/libslic3r/PresetBundle.cpp | 629 +++++++++- src/libslic3r/PresetBundle.hpp | 102 +- src/libslic3r/PrintConfig.hpp | 3 +- src/libslic3r/Semver.hpp | 13 + src/libslic3r/Utils.hpp | 9 - src/libslic3r/utils.cpp | 70 -- src/slic3r/Config/Snapshot.cpp | 11 +- src/slic3r/GUI/ConfigWizard.cpp | 68 +- src/slic3r/GUI/ConfigWizard_private.hpp | 4 +- src/slic3r/GUI/CreatePresetsDialog.cpp | 19 +- src/slic3r/GUI/WebGuideDialog.cpp | 393 +++++-- src/slic3r/GUI/WebGuideDialog.hpp | 17 +- src/slic3r/Utils/PresetUpdater.cpp | 66 +- tests/libslic3r/CMakeLists.txt | 1 + .../libslic3r/test_preset_bundle_loading.cpp | 2 +- tests/libslic3r/test_vendor_cache.cpp | 1036 +++++++++++++++++ 27 files changed, 2801 insertions(+), 327 deletions(-) create mode 100644 docs/HLSD/preset-cache.md create mode 100644 scripts/build_preset_cache.bat create mode 100755 scripts/build_preset_cache.sh create mode 100644 src/dev-utils/generate_system_cache.cpp create mode 100644 tests/libslic3r/test_vendor_cache.cpp diff --git a/.gitattributes b/.gitattributes index 4cab1f4d26..441bdfe1eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Set the default behavior, in case people don't have core.autocrlf set. * text=auto + +# Shell scripts are run by Git Bash on Windows CI, which cannot read a script +# with CRLF line endings: it fails on the first line. Windows checkouts default +# to core.autocrlf=true, so keep these LF whatever the platform. +*.sh text eol=lf diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index a7652c3bd6..3a42983ce4 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -162,6 +162,14 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (macOS) + if: runner.os == 'macOS' && !inputs.macos-combine-only + working-directory: ${{ github.workspace }} + shell: bash + # The bundle was already packed from resources/, so the caches have to be + # installed into it here; the source tree keeps its JSONs for later jobs. + run: ./scripts/build_preset_cache.sh -b build/${{ inputs.arch }} build/${{ inputs.arch }}/OrcaSlicer/OrcaSlicer.app/Contents/Resources/profiles + - name: Pack macOS app bundle ${{ inputs.arch }} if: runner.os == 'macOS' && !inputs.macos-combine-only working-directory: ${{ github.workspace }} @@ -390,6 +398,13 @@ jobs: if ($arch -eq "arm64") { .\build_release_vs.bat slicer arm64 tests } else { .\build_release_vs.bat slicer tests } shell: pwsh + - name: Build system preset cache (Windows) + if: runner.os == 'Windows' + shell: cmd + # Shipped into both the already-installed tree (portable zip, MSIX) and + # the checkout cpack re-installs from when it builds the NSIS installer. + run: scripts\build_preset_cache.bat "%BUILD_DIR%" "resources\profiles" "%BUILD_DIR%\OrcaSlicer\resources\profiles" + - name: Pack unit tests Win if: runner.os == 'Windows' working-directory: ${{ github.workspace }} @@ -539,6 +554,20 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + # Both were packed from resources/ before the caches existed, so the + # AppImage is unpacked first and the caches shipped into it and into + # the package tree; the source tree keeps its JSONs for later steps. + appimage=$(find build -maxdepth 1 -name "OrcaSlicer_Linux_AppImage*.AppImage" | head -1) + chmod +x "$appimage" + "$appimage" --appimage-extract + ./scripts/build_preset_cache.sh -b build build/package/resources/profiles squashfs-root/resources/profiles + appimagetool=$(find build -name "appimagetool.AppImage" | head -1) + ARCH=$(uname -m) "$appimagetool" --appimage-extract-and-run squashfs-root "$appimage" + rm -rf squashfs-root # Ship the freshly-built validator so slice_check_linux (build_all.yml) # can slice-sweep the shipped profiles with this PR's engine. Taken from # the aarch64 leg so the sweep also exercises the arm build; x86_64 on diff --git a/.gitignore b/.gitignore index 916c7207b7..4d3ccb5c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ internal_docs/ # Python bytecode __pycache__/ *.pyc +*.opc diff --git a/build_linux.sh b/build_linux.sh index 72ea742f1a..6d65a10e41 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -567,6 +567,8 @@ if [[ -n "${BUILD_ORCA}" ]] || [[ -n "${BUILD_TESTS}" ]] ; then print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer echo "Building OrcaSlicer_profile_validator .." print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer_profile_validator + echo "Building generate_system_cache ..." + print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target generate_system_cache ./scripts/run_gettext.sh fi if [[ -n "${BUILD_TESTS}" ]] ; then diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md new file mode 100644 index 0000000000..0923ce108f --- /dev/null +++ b/docs/HLSD/preset-cache.md @@ -0,0 +1,218 @@ +# System Preset Cache — High Level Design + +## Why it exists + +OrcaSlicer ships tens of thousands of system preset JSON files. Every launch used to +parse all of them: read each vendor profile, walk its machine, process and filament +sub-files, resolve inheritance, and build the preset collections from scratch. That +parse dominated startup, and it produced the same result every time, because system +presets only change when the app is updated or a profile update is installed. + +The preset cache replaces that parse with a read. Each vendor's fully-resolved presets +are serialized once — at build time, in CI — into a single binary file that the app +loads directly into memory. Nothing is recomputed at startup unless something changed. + +The cache is **only ever an optimization**. Every rule below exists to guarantee that a +cache is either provably equivalent to parsing the JSONs, or rejected. There is no +"mostly right" cache. + +## The unit is one vendor + +A cache covers exactly one vendor. `BBL.opc` sits beside `BBL.json` and holds +everything `BBL.json` and the `BBL/` sub-file tree would have produced. + +Per-vendor granularity is what makes the system practical: + +- A vendor whose profile is bumped invalidates only its own cache. The other 60-odd + vendors keep theirs. +- The setup wizard, which loads vendors one at a time, gets the same speedup as + startup without a second code path. +- A vendor with no cache, or a broken one, costs only that vendor a parse. + +A cache holds *system* presets only. User presets, project settings and modified +presets are never serialized — they have their own storage and their own lifecycle. + +## Where the files live + +| Location | Contents on a shipped build | Role | +|---|---|---| +| `resources/profiles/` | `.opc` alone — the profile and its preset JSONs both pruned | What the app ships with; the fallback everything falls back to | +| `/system/` | `.opc` alone, or `.json` + `/` after an update | What the user has installed | +| `/system/` (dev build) | `.json` + `/` + `.opc` written at runtime | A developer tree caches as it parses | + +Two forms of the same vendor therefore exist, and the system's central rule is that +**a vendor's cache is the whole of it**. Where a cache ships or is installed, no profile +and no preset JSONs sit beside it: the cache carries the presets, the vendor profile, +and the version stamp that says which release it came from. A vendor is "installed" if +either form is present, and its installed version is read from whichever form is there. + +What stays beside the caches in `resources/profiles/` is everything that is not a +preset: each vendor's directory of printer thumbnails, cover images, bed models and +hotend meshes, which are read from disk by path and were never part of the cache. Files +that are not vendors at all, `blacklist.json` chief among them, are untouched. + +The alternative — shipping both and treating the cache as a sidecar — was rejected. It +doubles the installed size, and it creates a class of bug where the two disagree and +the app's behavior depends on which one a given code path happened to read. + +## What a cache file is + +A fixed-size header followed by one binary stream. + +The header carries a magic number, the cache format version, the payload size and a +CRC32 of the payload. It exists so that a truncated download, a half-written file or a +file from an entirely different program is rejected in microseconds, before anything +tries to interpret it. + +The payload opens with the stamps that decide whether the cache may be used at all — +format version, schema fingerprint, vendor name, vendor version, filament library +version — and then the vendor's data: the vendor profiles, the five preset collections +(print, SLA print, filament, SLA material, printer), the config and filament-id lookup +maps, the obsolete-preset lists, and the count of errors the original parse hit. + +Two deliberate choices in the layout: + +- **Stamps come first**, so the question "what version is this vendor installed at?" + can be answered by reading the first kilobyte. The updater asks that question for + every vendor on every launch; reading tens of megabytes to answer it would give back + the startup time the cache saved. +- **Defaults are not stored.** Every collection reconstructs its default presets the + way the JSON path does, and the cache carries only what a parse would have added on + top. This keeps the cache a record of the vendor's data, not a memory image of the + program's state. + +## When a cache may be used + +A cache is accepted only if every gate below passes. Any failure means "parse the +JSONs instead" — never a hard error, never a partial load. + +**1. Integrity.** Magic number, plausible size, CRC32 over the payload. + +**2. Cache format version.** A single integer bumped by hand whenever the binary layout +changes in a way nothing else would catch: reordering or retyping a serialized field, +or changing what the cache's own stamps mean. + +**3. Schema fingerprint.** A checksum over the app version and the entire print-config +option schema — every option's key, type, wire ordinal and enum values. This is the +gate that makes the cache safe across development: adding a config option, changing its +type, or reordering the enum values of an existing one all change the fingerprint, so +caches from before the change are rejected without anyone having to remember to bump +anything. It also means a cache never crosses app versions. + +**4. Vendor identity and version.** The cache names the vendor it holds and the profile +version it was built from. It is accepted only if that version is at least as new as +the profile now on disk. Where no profile sits beside the cache — the shipped, +cache-only form — the comparison is skipped, because nothing on disk can be newer than +a cache that is the installation. + +**5. Filament library version.** Every vendor's filaments inherit from the shared Orca +filament library, so a vendor's cache is only valid against the library it was resolved +against. Bumping the library invalidates every vendor's cache, which is correct and +is why the library's version is stamped into all of them. + +A vendor profile with no parsable version is never cached and never served from a +cache. There would be no way to tell later whether the cache had gone stale, and a +cache nothing can invalidate is worse than no cache. + +## How a vendor is loaded + +When the app loads a vendor, it tries, in order: + +1. The cache in the directory it was asked to load from — normally `/system/`. +2. The shipped cache in `resources/profiles/`. +3. Parsing the JSONs — from the data directory if the profile is installed there, and + from `resources/profiles/` otherwise, which on a shipped build only has JSONs for a + vendor that has no cache. + +The second tier is what makes app upgrades work. After an upgrade, a cache the previous +version installed fails the fingerprint gate; the new build's own shipped cache answers +instead, and the user never sees a parse. The stale installed file is simply ignored +until the next profile update overwrites it. + +If a parse does happen and the vendor's profile carries a version, the app writes the +cache back beside where it looked for the vendor. That is how a developer build warms +itself up on second launch, and how a vendor delivered by a profile update becomes +cached without waiting for the next release. + +## How a vendor is installed + +Installing copies from `resources/profiles/` into `/system/`. A shipped build +offers only a cache and a source tree only JSONs, but a partially-generated tree can +have both, at different versions, so the installer picks the form that ships at the +**newer version** and installs only that one: + +- Cache newer or equal, and readable → copy the `.opc`, and delete any profile and + vendor directory a previous install left behind, so nothing can shadow it. +- Profile newer, or the cache unreadable or absent → copy the profile and the vendor's + preset JSONs exactly as the app did before caches existed, and delete any stale `.opc`. + +The result is that only one form of a vendor is ever present, and it is the newest one +the build has. This matters most for the update check, which compares what is installed +against what installing *would* lay down: if those two disagreed about which form +counts, a vendor could reinstall on every launch forever, or silently never update. + +Profile updates delivered over the air always arrive as JSONs, and they win — an +updated vendor's real profile lands in the data directory, the shipped cache is older +and gets rejected, and the vendor is parsed and re-cached. + +## How the caches are produced + +Cache generation is a build step, not something a user ever runs. + +One script per platform does the whole job, and CI calls it once on each. It builds a +small dev-utility that loads a profiles directory exactly as the app would, with cache +writing enabled, dropping a `.opc` beside every vendor profile it parses; then +it copies those caches into each packaged application it was pointed at and deletes +every preset JSON they replace — the vendor's own profile included. Only a vendor that +actually has a cache is pruned, so a vendor the generator skipped keeps its JSONs and is +simply parsed at startup. + +Because the schema fingerprint includes the app version, caches must be generated by +the same build that ships them. Generation runs after the build, in the same job. + +## Behavior when things go wrong + +The system is designed so that no cache problem is fatal: + +- **Corrupt, truncated or foreign file** — rejected at the header, vendor parsed. +- **Cache from another app version or schema** — rejected at the fingerprint, vendor + parsed or served from the shipped cache. +- **Stale cache** — rejected on the version stamps, vendor parsed and re-cached. +- **Failure part-way through reading** — the bundle is reset to a clean state before + falling back, so a half-loaded cache can never leak into the parsed result. +- **A vendor that can be neither read nor parsed** — logged, and left out. The setup + wizard drops that vendor from its list and opens with the rest; startup records the + error alongside the vendors that did load. One broken vendor never takes the app down. + +The one genuine limit: on a shipped build a vendor is its cache and nothing else, so a +rejected cache has nothing to fall back to for that vendor. This is by design — the +alternative is shipping every preset twice — and it is why the acceptance gates are +conservative and why CI generates the caches with the same build that ships them. The +recovery path is a profile update, which delivers real JSONs. + +It also means nothing may quietly assume a `.json` exists. Discovery, version +checks and the update decision all read whichever form is present, and a code path that +enumerates only `*.json` will find no vendors at all in a packaged build. + +## Maintenance rules + +- **Adding or changing a config option** needs nothing. The fingerprint covers it. +- **Changing what a cache serializes**, or the order it serializes it in, requires + bumping the cache format version by hand. +- **Bumping a vendor profile's version** invalidates that vendor's cache and nothing + else. Bumping the filament library invalidates all of them. +- **Caches are never committed.** They are build artifacts, generated per build, + ignored by git. + +## Where this lives in the tree + +| Area | Files | +|---|---| +| Cache format, read/write, load and save | `src/libslic3r/PresetBundle.{hpp,cpp}` | +| Per-preset serialization | `src/libslic3r/Preset.{hpp,cpp}` | +| Vendor discovery, installed/shipped versions, installation | `src/libslic3r/PresetBundle.cpp` | +| Update and reinstall decisions | `src/slic3r/Utils/PresetUpdater.cpp` | +| Setup wizard and printer-selection dialog | `src/slic3r/GUI/ConfigWizard.cpp`, `src/slic3r/GUI/WebGuideDialog.cpp` | +| Generator tool | `src/dev-utils/generate_system_cache.cpp` | +| Build and packaging script | `scripts/build_preset_cache.{sh,bat}` | +| Tests | `tests/libslic3r/test_vendor_cache.cpp` | diff --git a/scripts/build_preset_cache.bat b/scripts/build_preset_cache.bat new file mode 100644 index 0000000000..ac839b7c49 --- /dev/null +++ b/scripts/build_preset_cache.bat @@ -0,0 +1,122 @@ +@echo off +rem Build the per-vendor system preset caches (one .opc per vendor) by +rem running the generate_system_cache.exe dev tool against a profiles directory, +rem and make every profiles directory named on the command line ship-ready: +rem install the caches into it and delete the preset JSONs they replace, so a +rem build ships one copy of its presets instead of two. +rem +rem scripts\build_preset_cache.bat [build_dir] [target_dir ...] +rem +rem build_dir defaults to "build" +rem target_dir profiles directories to ship into. Caches are generated into +rem the source tree's resources\profiles, which is what every +rem packaging step copies from; a target may be that same +rem directory, which then only gets pruned. +rem +rem Shipping deletes, so it is a CI packaging step. A vendor's own .json +rem goes along with its preset JSONs: the cache carries the vendor profile and +rem the version it was built at, so discovery, version checks and installing all +rem read it there. Only a vendor that has a cache is pruned, so non-vendor JSONs +rem (blacklist.json) are left alone, as are the vendor directories themselves - +rem thumbnails, covers and bed models still live there. +rem +rem set CONFIG= to pin the build config for multi-config generators +rem (default: the config of the tool already in the build tree, else Release) +setlocal enabledelayedexpansion + +set "REPO_ROOT=%~dp0.." +set "BUILD_DIR=%~1" +if "%BUILD_DIR%"=="" set "BUILD_DIR=build" +if not exist "%BUILD_DIR%\" ( + echo ERROR: build tree not found: %BUILD_DIR% 1>&2 + exit /b 1 +) +if not "%~1"=="" shift + +rem Newest match wins: a stale binary silently produces a stale cache layout. +call :find_tool +if not defined CONFIG ( + for %%c in (Debug Release RelWithDebInfo MinSizeRel) do ( + echo !TOOL! | findstr /i "\\%%c\\" >nul && set "CONFIG=%%c" + ) +) +if not defined CONFIG set "CONFIG=Release" + +echo Building generate_system_cache in %BUILD_DIR% (%CONFIG%) +cmake --build "%BUILD_DIR%" --config %CONFIG% --target generate_system_cache +if errorlevel 1 ( + echo ERROR: could not build generate_system_cache - configure the build tree with -DORCA_TOOLS=ON: 1>&2 + echo cmake -S "%REPO_ROOT%" -B "%BUILD_DIR%" -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) +call :find_tool +if not defined TOOL ( + echo ERROR: generate_system_cache.exe not found under %BUILD_DIR% - build with -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) + +set "PROFILES=%REPO_ROOT%\resources\profiles" +if not exist "%PROFILES%\" ( + echo ERROR: profiles directory not found: %PROFILES% 1>&2 + exit /b 1 +) +for %%d in ("%PROFILES%") do set "PROFILES=%%~fd" + +rem Add the slicer's runtime DLL directory to PATH so generate_system_cache.exe +rem can resolve its dependencies (TKernel.dll etc.) without a full install step. +set "DLL_DIR=" +for /f "delims=" %%f in ('dir /s /b "%BUILD_DIR%\TKernel.dll" 2^>nul') do ( + if not defined DLL_DIR set "DLL_DIR=%%~dpf" +) +if defined DLL_DIR set "PATH=%DLL_DIR%;%PATH%" + +echo Generating per-vendor preset caches in %PROFILES% +rem Start clean so vendors that went away - and caches written by older tool +rem versions - don't linger next to the freshly generated ones. +del /q "%PROFILES%\*.opc" 2>nul +del /q "%PROFILES%\*.cache" 2>nul +"%TOOL%" --path "%PROFILES%" --log_level 2 +if errorlevel 1 exit /b %errorlevel% + +:next_target +if "%~1"=="" exit /b 0 +call :ship "%~1" +if errorlevel 1 exit /b 1 +shift +goto :next_target + +:ship +set "TARGET=%~1" +if not exist "%TARGET%\" ( + echo ERROR: profiles directory not found: %TARGET% 1>&2 + exit /b 1 +) +for %%d in ("%TARGET%") do set "TARGET=%%~fd" +if /i not "%TARGET%"=="%PROFILES%" copy /y "%PROFILES%\*.opc" "%TARGET%\" >nul + +set /a SHIPPED=0 +set /a PRUNED=0 +for %%c in ("%PROFILES%\*.opc") do ( + set /a SHIPPED+=1 + set "VENDOR=%%~nc" + if exist "%TARGET%\!VENDOR!.json" ( + del /q "%TARGET%\!VENDOR!.json" + set /a PRUNED+=1 + ) + if exist "%TARGET%\!VENDOR!\" ( + for /f %%n in ('dir /s /b "%TARGET%\!VENDOR!\*.json" 2^>nul ^| find /c /v ""') do set /a PRUNED+=%%n + del /s /q "%TARGET%\!VENDOR!\*.json" >nul 2>&1 + rem Deepest first, so a directory the delete above emptied goes too; rd + rem refuses the ones still holding covers or meshes. + for /f "delims=" %%d in ('dir /s /b /ad "%TARGET%\!VENDOR!" 2^>nul ^| sort /r') do rd "%%d" 2>nul + ) +) +echo %TARGET%: !SHIPPED! caches, dropped !PRUNED! preset JSONs +exit /b 0 + +:find_tool +set "TOOL=" +for /f "delims=" %%f in ('dir /s /b /o-d "%BUILD_DIR%\generate_system_cache.exe" 2^>nul') do ( + if not defined TOOL set "TOOL=%%f" +) +exit /b 0 diff --git a/scripts/build_preset_cache.sh b/scripts/build_preset_cache.sh new file mode 100755 index 0000000000..33a436f151 --- /dev/null +++ b/scripts/build_preset_cache.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Build the per-vendor system preset caches (one .opc per vendor) by +# running the generate_system_cache dev tool against a profiles directory, and +# make every profiles directory named on the command line ship-ready: install +# the caches into it and delete the preset JSONs they replace, so a build ships +# one copy of its presets instead of two. +# +# ./scripts/build_preset_cache.sh # caches into resources/profiles +# ./scripts/build_preset_cache.sh -b build/arm64 # search this build tree for the tool +# ./scripts/build_preset_cache.sh [ ...] # and ship into these profiles dirs +# +# Caches are generated into the source tree's resources/profiles, which is what +# every packaging step copies from. Shipping deletes, so it is a CI packaging +# step: pass packaged output directories, or the checkout of a build that is +# about to be packaged from it. +# +# A vendor's own .json goes along with its preset JSONs: the cache +# carries the vendor profile and the version it was built at, so discovery, +# version checks and installing all read it there. A shipped vendor is its cache +# and nothing else. Only a vendor that has a cache is pruned, so an ungenerated +# vendor keeps its JSONs and is simply parsed at startup; non-vendor JSONs +# (blacklist.json) are left alone, as are the vendor directories themselves — +# thumbnails, covers and bed models still live there. +# +# -b build tree holding the tool +# (default: build/arm64, build/x86_64, or build — first that exists) +# -p profiles directory to generate caches into +# (default: /resources/profiles) +# -c build config for multi-config generators +# (default: the config of the tool already in the build tree, else +# the build tree's CMAKE_BUILD_TYPE) +# -n skip the rebuild and run the tool already in the build tree +# -l tool log level (default: 2) +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd -P)" +build_dir="" +profiles_dir="" +config="" +build_tool=1 +log_level=2 + +while getopts "b:p:c:l:nh" opt; do + case $opt in + b) build_dir="$OPTARG" ;; + p) profiles_dir="$OPTARG" ;; + c) config="$OPTARG" ;; + n) build_tool=0 ;; + l) log_level="$OPTARG" ;; + h) sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "$build_dir" ]; then + for candidate in "$repo_root/build/arm64" "$repo_root/build/x86_64" "$repo_root/build"; do + if [ -d "$candidate" ]; then build_dir="$candidate"; break; fi + done +fi +if [ -z "$build_dir" ] || [ ! -d "$build_dir" ]; then + echo "ERROR: build tree not found (pass -b )" >&2 + exit 1 +fi + +# Newest match wins: multi-config trees keep one binary per config, and a stale +# one silently produces a stale cache layout. +find_tool() { + local best="" f + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ -z "$best" ] || [ "$f" -nt "$best" ]; then best="$f"; fi + done < <(find "$build_dir" -name generate_system_cache -type f 2>/dev/null) + printf '%s' "$best" +} + +tool=$(find_tool) +if [ -z "$config" ]; then + case "$tool" in + */Debug/*) config=Debug ;; + */Release/*) config=Release ;; + */RelWithDebInfo/*) config=RelWithDebInfo ;; + */MinSizeRel/*) config=MinSizeRel ;; + *) config=$(sed -n 's/^CMAKE_BUILD_TYPE:[A-Z]*=\(.\+\)$/\1/p' "$build_dir/CMakeCache.txt" 2>/dev/null | head -1 || true) ;; + esac +fi + +if [ "$build_tool" = 1 ]; then + echo "Building generate_system_cache in $build_dir${config:+ ($config)}" + build_args=(--build "$build_dir" --target generate_system_cache) + if [ -n "$config" ]; then build_args+=(--config "$config"); fi + if ! cmake "${build_args[@]}"; then + echo "ERROR: could not build generate_system_cache — configure the build tree with -DORCA_TOOLS=ON:" >&2 + echo " cmake -S \"$repo_root\" -B \"$build_dir\" -DORCA_TOOLS=ON" >&2 + exit 1 + fi + tool=$(find_tool) +fi + +if [ -z "$tool" ]; then + echo "ERROR: generate_system_cache not found under $build_dir — build with -DORCA_TOOLS=ON" >&2 + exit 1 +fi + +if [ -z "$profiles_dir" ]; then profiles_dir="$repo_root/resources/profiles"; fi +if [ ! -d "$profiles_dir" ]; then + echo "ERROR: profiles directory not found: $profiles_dir" >&2 + exit 1 +fi +profiles_dir=$(cd "$profiles_dir" && pwd -P) + +# Start clean so vendors that went away — and caches written by older tool +# versions — don't linger next to the freshly generated ones. +echo "Generating per-vendor preset caches in $profiles_dir" +rm -f "$profiles_dir"/*.opc "$profiles_dir"/*.cache +"$tool" --path "$profiles_dir" --log_level "$log_level" + +for target in "$@"; do + resolved=$(cd "$target" 2>/dev/null && pwd -P) || { + echo "ERROR: profiles directory not found: $target" >&2 + exit 1 + } + if [ "$resolved" != "$profiles_dir" ]; then + cp "$profiles_dir"/*.opc "$resolved"/ + fi + + pruned=0 + shipped=0 + for cache in "$profiles_dir"/*.opc; do + vendor=$(basename "$cache" .opc) + shipped=$(( shipped + 1 )) + if [ -f "$resolved/$vendor.json" ]; then + rm -f "$resolved/$vendor.json" + pruned=$(( pruned + 1 )) + fi + [ -d "$resolved/$vendor" ] || continue + n=$(find "$resolved/$vendor" -name '*.json' | wc -l) + find "$resolved/$vendor" -name '*.json' -delete + find "$resolved/$vendor" -type d -empty -delete + pruned=$(( pruned + n )) + done + echo "$resolved: $shipped caches, dropped $pruned preset JSONs" +done diff --git a/src/dev-utils/CMakeLists.txt b/src/dev-utils/CMakeLists.txt index e3534a024a..2cfce6a7c5 100644 --- a/src/dev-utils/CMakeLists.txt +++ b/src/dev-utils/CMakeLists.txt @@ -20,6 +20,16 @@ if (SLIC3R_ENC_CHECK) ) endif() +if (ORCA_TOOLS) + set(_DEV_DEFS -DBOOST_ALL_NO_LIB -DBOOST_USE_WINAPI_VERSION=0x602 -DBOOST_SYSTEM_USE_UTF8) + + # generate_system_cache: pre-generates per-vendor .opc files under resources/profiles for CI bundling. + add_executable(generate_system_cache generate_system_cache.cpp) + target_link_libraries(generate_system_cache libslic3r boost_headeronly) + target_compile_definitions(generate_system_cache PRIVATE ${_DEV_DEFS}) + +endif() + # Function that adds source file encoding check to a target # using the above encoding-check binary diff --git a/src/dev-utils/generate_system_cache.cpp b/src/dev-utils/generate_system_cache.cpp new file mode 100644 index 0000000000..23bf66fbdc --- /dev/null +++ b/src/dev-utils/generate_system_cache.cpp @@ -0,0 +1,85 @@ +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/Utils.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace Slic3r; +namespace fs = boost::filesystem; +namespace po = boost::program_options; + +int main(int argc, char* argv[]) +{ + po::options_description desc("OrcaSlicer System Cache Generator\nUsage"); + // clang-format off + desc.add_options() + ("help,h", "Show help") +#ifdef __APPLE__ + ("path,p", po::value()->default_value("../../../../../../../resources/profiles"), "Path to profiles directory") +#else + ("path,p", po::value()->default_value("../../../resources/profiles"), "Path to profiles directory") +#endif + ("log_level,l", po::value()->default_value(2), "Log level (0=trace, 2=info, 4=error)"); + // clang-format on + + po::variables_map vm; + try { + po::store(po::parse_command_line(argc, argv, desc), vm); + if (vm.count("help")) { std::cout << desc << "\n"; return 0; } + po::notify(vm); + } catch (const po::error& e) { + std::cerr << "Error: " << e.what() << "\n" << desc << "\n"; + return 1; + } + + const std::string profiles_path = vm["path"].as(); + const int log_level = vm["log_level"].as(); + + if (!fs::exists(profiles_path) || !fs::is_directory(profiles_path)) { + std::cerr << "Error: '" << profiles_path << "' is not a valid directory\n"; + return 1; + } + + set_logging_level(log_level); + set_data_dir(profiles_path); + set_resources_dir(fs::path(profiles_path).parent_path().make_preferred().string()); + + const fs::path user_dir = fs::path(data_dir()) / PRESET_USER_DIR; + if (!fs::exists(user_dir)) + fs::create_directories(user_dir); + + AppConfig app_config; + app_config.set("preset_folder", "default"); + + auto preset_bundle = std::make_unique(); + preset_bundle->set_is_validation_mode(true); + preset_bundle->set_default_suppressed(true); + preset_bundle->set_generate_vendor_caches(true); + + std::cout << "Loading system presets from: " << profiles_path << "\n"; + + try { + // In validation mode data_dir() is the profiles directory set above, so the + // loader writes each .opc next to its .json as it parses it. + preset_bundle->load_presets(app_config, ForwardCompatibilitySubstitutionRule::EnableSilent); + } catch (const std::exception& ex) { + std::cerr << "Failed to load presets: " << ex.what() << "\n"; + return 1; + } + + size_t cache_count = 0; + for (auto& entry : fs::directory_iterator(profiles_path)) + if (boost::iends_with(entry.path().string(), ".opc")) + ++ cache_count; + if (cache_count == 0) { + std::cerr << "No vendor cache files were generated under " << profiles_path << "\n"; + return 1; + } + std::cout << "Generated " << cache_count << " vendor cache file(s) under " << profiles_path << "\n"; + return 0; +} diff --git a/src/libslic3r/Preset.cpp b/src/libslic3r/Preset.cpp index 775c31b563..4c50b09535 100644 --- a/src/libslic3r/Preset.cpp +++ b/src/libslic3r/Preset.cpp @@ -147,6 +147,17 @@ Semver get_version_from_json(std::string file_path) return Semver(); //throw ConfigurationError(format("Failed loading configuration file \"%1%\": %2%", file_path, err.what())); } + catch(...) { + return Semver(); + } +} + +std::string get_vendor_cache_version(const std::string& json_path) +{ + // The version a vendor's cache is stamped with. A profile without a parsable + // version cannot be judged for staleness, so it is simply never cached. + const Semver ver = get_version_from_json(json_path); + return ver.valid() ? ver.to_string() : std::string(); } //BBS: add a function to load the key-values from xxx.json diff --git a/src/libslic3r/Preset.hpp b/src/libslic3r/Preset.hpp index b88b5a5ed5..9955210e06 100644 --- a/src/libslic3r/Preset.hpp +++ b/src/libslic3r/Preset.hpp @@ -16,6 +16,14 @@ #include "Semver.hpp" #include "ProjectTask.hpp" +#include +#include +#include +#include +#include +#include +#include + //BBS: change system directories #define PRESET_SYSTEM_DIR "system" #define PRESET_USER_DIR "user" @@ -114,6 +122,10 @@ extern Semver get_version_from_json(std::string file_path); //BBS: add a function to load the key-values from xxx.json extern int get_values_from_json(std::string file_path, std::vector& keys, std::map& key_values); +// Returns the version a vendor JSON's preset cache is stamped with: its Semver +// string, or an empty string when the profile carries no usable version. +extern std::string get_vendor_cache_version(const std::string& json_path); + extern ConfigFileType guess_config_file_type(const boost::property_tree::ptree &tree); extern void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil_to_default, const DynamicPrintConfig& defaults); @@ -131,6 +143,10 @@ public: PrinterVariant() {} PrinterVariant(const std::string &name) : name(name) {} std::string name; + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) { ar(name); } // PrinterVariant }; struct PrinterModel { @@ -139,7 +155,7 @@ public: std::string name; //BBS: this is internal id for the printer. Currently only used for searching in database std::string model_id; - PrinterTechnology technology; + PrinterTechnology technology = ptFFF; std::string family; std::vector variants; std::vector default_materials; @@ -162,6 +178,17 @@ public: } const PrinterVariant* variant(const std::string &name) const { return const_cast(this)->variant(name); } + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // PrinterModel + { + ar(id, name, model_id, technology, family, variants, default_materials, + not_support_bed_types, bed_model, bed_texture, image_bed_type, + bottom_texture_end_name, use_double_extruder_default_texture, + bottom_texture_rect, bottom_texture_rect_longer, middle_texture_rect, + hotend_model); + } }; std::vector models; @@ -173,6 +200,14 @@ public: bool valid() const { return ! name.empty() && ! id.empty() && config_version.valid(); } + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // VendorProfile + { + ar(name, id, config_version, config_update_url, changelog_url, + models, default_filaments, default_sla_materials); + } + // Load VendorProfile from an ini file. // If `load_all` is false, only the header with basic info (name, version, URLs) is loaded. static VendorProfile from_ini(const boost::filesystem::path &path, bool load_all=true); @@ -425,12 +460,30 @@ public: // BBS: move constructor to public Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {} - -protected: + // Default constructor is public so cereal can default-construct elements when + // deserializing std::vector (std::allocator is not a cereal::access friend). Preset() = default; +protected: friend class PresetCollection; friend class PresetBundle; + friend class cereal::access; + + // Hand-written cereal serialization for the per-vendor binary cache. + // Lists every data member except the two raw pointers: + // - loading_substitutions: transient parse state, never cached + // - vendor: re-pointed on load from the vendor id stored alongside each preset + // Keep this list in sync with the member declarations, in declaration order; + // bump CACHE_VERSION in PresetBundle.cpp when it changes. + template + void serialize(Archive& ar) + { + ar(type, is_default, is_external, is_system, is_visible, is_dirty, + is_compatible, is_project_embedded, name, file, loaded, config, + alias, renamed_from, m_excluded_from, m_from_orca_filament_lib, + bundle_id, version, ini_str, setting_id, filament_id, user_id, + base_id, sync_info, description, updated_time, key_values); + } }; bool is_compatible_with_print (const PresetWithVendorProfile &preset, const PresetWithVendorProfile &active_print, const PresetWithVendorProfile &active_printer); diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index 01cbc43bc2..f093223842 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -1,7 +1,15 @@ #include +#include #include +#include #include "PresetBundle.hpp" + +#include +#include +#include +#include +#include #include "PrintConfig.hpp" #include "libslic3r.h" #include "I18N.hpp" @@ -564,6 +572,8 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, substitution_rule %1%, preferred printer_model_id %2%")%substitution_rule%preferred_selection.printer_model_id; + const auto startup_t0 = std::chrono::steady_clock::now(); + //BBS: change system config to json std::tie(substitutions, errors_cummulative) = this->load_system_presets_from_json(substitution_rule); @@ -589,6 +599,12 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward set_calibrate_printer(""); + { + const auto total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - startup_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: all presets loaded in " << total_ms << " ms"; + } + //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" finished, returned substitutions %1%")%substitutions.size(); return substitutions; @@ -1001,6 +1017,8 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For bundles.m_bundles.clear(); bundles.WriteUnlock(); + const auto user_load_t0 = std::chrono::steady_clock::now(); + // Load bundle metadata from _local directory first fs::path local_dir(folder / PRESET_LOCAL_DIR); if (fs::exists(local_dir)) { @@ -1019,7 +1037,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.filament_presets.clear(); metadata.printer_presets.clear(); - // Add the profiles this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::LocalBundle, metadata.id)); @@ -1056,7 +1073,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.printer_presets.clear(); metadata.is_subscribed = true; - // Load presets from bundle (same logic as __local__) this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::SubscribedBundle, metadata.id)); @@ -1077,34 +1093,41 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For } } - // BBS do not load sla_print - // BBS: change directoties by design - try { - std::string print_selected_preset_name = prints.get_selected_preset().name; - this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); - prints.select_preset_by_name(print_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + // BBS: change directories by design + + { + const auto json_t0 = std::chrono::steady_clock::now(); + try { + std::string sel = prints.get_selected_preset().name; + this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); + prints.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = filaments.get_selected_preset().name; + this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); + filaments.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = printers.get_selected_preset().name; + this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); + printers.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + + const auto json_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - json_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user presets loaded from JSON in " << json_ms << " ms"; } - try { - std::string filament_selected_preset_name = filaments.get_selected_preset().name; - this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); - filaments.select_preset_by_name(filament_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + + { + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - user_load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user + bundle presets loaded in " << ms << " ms"; } - try { - std::string printer_selected_preset_name = printers.get_selected_preset().name; - this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); - printers.select_preset_by_name(printer_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); - } - if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + this->update_multi_material_filament_presets(); this->update_compatible(PresetSelectCompatibleType::Never); - set_calibrate_printer(""); return PresetsConfigSubstitutions(); @@ -1210,13 +1233,10 @@ bool PresetBundle::apply_vendor_config( : std::map(); // Find vendors that need installation - const auto vendor_dir = (fs::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - std::vector install_bundles; for (const auto &it : new_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir / (it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -2225,6 +2245,166 @@ void PresetBundle::remove_users_preset(AppConfig &config, std::map vendor_names_in(const boost::filesystem::path& dir) +{ + std::set names; + for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { + const auto& path = dir_entry.path(); + if (Slic3r::is_json_file(path.string()) || path.extension() == ".opc") + names.insert(path.stem().string()); + } + return names; +} + +// A vendor's preset cache is the whole of its installation: it carries the presets, +// the vendor profile and the version they were built at, so where one ships nothing +// else needs copying. Unless the profile beside it claims a newer version — a cache +// generated before that profile was bumped is out of date, and a cache that cannot +// be read is no installation at all — and the vendor is installed the way it was +// before caches existed, as its profile and the preset JSONs it points at. Returns +// the version the cache is stamped with, or nothing when it is not the form to install. +static std::string installable_cache_version(const boost::filesystem::path& dir, const std::string& vendor) +{ + const auto cache_ver = Semver::parse(PresetBundle::peek_vendor_cache_version((dir / (vendor + ".opc")).string(), vendor)); + if (! cache_ver) + return {}; + const Semver profile_ver = get_version_from_json((dir / (vendor + ".json")).string()); + return profile_ver.valid() && *cache_ver < profile_ver ? std::string() : cache_ver->to_string(); +} + +Semver resource_vendor_version(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(resources_dir()) / "profiles"; + const auto ver = Semver::parse(installable_cache_version(dir, vendor)); + return ver ? *ver : get_version_from_json((dir / (vendor + ".json")).string()); +} + +bool install_vendor_bundles_from_resources( + const std::vector& bundle_names, + const std::string& resource_subdir, + const std::string& data_subdir) +{ + namespace fs = boost::filesystem; + + fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; + fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; + + BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; + + for (const auto &bundle : bundle_names) { + try { + // Install the JSON file + auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); + auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); + auto cache_in_rsrc = (rsrc_path / bundle).replace_extension(".opc"); + auto cache_in_vendors = (vendor_path / bundle).replace_extension(".opc"); + + // Either form of the vendor will do: a build may ship it as a cache alone. + if (!fs::exists(path_in_rsrc) && !fs::exists(cache_in_rsrc)) { + BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; + return false; + } + + // Create target directory if needed + if (!fs::exists(vendor_path)) + fs::create_directories(vendor_path); + + std::string error_message; + bool installed_cache = false; + if (! installable_cache_version(rsrc_path, bundle).empty()) { + installed_cache = copy_file(cache_in_rsrc.string(), cache_in_vendors.string(), error_message, false) == CopyFileResult::SUCCESS; + if (! installed_cache) + BOOST_LOG_TRIVIAL(warning) << "Failed to copy " << bundle << ".opc: " << error_message; + } else { + boost::system::error_code ec; + fs::remove(cache_in_vendors, ec); + } + + if (! installed_cache) { + CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); + if (cfr != CopyFileResult::SUCCESS) { + BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; + return false; + } + } else { + // Left in place, an earlier install's profile would shadow the cache. + boost::system::error_code ec; + fs::remove(path_in_vendors, ec); + } + + // Copy the vendor directory (if it exists) + auto dir_in_rsrc = rsrc_path / bundle; + auto dir_in_vendors = vendor_path / bundle; + + // Whatever is installed came from an earlier version of this vendor and + // would be parsed in place of the one being installed now. + if (fs::exists(dir_in_vendors)) + fs::remove_all(dir_in_vendors); + + if (! installed_cache && fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { + fs::create_directories(dir_in_vendors); + + // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) + // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf + auto file_filter = [](const std::string name) -> bool { + return boost::iends_with(name, ".stl") || + boost::iends_with(name, ".png") || + boost::iends_with(name, ".svg") || + boost::iends_with(name, ".jpeg") || + boost::iends_with(name, ".jpg") || + boost::iends_with(name, ".3mf"); + }; + + copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); + } + + BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; + + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); + return false; + } + } + + return true; +} + //BBS: add json related logic, load system presets from json std::pair PresetBundle::load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule) { @@ -2243,22 +2423,19 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); + const auto load_t0 = std::chrono::steady_clock::now(); + + // The vendors below are loaded whole and against each other — the filament + // library first, then every other vendor with it as the base — so each parse + // is complete enough to be worth caching. + m_generate_vendor_caches = m_generate_vendor_caches || ! validation_mode; + PresetsConfigSubstitutions substitutions; std::string errors_cummulative; - bool first = true; - std::vector vendor_names; - // store all vendor names in vendor_names - for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { - std::string vendor_file = dir_entry.path().string(); - if (!Slic3r::is_json_file(vendor_file)) - continue; - - std::string vendor_name = dir_entry.path().filename().string(); - - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - vendor_names.push_back(vendor_name); - } + bool first = true; + // Sorted, so any duplicate-preset warning below comes out in the same order on + // every run. + const std::set vendor_names = vendor_names_in(dir); // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded // first because other vendors' filaments may inherit from it via the // `base_bundle` lookup in parse_subfile. The remaining vendors are @@ -2274,8 +2451,17 @@ std::pair PresetBundle::load_system_pre } // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (!orca_lib_vendor.empty()) { + if (! orca_lib_vendor.empty()) { try { + // Match a fresh launch before parsing: hold aliases and the error + // counter survive reset(), and would otherwise leak prior-cycle + // state into the library cache the load below writes. + this->prints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); + this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json(dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule).first); first = false; } catch (const std::runtime_error &err) { @@ -2298,10 +2484,10 @@ std::pair PresetBundle::load_system_pre for (size_t i = range.begin(); i < range.end(); ++i) { auto bundle = std::make_unique(); bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); try { auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, - compatibility_rule, this); + dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this); parallel_substitutions[i] = std::move(result.first); parallel_bundles[i] = std::move(bundle); } catch (const std::runtime_error &err) { @@ -2346,6 +2532,11 @@ std::pair PresetBundle::load_system_pre } this->update_system_maps(); + + const auto load_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; + //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; return std::make_pair(std::move(substitutions), errors_cummulative); @@ -4762,18 +4953,37 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool //BBS: Load a config bundle file from json std::pair PresetBundle::load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) { // Enable substitutions for user config bundle, throw an exception when loading a system profile. ConfigSubstitutionContext substitution_context { compatibility_rule }; PresetsConfigSubstitutions substitutions; //BBS: add config related logs - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%path.c_str()%compatibility_rule; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; if (flags.has(LoadConfigBundleAttribute::ResetUserProfile) || flags.has(LoadConfigBundleAttribute::LoadSystem)) // Reset this bundle, delete user profile files if SaveImported. this->reset(flags.has(LoadConfigBundleAttribute::SaveImported)); + // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only + // scans want a slice of one. Validation reads the JSONs whatever is cached. + const boost::filesystem::path dir_path(dir); + const bool cacheable = flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + + // Orca: a build that ships preset caches installs them without the preset + // JSONs, so a vendor left to be parsed is parsed from the profiles in + // resources. An update does deliver JSONs into `dir`, and those win. + const std::string path = (validation_mode || boost::filesystem::exists(dir_path / (vendor_name + ".json"))) + ? dir : (boost::filesystem::path(resources_dir()) / "profiles").string(); + // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); std::string root_file = path + "/" + vendor_name + ".json"; @@ -5331,6 +5541,21 @@ std::pair PresetBundle::load_vendor_configs_ } } + // Orca: leave the vendor's cache in step with the profile just parsed, so the + // next run reads it instead. It is written where the vendor was looked for, + // even when the profile came from resources, and stamped with the version that + // profile claims — a profile without one cannot be judged for staleness later, + // and a cache nothing can invalidate is worse than none. + if (cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid()) { + const std::string version = vendor_profile.config_version.to_string(); + // The library is its own reference point; every other vendor's cache holds + // filaments resolved against it, and is stamped with the version in effect. + const std::string lib_version = vendor_name == ORCA_FILAMENT_LIBRARY ? version : effective_lib_version(dir_path); + if (! lib_version.empty() && + ! this->save_vendor_cache((dir_path / (vendor_name + ".opc")).string(), vendor_name, version, lib_version)) + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; + } + //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", finished, presets_loaded %1%")%presets_loaded; return std::make_pair(std::move(substitutions), presets_loaded); @@ -5953,4 +6178,312 @@ bool BundleMetadata::save_to_json(const std::string& path) const return false; } } +// ---- Preset cache file format (shared by the per-vendor cache) ---------- + +namespace { + +#pragma pack(push, 1) +struct CacheFileHeader { + uint32_t magic; + uint32_t version; + uint64_t data_size; + uint32_t crc32; +}; +#pragma pack(pop) +static_assert(sizeof(CacheFileHeader) == 20, "CacheFileHeader must be 20 bytes"); + +constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ" +// Bump when the wire format changes in a way the schema fingerprint cannot +// detect: reordering, removing, or retyping a field of a hand-written +// serialize() (Preset, VendorProfile and its nested types), or when the +// cache's own layout or the meaning of its stamps changes (e.g. the move from +// one whole-bundle cache to one cache per vendor). +constexpr uint32_t CACHE_VERSION = 4; + +// A cache stays usable as long as it was built from a vendor profile — and a +// filament library — at least as new as the ones now on disk. Profiles without +// a version cannot be judged this way and are never served from cache; where no +// profile sits beside the cache at all, nothing can be newer than it. +static bool cache_covers_version(const std::string& cached, const std::string& on_disk) +{ + if (on_disk == PresetBundle::CACHE_ANY_VERSION) + return true; + const auto cached_ver = Semver::parse(cached); + const auto on_disk_ver = Semver::parse(on_disk); + return cached_ver && on_disk_ver && *cached_ver >= *on_disk_ver; +} + +// Fingerprint of everything that determines the cache wire format: the app +// version and the DynamicPrintConfig option schema (key/type/ordinal/enum +// values — serialization_key_ordinal IS the config wire format). Any mismatch +// means bytes written by another build could deserialize into the wrong +// fields, so the cache is rejected wholesale before its body is read. +std::string compute_cache_schema_fingerprint() +{ + std::string schema; + schema += SLIC3R_VERSION; + schema += ';'; + for (const auto& [key, def] : print_config_def.options) { // std::map => stable order + schema += key; + schema += '#'; schema += std::to_string(int(def.type)); + schema += '@'; schema += std::to_string(def.serialization_key_ordinal); + for (const std::string& ev : def.enum_values) { schema += ','; schema += ev; } + schema += ';'; + } + boost::crc_32_type crc; + crc.process_bytes(schema.data(), schema.size()); + return std::to_string(crc.checksum()); +} + +} // anonymous namespace + +// static +bool PresetBundle::read_cache_blob(const std::string& path, std::string& out_blob) +{ + try { + boost::nowide::ifstream ifs(path, std::ios::binary); + if (!ifs.is_open()) + return false; + CacheFileHeader fhdr; + if (!ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr))) + return false; + if (fhdr.magic != CACHE_MAGIC) + return false; + if (fhdr.data_size == 0 || fhdr.data_size > 512u * 1024u * 1024u) + return false; + out_blob.assign(fhdr.data_size, '\0'); + if (!ifs.read(&out_blob[0], static_cast(fhdr.data_size))) + return false; + boost::crc_32_type crc; + crc.process_bytes(out_blob.data(), out_blob.size()); + if (crc.checksum() != fhdr.crc32) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: CRC mismatch: " << path; + return false; + } + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: read failed (" << path << "): " << e.what(); + return false; + } +} + +// static +void PresetBundle::write_cache_blob(const std::string& path, const std::string& blob) +{ + boost::crc_32_type crc; + crc.process_bytes(blob.data(), blob.size()); + try { + boost::filesystem::create_directories(boost::filesystem::path(path).parent_path()); + boost::nowide::ofstream ofs(path, std::ios::binary | std::ios::trunc); + if (!ofs.is_open()) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: cannot open for writing: " << path; + return; + } + CacheFileHeader fhdr; + fhdr.magic = CACHE_MAGIC; + fhdr.version = CACHE_VERSION; + fhdr.data_size = static_cast(blob.size()); + fhdr.crc32 = crc.checksum(); + ofs.write(reinterpret_cast(&fhdr), sizeof(fhdr)); + ofs.write(blob.data(), static_cast(blob.size())); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: write failed (" << path << "): " << e.what(); + } +} + +// ---- Per-vendor preset cache --------------------------------------------- + +// Serializes every non-default, non-external preset of one collection plus +// its m_printer_hold_alias. Defaults are constructor-derived state and are +// skipped; load re-installs the loaded presets after the constructor's +// defaults, exactly like the JSON parse path (reset-to-defaults + sorted +// append). +void PresetBundle::save_collection(cereal::BinaryOutputArchive& ar, const PresetCollection& coll) +{ + uint64_t count = 0; + for (const Preset& p : coll.m_presets) + if (! p.is_default && ! p.is_external) + ++ count; + ar(count); + for (const Preset& p : coll.m_presets) { + if (p.is_default || p.is_external) + continue; + std::string vendor_id = p.vendor ? p.vendor->id : std::string(); + ar(vendor_id, p); + } + // unordered containers -> sorted, for byte-deterministic output (parity test) + std::map> hold; + for (const auto& entry : coll.m_printer_hold_alias) + hold.emplace(entry.first, std::set(entry.second.begin(), entry.second.end())); + ar(hold); +} + +void PresetBundle::load_collection(cereal::BinaryInputArchive& ar, PresetCollection& coll, const VendorMap& vendors) +{ + coll.m_printer_hold_alias.clear(); + // Drop everything but the constructor-installed defaults, like PresetCollection::reset(). + coll.m_presets.erase(coll.m_presets.begin() + coll.m_num_default_presets, coll.m_presets.end()); + // Mirror PresetCollection::reset(), which follows the same truncation with + // select_preset(0): re-point selection/edited-preset state at the default + // preset before the loaded presets are appended below, so a cache hit onto + // an already-populated bundle (e.g. a reload) leaves selection state + // identical to the JSON-parse path instead of carrying over a stale index. + coll.select_preset(0); + uint64_t count = 0; + ar(count); + for (uint64_t i = 0; i < count; ++ i) { + std::string vendor_id; + Preset preset(coll.m_type, std::string()); + ar(vendor_id, preset); + if (! vendor_id.empty()) { + auto it = vendors.find(vendor_id); + if (it == vendors.end()) + throw std::runtime_error("vendor cache references unknown vendor: " + vendor_id); + preset.vendor = &it->second; + } + coll.m_presets.emplace_back(std::move(preset)); + } + std::map> hold; + ar(hold); + for (auto& entry : hold) + coll.m_printer_hold_alias.emplace(entry.first, std::unordered_set(entry.second.begin(), entry.second.end())); +} + +// ---- Per-vendor preset cache implementation ------------------------------ + +bool PresetBundle::save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const std::string& lib_version) const +{ + try { + std::ostringstream body(std::ios::binary); + { + cereal::BinaryOutputArchive ar(body); + ar(CACHE_VERSION); + ar(compute_cache_schema_fingerprint()); + ar(vendor_name, vendor_version, lib_version); + ar(this->vendors); + save_collection(ar, this->prints); + save_collection(ar, this->sla_prints); + save_collection(ar, this->filaments); + save_collection(ar, this->sla_materials); + save_collection(ar, this->printers); + ar(this->m_config_maps, this->m_filament_id_maps); + ar(this->obsolete_presets.prints, this->obsolete_presets.sla_prints, + this->obsolete_presets.filaments, this->obsolete_presets.sla_materials, + this->obsolete_presets.printers); + ar(this->m_errors); + } + write_cache_blob(cache_path, body.str()); + std::string verify; + return read_cache_blob(cache_path, verify); // magic + size + CRC + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache " << cache_path << ": " << e.what(); + return false; + } +} + +// static +std::string PresetBundle::peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name) +{ + try { + boost::nowide::ifstream ifs(cache_path, std::ios::binary); + CacheFileHeader fhdr; + if (! ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr)) || fhdr.magic != CACHE_MAGIC) + return {}; + // Only the head of the body is read, and its CRC left unverified: the stamps + // sit at the front, this answers "what version is this vendor at?" once per + // vendor on every update check, and reading tens of megabytes to do so is not + // worth it. A stamp that comes out garbled fails to parse as a version, which + // is the same answer as none. + std::string head(static_cast(std::min(fhdr.data_size, 1024)), '\0'); + if (! ifs.read(&head[0], static_cast(head.size()))) + return {}; + std::istringstream body(head, std::ios::binary); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + std::string fingerprint, vendor_name, vendor_version; + ar(fingerprint); + ar(vendor_name, vendor_version); + // The fingerprint is deliberately not checked: the version a cache carries + // is what this build installed, whether or not this build can still read it. + if (cache_version != CACHE_VERSION || vendor_name != expected_vendor_name) + return {}; + return vendor_version; + } catch (const std::exception&) { + return {}; + } +} + +bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name) +{ + // Whichever cache answers is judged against the vendor as installed in `dir`: + // the profile there, or — with none, as when the cache is the whole of the + // installation — nothing, since nothing on disk can then be newer than it. + // Plus the filament library in effect, which a vendor's filaments were + // resolved against when its cache was built. + const boost::filesystem::path profile = dir / (vendor_name + ".json"); + const std::string version = boost::filesystem::exists(profile) ? get_vendor_cache_version(profile.string()) + : std::string(CACHE_ANY_VERSION); + const std::string lib_version = effective_lib_version(dir); + const boost::filesystem::path rsrc = boost::filesystem::path(resources_dir()) / "profiles"; + return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, lib_version) + || (dir != rsrc && this->load_vendor_cache((rsrc / (vendor_name + ".opc")).string(), vendor_name, version, lib_version)); +} + +bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const std::string& expected_vendor_version, const std::string& expected_lib_version) +{ + std::string blob; + if (! read_cache_blob(cache_path, blob)) + return false; + try { + std::istringstream body(blob, std::ios::binary); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + if (cache_version != CACHE_VERSION) + return false; + std::string fingerprint; + ar(fingerprint); + if (fingerprint != compute_cache_schema_fingerprint()) + return false; + std::string vendor_name, vendor_version, lib_version; + ar(vendor_name, vendor_version, lib_version); + if (vendor_name != expected_vendor_name || + ! cache_covers_version(vendor_version, expected_vendor_version) || + ! cache_covers_version(lib_version, expected_lib_version)) + return false; + ar(this->vendors); + load_collection(ar, this->prints, this->vendors); + load_collection(ar, this->sla_prints, this->vendors); + load_collection(ar, this->filaments, this->vendors); + load_collection(ar, this->sla_materials, this->vendors); + load_collection(ar, this->printers, this->vendors); + ar(this->m_config_maps, this->m_filament_id_maps); + ar(this->obsolete_presets.prints, this->obsolete_presets.sla_prints, + this->obsolete_presets.filaments, this->obsolete_presets.sla_materials, + this->obsolete_presets.printers); + ar(this->m_errors); + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); + // Restore a clean state so the caller can fall back to the JSON parse. + this->reset(false); + this->vendors.clear(); + this->m_config_maps.clear(); + this->m_filament_id_maps.clear(); + this->m_errors = 0; + // reset() does not clear m_printer_hold_alias, and a mid-body failure may + // have left collections it never reached (each load_collection clears its + // own collection's map only when it runs) with stale aliases. + this->prints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); + return false; + } +} + } // namespace Slic3r diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 685687975b..bde0bbbeba 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -6,6 +6,7 @@ #include "enum_bitmask.hpp" #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include + #define DEFAULT_USER_FOLDER_NAME "default" #define BUNDLE_STRUCTURE_JSON_NAME "bundle_structure.json" @@ -170,6 +172,44 @@ struct PresetBundleMetadata class PresetBundle { public: + // ---- Per-vendor preset cache -------------------------------------------- + // One cache file per vendor (plus the Orca filament library), stamped with + // the vendor's own profile version rather than a directory scan. + + // The cache is not something a caller loads from: a vendor is loaded with + // load_vendor_configs_from_json, which comes from the cache whenever one covers + // it. What is public here is what the cache's own tests drive directly. + + // Save this bundle's slice belonging to one vendor (vendor_name at + // vendor_version), built against the given filament library version. + bool save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const std::string& lib_version) const; + + // Expected version meaning "no profile sits beside this cache", so nothing can + // be newer than it and only its own integrity decides whether it is used. Not + // the same as an empty version, which means a profile is there but unreadable. + static constexpr const char* CACHE_ANY_VERSION = "*"; + + // Load a validated per-vendor cache into this bundle. Rejects (returns + // false) unless the cache version, schema fingerprint and vendor name match + // and the cache was built from a vendor profile and filament library at + // least as new as the expected ones. Profiles without a version (empty + // string) are never served from cache. + bool load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const std::string& expected_vendor_version, const std::string& expected_lib_version); + + // Read the profile version a cache was stamped with, without deserializing its + // presets. Empty if the file is unreadable, not a cache this build understands, + // or not this vendor's. This is how an installed vendor's version is known when + // only its cache is installed. + static std::string peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name); + + // Enable writing a per-vendor cache after a JSON parse (off by default). Only + // for bundles whose parses are complete — load_system_presets_from_json loads + // the filament library first and every other vendor against it, so its parses + // qualify; a wizard's one-off parse of a single vendor does not. + void set_generate_vendor_caches(bool enable) { m_generate_vendor_caches = enable; } + static DynamicPrintConfig construct_full_config(Preset &in_printer_preset, Preset &in_print_preset, const DynamicPrintConfig &project_config, @@ -444,8 +484,12 @@ public: /*std::pair load_configbundle( const std::string &path, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule);*/ //Orca: load config bundle from json, pass the base bundle to support cross vendor inheritance + // Orca: `dir` is where the vendor is looked for — its own directory, whether or + // not the profile JSONs are still there. A whole-vendor load comes from the + // vendor's preset cache whenever one covers the profile on disk, and is parsed + // from the JSONs (falling back to the ones in resources) only when none does. std::pair load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); // Export a config bundle file containing all the presets and the names of the active presets. //void export_configbundle(const std::string &path, bool export_system_settings = false, bool export_physical_printers = false); @@ -521,7 +565,33 @@ public: // compatible_prints references a deleted (unknown) or renamed (old) preset name. bool check_preset_references() const; + // Merge one vendor's presets with the other vendor's presets, report duplicates. + // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a + // bundle out of several per-vendor caches loaded into separate PresetBundle instances. + std::vector merge_presets(PresetBundle &&other); + private: + // Load one vendor from its preset cache: the one in `dir`, or — when that is + // missing or stale — the one shipped in resources/profiles, both judged against + // the vendor as installed in `dir` and against the filament library in effect. + // False, with this bundle left clean, when neither is usable and the vendor has + // to be parsed. This is how load_vendor_configs_from_json reads a cache. + bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name); + + // Read raw cache blob: verify magic, size, CRC. + static bool read_cache_blob(const std::string& path, std::string& out_blob); + // Write a cache blob with the standard 20-byte file header. + static void write_cache_blob(const std::string& path, const std::string& blob); + + // (De)serialization of one collection's slice for the per-vendor cache: + // every non-default, non-external preset (system or user) plus + // m_printer_hold_alias. See save_vendor_cache/load_vendor_cache. + static void save_collection(cereal::BinaryOutputArchive& ar, const PresetCollection& coll); + static void load_collection(cereal::BinaryInputArchive& ar, PresetCollection& coll, const VendorMap& vendors); + + // Whether to (re)write a per-vendor cache after a JSON parse. + bool m_generate_vendor_caches { false }; + // Orca: validation only - flag any printer with two or more compatible // filament presets sharing one filament_id (ambiguous AMS subtype match). bool check_duplicate_filament_subtypes() const; @@ -529,8 +599,6 @@ private: //std::pair load_system_presets(ForwardCompatibilitySubstitutionRule compatibility_rule); //BBS: add json related logic std::pair load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule); - // Merge one vendor's presets with the other vendor's presets, report duplicates. - std::vector merge_presets(PresetBundle &&other); // Update the multicolor information for filaments. void update_filament_multi_color(); // Update renamed_from and alias maps of system profiles. @@ -565,6 +633,34 @@ private: ENABLE_ENUM_BITMASK_OPERATORS(PresetBundle::LoadConfigBundleAttribute) +// True if `vendor` is installed in data_dir()/system. A build that ships preset +// caches installs the cache alone, so it — not the profile — marks a vendor +// installed, and either one on its own counts. +extern bool is_vendor_installed(const std::string& vendor); + +// The version of the installed vendor: what its profile claims, or what its cache +// was stamped with where only the cache is installed. Invalid Semver if neither is. +extern Semver installed_vendor_version(const std::string& vendor); + +// The vendors `dir` holds, sorted: one is named by its profile or, in a build that +// ships preset caches instead of the raw profile JSONs, by its cache alone. +extern std::set vendor_names_in(const boost::filesystem::path& dir); + +// The version a build ships `vendor` at: whichever of its preset cache and its +// profile is newer, that being the one installing lays down. Invalid Semver if the +// build ships neither. +extern Semver resource_vendor_version(const std::string& vendor); + +// Install vendors from the resources directory into the data directory, each as +// its preset cache or as its profile and preset JSONs — whichever of the two the +// build ships at the newer version. Anything the previous install of that vendor +// left behind goes, so only the form just installed is there to be loaded. +// bundle_names: vendor names, without extension. +// Returns false on the first vendor that cannot be installed. +extern bool install_vendor_bundles_from_resources(const std::vector& bundle_names, + const std::string& resource_subdir = "profiles", + const std::string& data_subdir = "system"); + } // namespace Slic3r #endif /* slic3r_PresetBundle_hpp_ */ diff --git a/src/libslic3r/PrintConfig.hpp b/src/libslic3r/PrintConfig.hpp index 67841c6404..5f77baad86 100644 --- a/src/libslic3r/PrintConfig.hpp +++ b/src/libslic3r/PrintConfig.hpp @@ -2410,7 +2410,8 @@ namespace cereal { archive(serialization_key_ordinal); assert(serialization_key_ordinal > 0); auto it = Slic3r::print_config_def.by_serialization_key_ordinal.find(serialization_key_ordinal); - assert(it != Slic3r::print_config_def.by_serialization_key_ordinal.end()); + if (it == Slic3r::print_config_def.by_serialization_key_ordinal.end()) + throw std::runtime_error("VendorCache: unknown serialization_key_ordinal " + std::to_string(serialization_key_ordinal) + " - cache is stale"); config.set_key_value(it->second->opt_key, it->second->load_option_from_archive(archive)); } } diff --git a/src/libslic3r/Semver.hpp b/src/libslic3r/Semver.hpp index 4d64b1c7db..d3683b4eb8 100644 --- a/src/libslic3r/Semver.hpp +++ b/src/libslic3r/Semver.hpp @@ -190,6 +190,19 @@ public: os << self.to_string(); return os; } + + // cereal: round-trip through the standard 3-part string (major.minor.patch). + // to_string() uses a BBS 4-part format that semver_parse() cannot read back. + template + std::string save_minimal(const Archive&) const { return to_string_sf(); } + template + void load_minimal(const Archive&, const std::string& s) { + auto v = Semver::parse(s); + if (! v) + throw std::runtime_error("Semver: cannot parse serialized version: " + s); + *this = std::move(*v); + } + private: semver_t ver; diff --git a/src/libslic3r/Utils.hpp b/src/libslic3r/Utils.hpp index 62b2eeb78e..c52a120eff 100644 --- a/src/libslic3r/Utils.hpp +++ b/src/libslic3r/Utils.hpp @@ -722,15 +722,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, std::function filter = nullptr, bool merge_mode = false); -// Install vendor bundles from resources directory to data directory -// bundle_names: vector of vendor bundle names (without .json extension) -// resource_subdir: subdirectory under resources_dir() (default: "profiles") -// data_subdir: subdirectory under data_dir() (default: "system") -// Returns: true if all bundles installed successfully, false otherwise -bool install_vendor_bundles_from_resources(const std::vector& bundle_names, - const std::string& resource_subdir = "profiles", - const std::string& data_subdir = "system"); - // Orca: Since 1.7.9 Boost deprecated save_string_file and load_string_file, copy and modified from boost 1.7.8 void save_string_file(const boost::filesystem::path& p, const std::string& str); void load_string_file(const boost::filesystem::path& p, std::string& str); diff --git a/src/libslic3r/utils.cpp b/src/libslic3r/utils.cpp index 5f429f076a..4f4cc3450d 100644 --- a/src/libslic3r/utils.cpp +++ b/src/libslic3r/utils.cpp @@ -1724,76 +1724,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, return; } -bool install_vendor_bundles_from_resources( - const std::vector& bundle_names, - const std::string& resource_subdir, - const std::string& data_subdir) -{ - namespace fs = boost::filesystem; - - fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; - fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; - - BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; - - for (const auto &bundle : bundle_names) { - try { - // Install the JSON file - auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); - auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); - - if (!fs::exists(path_in_rsrc)) { - BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; - return false; - } - - // Create target directory if needed - if (!fs::exists(vendor_path)) - fs::create_directories(vendor_path); - - // Copy JSON file - std::string error_message; - CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); - if (cfr != CopyFileResult::SUCCESS) { - BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; - return false; - } - - // Copy the vendor directory (if it exists) - auto dir_in_rsrc = rsrc_path / bundle; - auto dir_in_vendors = vendor_path / bundle; - - if (fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { - // Remove existing directory - if (fs::exists(dir_in_vendors)) - fs::remove_all(dir_in_vendors); - fs::create_directories(dir_in_vendors); - - // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) - // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf - auto file_filter = [](const std::string name) -> bool { - return boost::iends_with(name, ".stl") || - boost::iends_with(name, ".png") || - boost::iends_with(name, ".svg") || - boost::iends_with(name, ".jpeg") || - boost::iends_with(name, ".jpg") || - boost::iends_with(name, ".3mf"); - }; - - copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); - } - - BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; - - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); - return false; - } - } - - return true; -} - void save_string_file(const boost::filesystem::path& p, const std::string& str) { boost::nowide::ofstream file; diff --git a/src/slic3r/Config/Snapshot.cpp b/src/slic3r/Config/Snapshot.cpp index 4b071994fc..a7135eac6f 100644 --- a/src/slic3r/Config/Snapshot.cpp +++ b/src/slic3r/Config/Snapshot.cpp @@ -432,14 +432,9 @@ const Snapshot& SnapshotDB::take_snapshot(const AppConfig &app_config, Snapshot: cfg.models_variants_installed.erase(it ++); else ++ it; - // Read the active config bundle, parse the config version. - PresetBundle bundle; - //BBS: change directoties by design - //bundle.load_configbundle((data_dir / PRESET_SYSTEM_DIR / (cfg.name + ".ini")).string(), PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - bundle.load_vendor_configs_from_json((data_dir/PRESET_SYSTEM_DIR).string(), cfg.name, PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - for (const auto &vp : bundle.vendors) - if (vp.second.id == cfg.name) - cfg.version.config_version = vp.second.config_version; + // Orca: the version the vendor is installed at, read from its profile or — + // where the cache is the whole installation — from the cache's own stamp. + cfg.version.config_version = installed_vendor_version(cfg.name); snapshot.vendor_configs.emplace_back(std::move(cfg)); } diff --git a/src/slic3r/GUI/ConfigWizard.cpp b/src/slic3r/GUI/ConfigWizard.cpp index 0bbbc15f87..dba8699105 100644 --- a/src/slic3r/GUI/ConfigWizard.cpp +++ b/src/slic3r/GUI/ConfigWizard.cpp @@ -66,41 +66,41 @@ using Config::SnapshotDB; // Configuration data structures extensions needed for the wizard //BBS: set BBL as default -bool Bundle::load(fs::path source_path, bool ais_in_resources, bool ais_bbl_bundle) +bool Bundle::load(fs::path dir, const std::string &vendor_name, bool ais_in_resources, bool ais_bbl_bundle) { this->preset_bundle = std::make_unique(); this->is_in_resources = ais_in_resources; this->is_bbl_bundle = ais_bbl_bundle; - std::string path_string = source_path.string(); - std::string parent_path = source_path.parent_path().string(); //BBS: add json logic for vendor bundles - std::string vendor_name = source_path.filename().string(); - if (Slic3r::is_json_file(path_string)) { - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - } - else + // Orca: served from the vendor's preset cache where one covers it — which is + // how a shipped build carries its vendors — and parsed from the JSONs otherwise. + // A vendor that can be neither read nor parsed — a cache the build cannot use + // with the preset JSONs behind it pruned, say — is one the wizard cannot offer. + // Every other vendor still can be, so it is left out rather than thrown over. + size_t presets_loaded = 0; + try { + auto [config_substitutions, loaded] = preset_bundle->load_vendor_configs_from_json( + dir.string(), vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + UNUSED(config_substitutions); + // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. + assert(config_substitutions.empty()); + presets_loaded = loaded; + } catch (const std::exception &e) { + BOOST_LOG_TRIVIAL(fatal) << boost::format("Vendor bundle: `%1%`: cannot be loaded, leaving it out: %2%") % vendor_name % e.what(); return false; - - // Throw when parsing invalid configuration. Only valid configuration is supposed to be provided over the air. - //BBS: add json logic for vendor bundles - auto [config_substitutions, presets_loaded] = preset_bundle->load_vendor_configs_from_json( - parent_path, vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); - UNUSED(config_substitutions); - // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. - assert(config_substitutions.empty()); + } auto first_vendor = preset_bundle->vendors.begin(); if (first_vendor == preset_bundle->vendors.end()) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % vendor_name; return false; } if (presets_loaded == 0) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % vendor_name; return false; - } + } - BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % path_string % presets_loaded; + BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % vendor_name % presets_loaded; this->vendor_profile = &first_vendor->second; return true; } @@ -125,15 +125,10 @@ BundleMap BundleMap::load() //Orca: add custom as default //Orca: add json logic for vendor bundle - auto orca_bundle_path = (vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - auto orca_bundle_rsrc = false; - if (!boost::filesystem::exists(orca_bundle_path)) { - orca_bundle_path = (rsrc_vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - orca_bundle_rsrc = true; - } { + const bool from_rsrc = ! is_vendor_installed(PresetBundle::ORCA_DEFAULT_BUNDLE); Bundle bbl_bundle; - if (bbl_bundle.load(std::move(orca_bundle_path), orca_bundle_rsrc, true)) + if (bbl_bundle.load(from_rsrc ? rsrc_vendor_dir : vendor_dir, PresetBundle::ORCA_DEFAULT_BUNDLE, from_rsrc, true)) res.emplace(PresetBundle::ORCA_DEFAULT_BUNDLE, std::move(bbl_bundle)); } @@ -141,18 +136,13 @@ BundleMap BundleMap::load() // and then additionally from resources/profiles. bool is_in_resources = false; for (auto dir : { &vendor_dir, &rsrc_vendor_dir }) { - for (const auto &dir_entry : boost::filesystem::directory_iterator(*dir)) { - //BBS: add json logic for vendor bundle - if (Slic3r::is_json_file(dir_entry.path().string())) { - std::string id = dir_entry.path().stem().string(); // stem() = filename() without the trailing ".json" part + for (const std::string &id : vendor_names_in(*dir)) { + // Don't load this bundle if we've already loaded it. + if (res.find(id) != res.end()) { continue; } - // Don't load this bundle if we've already loaded it. - if (res.find(id) != res.end()) { continue; } - - Bundle bundle; - if (bundle.load(dir_entry.path(), is_in_resources)) - res.emplace(std::move(id), std::move(bundle)); - } + Bundle bundle; + if (bundle.load(*dir, id, is_in_resources)) + res.emplace(id, std::move(bundle)); } is_in_resources = true; diff --git a/src/slic3r/GUI/ConfigWizard_private.hpp b/src/slic3r/GUI/ConfigWizard_private.hpp index 364d378b42..7b9674b216 100644 --- a/src/slic3r/GUI/ConfigWizard_private.hpp +++ b/src/slic3r/GUI/ConfigWizard_private.hpp @@ -71,9 +71,11 @@ struct Bundle Bundle() = default; Bundle(Bundle&& other); + // Load the vendor `vendor_name` as it is installed in `dir`, from its preset + // cache or its profile JSONs, whichever is usable. // Returns false if not loaded. Reason for that is logged as boost::log error. //BBS: set BBL as default - bool load(fs::path source_path, bool is_in_resources, bool is_bbl_bundle = false); + bool load(fs::path dir, const std::string &vendor_name, bool is_in_resources, bool is_bbl_bundle = false); const std::string& vendor_id() const { return vendor_profile->id; } }; diff --git a/src/slic3r/GUI/CreatePresetsDialog.cpp b/src/slic3r/GUI/CreatePresetsDialog.cpp index 1bd80d5f00..33f49c2a38 100644 --- a/src/slic3r/GUI/CreatePresetsDialog.cpp +++ b/src/slic3r/GUI/CreatePresetsDialog.cpp @@ -2201,25 +2201,14 @@ bool CreatePrinterPresetDialog::load_system_and_user_presets_with_curr_model(Pre } else { selected_vendor_id = m_printer_preset_vendor_selected.id; - if (boost::filesystem::exists(boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(); - } else if (boost::filesystem::exists(boost::filesystem::path(Slic3r::resources_dir()) / "profiles" / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::resources_dir()) / "profiles").string(); - } - - if (preset_path.empty()) { - BOOST_LOG_TRIVIAL(info) << "Preset path was not found"; - MessageDialog dlg(this, _L("Preset path was not found; please reselect vendor."), wxString(SLIC3R_APP_FULL_NAME) + " - " + _L("Info"), - wxYES_NO | wxYES_DEFAULT | wxCENTRE); - dlg.ShowModal(); - return false; - } - try { // Pass the app's preset bundle (which already holds OrcaFilamentLibrary) as the base // bundle so vendor filaments that inherit OFL bases resolve via the existing // cross-vendor inheritance path. - temp_preset_bundle.load_vendor_configs_from_json(preset_path, selected_vendor_id, + // Orca: served from the vendor's preset cache where one covers it — a shipped + // build carries that instead of the raw preset JSONs — and parsed otherwise. + temp_preset_bundle.load_vendor_configs_from_json((boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(), + selected_vendor_id, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent, wxGetApp().preset_bundle); diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index 0d2f6c724b..a076fb635e 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -2,6 +2,7 @@ #include "ConfigWizard.hpp" #include +#include #include #include #include @@ -9,6 +10,7 @@ #include "I18N.hpp" #include "libslic3r/AppConfig.hpp" #include "libslic3r/Config.hpp" +#include "libslic3r/Preset.hpp" #include "libslic3r/PresetBundle.hpp" #include "slic3r/GUI/wxExtensions.hpp" #include "slic3r/GUI/GUI_App.hpp" @@ -41,8 +43,6 @@ using namespace nlohmann; namespace Slic3r { namespace GUI { -json m_ProfileJson; - static wxString update_custom_filaments() { json m_Res = json::object(); @@ -191,11 +191,10 @@ GuideFrame::GuideFrame(GUI_App *pGUI, long style) GuideFrame::~GuideFrame() { m_destroy = true; - if (m_load_task && m_load_task->joinable()) { + *m_cancel_token = true; // signal any queued CallAfter lambdas before join + if (m_load_task && m_load_task->joinable()) m_load_task->join(); - delete m_load_task; - m_load_task = nullptr; - } + m_load_task.reset(); if (m_browser) { delete m_browser; m_browser = nullptr; @@ -301,15 +300,65 @@ void GuideFrame::OnNavigationRequest(wxWebViewEvent &evt) /** * Callback invoked when a navigation request was accepted */ +void GuideFrame::init_guide_paths() +{ + m_ProfileJson = json::parse("{}"); + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + + vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); + rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); + orca_bundle_rsrc = true; + + if (boost::filesystem::exists(vendor_dir)) { + for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { + if (!boost::filesystem::is_directory(entry) && + boost::iequals(entry.path().extension().string(), ".json") && + !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { + orca_bundle_rsrc = false; + break; + } + } + } + + auto lib_json = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + m_OrcaFilaLibPath = boost::filesystem::exists(vendor_dir / lib_json) + ? (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string() + : (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); +} + +void GuideFrame::on_profile_loaded() +{ + // Must be called on the main thread. + SaveProfileData(); + const std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents:\n" << strAll; + json res; + res["command"] = "userguide_profile_load_finish"; + res["sequence_id"] = "10001"; + RunScript(wxString::Format("HandleStudio(%s)", res.dump(-1, ' ', true))); +} + void GuideFrame::OnNavigationComplete(wxWebViewEvent &evt) { //wxLogMessage("%s", "Navigation complete; url='" + evt.GetURL() + "'"); if (!bFirstComplete) { - m_load_task = new boost::thread(boost::bind(&GuideFrame::LoadProfileData, this)); - // boost::thread LoadProfileThread(boost::bind(&GuideFrame::LoadProfileData, this)); - //LoadProfileThread.detach(); - bFirstComplete = true; + try { + init_guide_paths(); + if (BuildProfileDataFromPresetBundle()) { + if (!m_destroy) + on_profile_loaded(); + } else { + // Presets not yet in memory — delegate to background thread. + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", init error: " << e.what(); + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } } m_browser->Show(); @@ -762,11 +811,9 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle bool check_unsaved_preset_changes = false; std::vector install_bundles; std::vector remove_bundles; - const auto vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); for (const auto &it : enabled_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir/(it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -777,8 +824,7 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle if (it.second.size() > 0) { if (enabled_vendors.find(it.first) != enabled_vendors.end()) continue; - auto vendor_file = vendor_dir/(it.first + ".json"); - if (fs::exists(vendor_file)) { + if (is_vendor_installed(it.first)) { remove_bundles.emplace_back(it.first); } } @@ -1127,99 +1173,256 @@ int GuideFrame::GetFilamentInfo( std::string VendorDirectory, json & pFilaList, return status; } -int GuideFrame::LoadProfileData() +bool GuideFrame::BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors) { try { - m_ProfileJson = json::parse("{}"); + // Models from vendor profiles + for (const auto& [vendor_id, vp] : bundle.vendors) { + for (const auto& model : vp.models) { + std::string nozzle_str; + for (const auto& v : model.variants) { + if (!nozzle_str.empty()) nozzle_str += ";"; + nozzle_str += v.name; + } + std::string materials_str; + for (const auto& m : model.default_materials) { + if (!materials_str.empty()) materials_str += ";"; + materials_str += m; + } + boost::filesystem::path cover_path = + (boost::filesystem::path(resources_dir()) / "profiles" / vp.id / (model.id + "_cover.png")) + .make_preferred(); + if (!boost::filesystem::exists(cover_path)) + cover_path = + (boost::filesystem::path(resources_dir()) / "web/image/printer" / (model.id + "_cover.png")) + .make_preferred(); + + json entry; + entry["model"] = model.id; + entry["name"] = model.name; + entry["vendor"] = vp.id; + entry["nozzle_diameter"] = nozzle_str; + entry["materials"] = materials_str; + entry["cover"] = cover_path.string(); + entry["nozzle_selected"] = ""; + entry["sub_path"] = ""; + m_ProfileJson["model"].push_back(entry); + } + } + + // Machine map: preset name -> {model, nozzle variant} + for (const Preset& p : bundle.printers()) { + if (!p.is_system || !p.vendor) continue; + const auto* printer_model = p.config.option("printer_model"); + const auto* printer_variant = p.config.option("printer_variant"); + if (!printer_model || printer_model->value.empty() || !printer_variant) continue; + + json mach; + mach["model"] = printer_model->value; + mach["nozzle"] = printer_variant->value; + m_ProfileJson["machine"][p.name] = mach; + } + + // Filament map from system filament presets (vendor/type already resolved in config) + for (const Preset& p : bundle.filaments()) { + if (!p.is_system || !p.vendor) continue; + const auto* fila_vendor = p.config.option("filament_vendor"); + const auto* fila_type = p.config.option("filament_type"); + const auto* compat_printers = p.config.option("compatible_printers"); + + std::string vendor = (fila_vendor && !fila_vendor->values.empty()) ? fila_vendor->values[0] : ""; + std::string type = (fila_type && !fila_type->values.empty()) ? fila_type->values[0] : ""; + + std::string model_list; + if (compat_printers) { + for (const std::string& pname : compat_printers->values) { + if (m_ProfileJson["machine"].contains(pname)) { + std::string m = m_ProfileJson["machine"][pname]["model"]; + std::string n = m_ProfileJson["machine"][pname]["nozzle"]; + model_list += "[" + m + "++" + n + "]"; + } + } + } + + json ff; + ff["name"] = p.name; + ff["sub_path"] = p.file; + ff["vendor"] = vendor; + ff["type"] = type; + ff["models"] = model_list; + ff["selected"] = 0; + m_ProfileJson["filament"][p.name] = ff; + } + + // Process list from visible system print presets + for (const Preset& p : bundle.prints()) { + if (!p.is_system || !p.vendor || !p.is_visible) continue; + json entry; + entry["name"] = p.name; + entry["sub_path"] = p.file; + m_ProfileJson["process"].push_back(entry); + } + + if (require_all_resource_vendors) { + // If rsrc_vendor_dir has vendor JSONs not covered by the current bundle, the + // bundle is incomplete (e.g. dev env where data_dir/system only has + // OrcaFilamentLibrary+Custom). Fall back so LoadProfileFamily reads both dirs. + try { + for (const auto& e : boost::filesystem::directory_iterator(rsrc_vendor_dir)) { + if (e.path().extension().string() != ".json") continue; + const std::string stem = e.path().stem().string(); + if (bundle.vendors.find(stem) == bundle.vendors.end()) { + BOOST_LOG_TRIVIAL(info) << "GuideFrame: vendor '" << stem + << "' in resources but not in preset_bundle — falling back to JSON loading"; + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + return false; + } + } + } catch (const std::exception&) {} + } + + BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data (" + << m_ProfileJson["model"].size() << " models, " + << m_ProfileJson["machine"].size() << " machines, " + << m_ProfileJson["filament"].size() << " filaments)"; + return !m_ProfileJson["machine"].empty(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileJson failed: " << e.what() + << " — falling back to JSON loading"; m_ProfileJson["model"] = json::array(); m_ProfileJson["machine"] = json::object(); m_ProfileJson["filament"] = json::object(); m_ProfileJson["process"] = json::array(); + return false; + } +} - vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); +bool GuideFrame::BuildProfileDataFromPresetBundle() +{ + PresetBundle* pb = wxGetApp().preset_bundle; + if (!pb || pb->vendors.empty()) + return false; + return BuildProfileJson(*pb, /*require_all_resource_vendors=*/true); +} - // Orca: add custom as default - // Orca: add json logic for vendor bundle - orca_bundle_rsrc = true; +bool GuideFrame::BuildProfileDataFromVendors() +{ + try { + // Same vendor set and precedence as the JSON scan in LoadProfileData: a + // vendor in the user's system dir shadows the bundled one of that name. + // A vendor is named by its profile or, where a build ships preset caches + // instead, by its cache alone — so both forms name one here. + std::map vendor_files; + auto collect = [&vendor_files](const boost::filesystem::path& dir) { + boost::system::error_code ec; + if (!boost::filesystem::exists(dir, ec)) + return; + for (const auto& e : boost::filesystem::directory_iterator(dir, ec)) + if (Slic3r::is_json_file(e.path().string()) || e.path().extension() == ".opc") + vendor_files.emplace(e.path().stem().string(), e.path()); // first wins + }; + collect(vendor_dir); + collect(rsrc_vendor_dir); - // search if there exists a .json file in vendor_dir folder, if exists, set orca_bundle_rsrc to false - for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { - if (!boost::filesystem::is_directory(entry) && boost::iequals(entry.path().extension().string(), ".json") && !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { - orca_bundle_rsrc = false; - break; - } - } - - // load the default filament library first - std::set loaded_vendors; - auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); - if (boost::filesystem::exists(vendor_dir / filament_library_name)) { - m_OrcaFilaLibPath = (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); - } else { - m_OrcaFilaLibPath = (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); - } - loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - - //load custom bundle from user data path - boost::filesystem::directory_iterator endIter; - for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if(strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } + // Each vendor comes from its preset cache where one covers it, which is what + // makes this worth doing instead of the scan below; the filament library goes + // first because the others' filaments inherit from it, and resolving those on + // the vendors the cache does not cover needs it already loaded. + PresetBundle bundle; + auto load_vendor = [this](PresetBundle& into, const std::string& vendor, const PresetBundle* base) { + into.load_vendor_configs_from_json(vendor_dir.string(), vendor, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, base); + }; + const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); + if (vendor_files.count(filament_library)) + load_vendor(bundle, filament_library, nullptr); + for (const auto& entry : vendor_files) { if (m_destroy) - return 0; + return false; // as in the scan below: a vendor without a cache is parsed, and that takes time + const std::string& vendor = entry.first; + // A cache is only ever written for a versioned vendor; a JSON has to be + // asked, so that an unversioned one (blacklist.json) carrying no presets + // is passed over. + if (vendor == filament_library || + (entry.second.extension() != ".opc" && get_vendor_cache_version(entry.second.string()).empty())) + continue; + PresetBundle tmp; + load_vendor(tmp, vendor, &bundle); + bundle.merge_presets(std::move(tmp)); + } + if (bundle.vendors.empty()) + return false; + return BuildProfileJson(bundle, /*require_all_resource_vendors=*/false); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << " failed: " << e.what(); + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + return false; + } +} + +int GuideFrame::LoadProfileData() +{ + // Background thread: the fast path in OnNavigationComplete failed (presets not yet loaded). + // Loading order (fastest to slowest): + // 1. Load every vendor, from its preset cache wherever one covers it + // 2. Read all vendor JSONs by hand + try { + if (!BuildProfileDataFromVendors()) { + // Last resort — read all vendor JSONs + std::set loaded_vendors; + auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + if (boost::filesystem::exists(vendor_dir / filament_library_name)) + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); + else + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); + loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); + + boost::filesystem::directory_iterator endIter; + for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (m_destroy) return 0; + } + + boost::filesystem::directory_iterator others_endIter; + for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (m_destroy) return 0; + } } - boost::filesystem::directory_iterator others_endIter; - for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } - if (m_destroy) - return 0; - } - - wxGetApp().CallAfter([this] { - if (!m_destroy) { - //sync to appconfig first to populate current selections - SaveProfileData(); - - //sync to web after selections are populated - std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); - - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents: " << std::endl << strAll; - json m_Res = json::object(); - m_Res["command"] = "userguide_profile_load_finish"; - m_Res["sequence_id"] = "10001"; - wxString strJS = wxString::Format("HandleStudio(%s)", m_Res.dump(-1, ' ', true)); - - RunScript(strJS); - } + // Capture the cancel token by value (shared_ptr) so the lambda doesn't + // touch `this` if GuideFrame is destroyed before the event fires. + auto tok = m_cancel_token; + wxGetApp().CallAfter([this, tok] { + if (!*tok) + on_profile_loaded(); }); - } catch (std::exception& e) { - // wxLogMessage("GUIDE: load_profile_error %s ", e.what()); - // wxMessageBox(e.what(), "", MB_OK); - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what() << std::endl; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what(); } filament_info_cache.clear(); diff --git a/src/slic3r/GUI/WebGuideDialog.hpp b/src/slic3r/GUI/WebGuideDialog.hpp index fcdb0841db..71e2409ae5 100644 --- a/src/slic3r/GUI/WebGuideDialog.hpp +++ b/src/slic3r/GUI/WebGuideDialog.hpp @@ -30,10 +30,14 @@ #include "libslic3r/PresetBundle.hpp" #include "slic3r/Utils/PresetUpdater.hpp" +#include +#include #include #include +#include + namespace Slic3r { namespace GUI { class GuideFrame : public DPIDialog @@ -78,6 +82,11 @@ public: int LoadProfileData(); int SaveProfileData(); int LoadProfileFamily(std::string strVendor, std::string strFilePath); + void init_guide_paths(); + void on_profile_loaded(); + bool BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors); + bool BuildProfileDataFromPresetBundle(); + bool BuildProfileDataFromVendors(); int SaveProfile(); int GetFilamentInfo( std::string VendorDirectory,json & pFilaList, std::string filepath, std::string &sVendor, std::string &sType); @@ -112,8 +121,11 @@ private: //First Load bool bFirstComplete{false}; - bool m_destroy{false}; - boost::thread* m_load_task{ nullptr }; + std::atomic m_destroy{false}; + // Shared cancel token captured by CallAfter lambdas so they don't touch + // `this` after the destructor has run and the object is freed. + std::shared_ptr> m_cancel_token{std::make_shared>(false)}; + std::unique_ptr m_load_task; // User Config bool PrivacyUse; @@ -123,6 +135,7 @@ private: bool InstallNetplugin; bool network_plugin_ready {false}; + json m_ProfileJson; json m_OrcaFilaList; std::string m_OrcaFilaLibPath; diff --git a/src/slic3r/Utils/PresetUpdater.cpp b/src/slic3r/Utils/PresetUpdater.cpp index 9ec57fd653..0e94d00ceb 100644 --- a/src/slic3r/Utils/PresetUpdater.cpp +++ b/src/slic3r/Utils/PresetUpdater.cpp @@ -1042,47 +1042,48 @@ void PresetUpdater::priv::check_installed_vendor_profiles() const std::set bundles; // Orca: always install filament library bundles.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - for (auto &dir_entry : boost::filesystem::directory_iterator(rsrc_path)) { - const auto &path = dir_entry.path(); - std::string file_path = path.string(); - if (is_json_file(file_path)) { - const auto path_in_vendor = vendor_path / path.filename(); - std::string vendor_name = path.filename().string(); - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - if (bundles.find(vendor_name) != bundles.end())continue; + // A vendor is named by its profile or, where the build ships preset caches + // instead of the raw profile JSONs, by its cache alone. + for (const std::string &vendor_name : vendor_names_in(rsrc_path)) { + if (bundles.find(vendor_name) != bundles.end())continue; - const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE - || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); - if (enabled_config_update) { - if ( fs::exists(path_in_vendor)) { - if (is_vendor_enabled) { - Semver resource_ver = get_version_from_json(file_path); - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE + || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); + if (enabled_config_update) { + if (is_vendor_installed(vendor_name)) { + if (is_vendor_enabled) { + // Orca: whichever form of the vendor resources ships at the newer + // version is the one installing lays down, and the one to judge + // what is installed against. + Semver resource_ver = resource_vendor_version(vendor_name); + // Orca: a vendor installed as a preset cache has no profile + // beside it; the version it was installed at is in the cache. + Semver vendor_ver = installed_vendor_version(vendor_name); - bool version_match = ((resource_ver.maj() == vendor_ver.maj()) && (resource_ver.min() == vendor_ver.min())); + bool version_match = ((resource_ver.maj() == vendor_ver.maj()) && (resource_ver.min() == vendor_ver.min())); - if (!version_match || (vendor_ver < resource_ver)) { - BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor "< 0) { @@ -1162,11 +1163,12 @@ Updates PresetUpdater::priv::get_config_updates(const Semver &old_slic3r_version auto filament_in_cache = (cache_profile_path / vendor_name / PRESET_FILAMENT_NAME); auto machine_in_cache = (cache_profile_path / vendor_name / PRESET_PRINTER_NAME); - if (( fs::exists(path_in_vendor)) + if (is_vendor_installed(vendor_name) || fs::exists(print_in_cache) || fs::exists(filament_in_cache) || fs::exists(machine_in_cache)) { - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + // Orca: a vendor installed as a preset cache carries its version there. + Semver vendor_ver = installed_vendor_version(vendor_name); std::map key_values; std::vector keys(3); diff --git a/tests/libslic3r/CMakeLists.txt b/tests/libslic3r/CMakeLists.txt index 7524c27479..66a181873b 100644 --- a/tests/libslic3r/CMakeLists.txt +++ b/tests/libslic3r/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(${_TEST_NAME}_tests test_preset_bundle_loading.cpp test_preset_setting_id.cpp test_preset_diff.cpp + test_vendor_cache.cpp test_elephant_foot_compensation.cpp test_geometry.cpp test_multimaterial_segmentation.cpp diff --git a/tests/libslic3r/test_preset_bundle_loading.cpp b/tests/libslic3r/test_preset_bundle_loading.cpp index 351535dd9d..d2db84f077 100644 --- a/tests/libslic3r/test_preset_bundle_loading.cpp +++ b/tests/libslic3r/test_preset_bundle_loading.cpp @@ -146,7 +146,7 @@ TEST_CASE("Current vendor type tolerates missing printer model", "[Preset][Bundl { PresetBundle bundle; - VendorProfile orca_vendor("ORCA"); + VendorProfile orca_vendor; orca_vendor.id = "ORCA"; VendorProfile::PrinterModel model; model.name = "Orca Test"; orca_vendor.models.emplace_back(model); diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp new file mode 100644 index 0000000000..c3b90fcde8 --- /dev/null +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -0,0 +1,1036 @@ +#include + +#include +#include +#include +#include + +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/PrintConfig.hpp" +#include "libslic3r/Utils.hpp" + +using namespace Slic3r; +namespace fs = boost::filesystem; + +namespace { + +struct TempDir { + fs::path path; + TempDir() { + path = fs::temp_directory_path() / fs::unique_path("orca-cache-test-%%%%-%%%%"); + fs::create_directories(path); + } + ~TempDir() { boost::system::error_code ec; fs::remove_all(path, ec); } +}; + +std::string write_vendor_json(const fs::path& dir, const std::string& vendor_id, + const std::string& version = "1.0.0") +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"version":")" << version << R"(","name":")" << vendor_id << R"("})"; + return p.string(); +} + +// One vendor profile with a single process preset beside it, as an install or an +// update lays it down: /.json plus //process/standard.json. +void write_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor + << R"(","process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}]})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; +} + +std::string write_versionless_vendor_json(const fs::path& dir, const std::string& vendor_id) +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"name":")" << vendor_id << R"("})"; + return p.string(); +} + +void corrupt_blob_byte(const std::string& path) +{ + std::fstream f(path, std::ios::in | std::ios::out | std::ios::binary); + f.seekp(30); + char b = 0; f.read(&b, 1); + f.seekp(30); + b ^= 0xFF; + f.write(&b, 1); +} + +// Patch cache_version (blob[0..3], i.e. file offset 20) and recompute CRC so the +// file passes the CRC check but fails the cache_version check in load_vendor_cache. +void patch_cache_version(const std::string& path, uint32_t wrong_version) +{ + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + if (data.size() < 24) return; + std::memcpy(&data[20], &wrong_version, 4); + boost::crc_32_type crc; + crc.process_bytes(&data[20], data.size() - 20); + const uint32_t new_crc = crc.checksum(); + std::memcpy(&data[16], &new_crc, 4); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); +} + +// Truncates the cache's PAYLOAD (everything after the 20-byte header) by +// `truncate_by` bytes and recomputes data_size/crc32 in the header, exactly +// as write_cache_blob computes them, so read_cache_blob's size and CRC checks +// still pass but cereal runs out of bytes partway through deserializing the +// body — exercising load_vendor_cache's catch block instead of its early +// (pre-body) rejection paths. +void truncate_payload_and_fix_header(const std::string& path, size_t truncate_by) +{ + constexpr size_t header_size = 20; // magic(4) + version(4) + data_size(8) + crc32(4) + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + REQUIRE(data.size() > header_size + truncate_by); + const size_t new_payload_size = data.size() - header_size - truncate_by; + const uint64_t data_size_field = static_cast(new_payload_size); + boost::crc_32_type crc; + crc.process_bytes(&data[header_size], new_payload_size); + const uint32_t crc_field = crc.checksum(); + std::memcpy(&data[8], &data_size_field, sizeof(data_size_field)); // data_size offset + std::memcpy(&data[16], &crc_field, sizeof(crc_field)); // crc32 offset + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(header_size + new_payload_size)); +} + +void add_vendor(PresetBundle& bundle, const std::string& vendor_id, + const std::string& name = "", Semver ver = Semver(1, 0, 0)) +{ + VendorProfile vp; vp.id = vendor_id; + vp.name = name.empty() ? vendor_id + " Corp" : name; + vp.config_version = ver; + bundle.vendors.emplace(vendor_id, vp); +} + +Preset& add_system_preset(PresetCollection& coll, const std::string& name, + const VendorProfile* vp) +{ + Preset& p = coll.load_preset("", name, DynamicPrintConfig(coll.default_preset().config), false); + p.is_system = true; + p.vendor = vp; + return p; +} + +static bool save_one_vendor(const PresetBundle& bundle, const std::string& path, + const std::string& vendor, const std::string& vendor_version, + const std::string& lib_version = "1.0.0") +{ + return bundle.save_vendor_cache(path, vendor, vendor_version, lib_version); +} + +// resources_dir()/data_dir() are process-wide, so restore them however the test +// leaves — including through a failed REQUIRE — to stay green under --order rand. +struct ScopedDirs { + std::string prev_data{data_dir()}, prev_rsrc{resources_dir()}; + ScopedDirs(const fs::path& data, const fs::path& rsrc) + { + set_data_dir(data.string()); + set_resources_dir(rsrc.string()); + } + ~ScopedDirs() { set_data_dir(prev_data); set_resources_dir(prev_rsrc); } +}; + +// Helper: filter a collection by vendor_id. +std::vector presets_for(const PresetCollection& coll, const std::string& vendor_id) +{ + std::vector out; + for (const Preset& p : coll()) + if (p.is_system && p.vendor && p.vendor->id == vendor_id) + out.push_back(&p); + return out; +} + +} // namespace + +namespace Slic3r { +inline bool operator==(const VendorProfile::PrinterVariant& a, const VendorProfile::PrinterVariant& b) { return a.name == b.name; } +inline bool operator==(const VendorProfile::PrinterModel& a, const VendorProfile::PrinterModel& b) +{ + return a.id == b.id && a.name == b.name && a.model_id == b.model_id && a.technology == b.technology + && a.family == b.family && a.variants == b.variants && a.default_materials == b.default_materials + && a.not_support_bed_types == b.not_support_bed_types && a.bed_model == b.bed_model + && a.bed_texture == b.bed_texture && a.image_bed_type == b.image_bed_type + && a.bottom_texture_end_name == b.bottom_texture_end_name + && a.use_double_extruder_default_texture == b.use_double_extruder_default_texture + && a.bottom_texture_rect == b.bottom_texture_rect + && a.bottom_texture_rect_longer == b.bottom_texture_rect_longer + && a.middle_texture_rect == b.middle_texture_rect && a.hotend_model == b.hotend_model; +} +} // namespace Slic3r + +static bool vendor_deep_equal(const VendorProfile& a, const VendorProfile& b) +{ + return a.name == b.name && a.id == b.id && a.config_version == b.config_version + && a.config_update_url == b.config_update_url && a.changelog_url == b.changelog_url + && a.models == b.models && a.default_filaments == b.default_filaments + && a.default_sla_materials == b.default_sla_materials; +} + +static bool preset_deep_equal(const Preset& a, const Preset& b) +{ + return a.type == b.type && a.is_default == b.is_default && a.is_external == b.is_external + && a.is_system == b.is_system && a.is_visible == b.is_visible && a.is_dirty == b.is_dirty + && a.is_compatible == b.is_compatible && a.is_project_embedded == b.is_project_embedded + && a.name == b.name && a.file == b.file && a.loaded == b.loaded + && a.config.equals(b.config) + && a.alias == b.alias && a.renamed_from == b.renamed_from + && a.m_excluded_from == b.m_excluded_from && a.m_from_orca_filament_lib == b.m_from_orca_filament_lib + && a.bundle_id == b.bundle_id && a.version == b.version && a.ini_str == b.ini_str + && a.setting_id == b.setting_id && a.filament_id == b.filament_id && a.user_id == b.user_id + && a.base_id == b.base_id && a.sync_info == b.sync_info && a.description == b.description + && a.updated_time == b.updated_time && a.key_values == b.key_values; +} + +TEST_CASE("a saved cache loads back with names, aliases and filament ids intact", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + const VendorProfile* vp = &src.vendors.at(vid); + + Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", vp); + fp.alias = "Acme PLA"; + fp.filament_id = "GFL_acme_pla"; + add_system_preset(src.printers, vid + " Printer 0.4", vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 1); + CHECK(fi[0]->name == vid + " PLA @0.4"); + CHECK(fi[0]->alias == "Acme PLA"); + CHECK(fi[0]->filament_id == "GFL_acme_pla"); + REQUIRE(pr.size() == 1); + CHECK(pr[0]->name == vid + " Printer 0.4"); +} + +TEST_CASE("loading a missing cache file returns false", "[VendorCache]") +{ + TempDir tmp; + PresetBundle out; + REQUIRE(!out.load_vendor_cache((tmp.path / "nonexistent.opc").string(), "Acme", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache with a corrupted byte is rejected by the CRC check", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + corrupt_blob_byte(cache.string()); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("two vendors produce two independent cache files", "[VendorCache]") +{ + TempDir tmp; + const fs::path cacheA = tmp.path / "vendorA.opc"; + const fs::path cacheB = tmp.path / "vendorB.opc"; + + PresetBundle srcA; + add_vendor(srcA, "VendorA"); + add_system_preset(srcA.filaments, "VendorA PLA", &srcA.vendors.at("VendorA")); + REQUIRE(save_one_vendor(srcA, cacheA.string(), "VendorA", "1.0.0")); + + PresetBundle srcB; + add_vendor(srcB, "VendorB"); + add_system_preset(srcB.filaments, "VendorB PLA", &srcB.vendors.at("VendorB")); + REQUIRE(save_one_vendor(srcB, cacheB.string(), "VendorB", "1.0.0")); + + // Corrupt only vendor B's file; vendor A's must be unaffected. + corrupt_blob_byte(cacheB.string()); + + PresetBundle outA; + REQUIRE(outA.load_vendor_cache(cacheA.string(), "VendorA", "1.0.0","1.0.0")); + REQUIRE(outA.vendors.count("VendorA") == 1); + REQUIRE(presets_for(outA.filaments, "VendorA").size() == 1); + + PresetBundle outB; + REQUIRE(!outB.load_vendor_cache(cacheB.string(), "VendorB", "1.0.0","1.0.0")); + REQUIRE(outB.vendors.empty()); +} + +TEST_CASE("vendor profile fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + VendorProfile vp; vp.id = vid; + vp.name = "Acme Corporation"; + vp.config_version = Semver(2, 5, 1); + VendorProfile::PrinterModel model; + model.id = "AcmePro"; + model.name = "Acme Pro"; + VendorProfile::PrinterVariant v0_4; v0_4.name = "0.4"; + model.variants.push_back(v0_4); + vp.models.push_back(model); + src.vendors.emplace(vid, vp); + REQUIRE(save_one_vendor(src, cache.string(), vid, "2.5.1")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "2.5.1","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + const VendorProfile& gvp = out.vendors.at(vid); + REQUIRE(vendor_deep_equal(gvp, src.vendors.at(vid))); + // Spot-check the fields the old test asserted directly, so a + // vendor_deep_equal regression still points at what actually broke. + CHECK(gvp.id == vid); + CHECK(gvp.name == "Acme Corporation"); + REQUIRE(gvp.models.size() == 1); + CHECK(gvp.models[0].id == "AcmePro"); + CHECK(gvp.models[0].name == "Acme Pro"); + REQUIRE(gvp.models[0].variants.size() == 1); + CHECK(gvp.models[0].variants[0].name == "0.4"); +} + +TEST_CASE("config option values survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + Preset& fp = add_system_preset(src.filaments, vid + " PETG @0.4", &src.vendors.at(vid)); + fp.config.set_key_value("filament_type", new ConfigOptionStrings({"PETG"})); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + auto fi = presets_for(out.filaments, vid); + REQUIRE(fi.size() == 1); + const auto* ft = fi[0]->config.option("filament_type"); + REQUIRE(ft != nullptr); + REQUIRE(ft->values.size() >= 1); + CHECK(ft->values[0] == "PETG"); +} + +TEST_CASE("multiple presets in one collection all round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + const VendorProfile* vp = &src.vendors.at(vid); + + const std::vector fi_names = {vid + " PLA", vid + " PETG", vid + " ABS"}; + const std::vector pr_names = {vid + " Printer 0.4", vid + " Printer 0.6"}; + for (const auto& n : fi_names) add_system_preset(src.filaments, n, vp); + for (const auto& n : pr_names) add_system_preset(src.printers, n, vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 3); + REQUIRE(pr.size() == 2); + + std::set fi_got, pr_got; + for (const auto* p : fi) fi_got.insert(p->name); + for (const auto* p : pr) pr_got.insert(p->name); + for (const auto& n : fi_names) CHECK(fi_got.count(n) == 1); + for (const auto& n : pr_names) CHECK(pr_got.count(n) == 1); +} + +TEST_CASE("a truncated cache file is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "truncated.opc"; + { + std::ofstream f(cache.string(), std::ios::binary); + const char data[] = {0x4F, 0x52, 0x43}; + f.write(data, sizeof(data)); + } + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache with the wrong magic number is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + { + std::fstream f(cache.string(), std::ios::in | std::ios::out | std::ios::binary); + const uint32_t bad = 0xDEADBEEFu; + f.write(reinterpret_cast(&bad), sizeof(bad)); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("a vendor with no presets saves and loads cleanly", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + VendorProfile vp; vp.id = vid; + vp.name = "Acme Corporation"; + vp.config_version = Semver(1, 0, 0); + src.vendors.emplace(vid, vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + CHECK(out.vendors.at(vid).id == vid); + CHECK(out.vendors.at(vid).name == "Acme Corporation"); + CHECK(presets_for(out.filaments, vid).empty()); + CHECK(presets_for(out.printers, vid).empty()); + CHECK(presets_for(out.prints, vid).empty()); +} + +TEST_CASE("all preset metadata fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", &src.vendors.at(vid)); + fp.setting_id = "sid-test-001"; + fp.description = "A test filament preset"; + fp.bundle_id = "bundle-xyz"; + fp.user_id = "user-abc"; + fp.base_id = "base-123"; + fp.sync_info = "update"; + fp.updated_time = 1700000000LL; + fp.key_values = {{"color", "red"}, {"diameter", "1.75"}}; + fp.ini_str = "[filament]\nnozzle_temperature = 230\n"; + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + const Preset* reloaded = out.filaments.find_preset(vid + " PLA @0.4", false); + REQUIRE(reloaded != nullptr); + const Preset* original = src.filaments.find_preset(vid + " PLA @0.4", false); + REQUIRE(original != nullptr); + REQUIRE(preset_deep_equal(*reloaded, *original)); +} + +TEST_CASE("a cache with the wrong cache version is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + patch_cache_version(cache.string(), 0xFFFFFFFFu); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("a cache truncated mid-blob is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + { + std::ifstream in(cache.string(), std::ios::binary); + std::vector buf(30); // 20-byte header + 10 bytes of blob + in.read(buf.data(), 30); + in.close(); + std::ofstream out(cache.string(), std::ios::binary | std::ios::trunc); + out.write(buf.data(), 30); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("get_vendor_cache_version returns the version field, or nothing", "[VendorCache]") +{ + TempDir tmp; + const std::string versioned_path = write_vendor_json(tmp.path, "Acme", "2.5.1"); + const std::string versionless_path = write_versionless_vendor_json(tmp.path, "Beta"); + + // Semver::to_string() re-splits patch into two BBS sub-fields (patch/100, + // patch%100), so derive the expected string from the same Semver the JSON + // was written with rather than hardcoding its form. + const auto expected_ver = Semver::parse("2.5.1"); + REQUIRE(expected_ver.has_value()); + CHECK(get_vendor_cache_version(versioned_path) == expected_ver->to_string()); + + // A profile without a version has nothing to validate a cache against. + CHECK(get_vendor_cache_version(versionless_path).empty()); +} + +TEST_CASE("printer model bed texture fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + VendorProfile::PrinterModel model; + model.id = "N1"; + model.name = "Neat One"; + model.bottom_texture_rect_longer = "5,5,50,10"; + src.vendors.at(vid).models.push_back(model); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.at(vid).models.size() == 1); + REQUIRE(vendor_deep_equal(out.vendors.at(vid), src.vendors.at(vid))); + CHECK(out.vendors.at(vid).models[0].bottom_texture_rect_longer == "5,5,50,10"); +} + +TEST_CASE("a cache older than the vendor profile on disk is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.1","1.0.0")); +} + +TEST_CASE("a cache newer than the vendor profile on disk is used", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + add_system_preset(src.filaments, "Acme PLA", &src.vendors.at("Acme")); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.2.0", "2.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "1.0.0")); + CHECK(presets_for(out.filaments, "Acme").size() == 1); +} + +TEST_CASE("a cache built against an older filament library is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0", "2.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "2.0.1")); +} + +TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + + PresetBundle out; + // An unversioned vendor profile has no version to compare against. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "", "1.0.0")); + // Neither has an unversioned filament library. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "")); + // And a cache carrying no version of its own is unusable either way. + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "", "")); + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "1.0.0")); + REQUIRE(out.vendors.empty()); +} + +TEST_CASE("a vendor's cache is its whole installation", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources"; + const fs::path data = tmp.path / "data"; + fs::create_directories(rsrc / "profiles" / "Acme" / "machine"); + write_vendor_json(rsrc / "profiles", "Acme"); + std::ofstream((rsrc / "profiles" / "Acme" / "machine" / "printer.json").string()) << "{}"; + + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, (rsrc / "profiles" / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(data, rsrc); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + // The cache carries the presets, the vendor profile and the version they were + // built at, so it is installed on its own. + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); + CHECK(is_vendor_installed("Acme")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // A vendor with no cache is installed as its profile and preset JSONs instead, + // parsing them being the only way left to load it — and the cache the previous + // install left behind has to go, or it would shadow the profile just installed. + fs::remove(rsrc / "profiles" / "Acme.opc"); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(!fs::exists(data / "system" / "Acme.opc")); + CHECK(fs::exists(data / "system" / "Acme" / "machine" / "printer.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // Installing the cache again takes the profile and its preset JSONs back out. + REQUIRE(save_one_vendor(src, (rsrc / "profiles" / "Acme.opc").string(), "Acme", "1.0.0")); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); +} + +TEST_CASE("a vendor shipped as a cache alone is installed and loaded from it", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // A packaged build: every vendor is its cache, with no profile of any kind + // beside it — not even the filament library's, and every other vendor's cache + // is judged against the library in effect. + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle shipped_lib; + add_vendor(shipped_lib, lib, "Shipped Library"); + REQUIRE(save_one_vendor(shipped_lib, (rsrc / (lib + ".opc")).string(), lib, "1.0.0")); + PresetBundle shipped; + add_vendor(shipped, "Acme", "Shipped Acme"); + REQUIRE(save_one_vendor(shipped, (rsrc / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The version the build ships the vendor at comes from the cache, there being + // no profile to read it from. + CHECK(resource_vendor_version("Acme") == Semver(1, 0, 0)); + + // Nothing installed yet, so the library the shipped cache is judged against is + // the shipped one — itself a cache. + PresetBundle before; + before.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(before.vendors.at("Acme").name == "Shipped Acme"); + + REQUIRE(install_vendor_bundles_from_resources({lib, "Acme"})); + CHECK(fs::exists(user / "Acme.opc")); + CHECK(!fs::exists(user / "Acme.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + PresetBundle after; + after.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(after.vendors.at("Acme").name == "Shipped Acme"); +} + +TEST_CASE("a vendor cache installed in the data dir shadows the shipped one", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + write_vendor_tree(rsrc, "Acme", "1.0.0"); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle shipped; + add_vendor(shipped, "Acme", "Shipped Acme"); + REQUIRE(save_one_vendor(shipped, (rsrc / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The vendor profiles name the vendor "Acme"; the caches name it after where + // they came from, so the loaded name says which source answered. + auto loaded_name = [&user](PresetBundle& bundle) { + bundle.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + return bundle.vendors.at("Acme").name; + }; + + // Nothing installed yet: the shipped cache answers. + PresetBundle from_rsrc; + CHECK(loaded_name(from_rsrc) == "Shipped Acme"); + + // Installing a newer vendor profile makes the shipped cache too old for it, even + // though that cache still matches the profile sitting beside it in resources. + write_vendor_tree(user, "Acme", "2.0.0"); + write_vendor_json(user, PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle stale; + CHECK(loaded_name(stale) == "Acme"); + + // The cache installed alongside it does answer, and wins over the shipped one. + PresetBundle installed; + add_vendor(installed, "Acme", "Installed Acme"); + REQUIRE(save_one_vendor(installed, (user / "Acme.opc").string(), "Acme", "2.0.0")); + PresetBundle from_user; + CHECK(loaded_name(from_user) == "Installed Acme"); +} + +TEST_CASE("a cache installed with no profile beside it is used whatever its version", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, "Acme"); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + // Installed at an older version than the one now shipped in resources. Nothing + // sits beside it claiming to be newer, so the cache is what the vendor is. + PresetBundle src; + add_vendor(src, "Acme", "Installed Acme"); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "0.9.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "0.9.0"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Other").empty()); + CHECK(installed_vendor_version("Acme") == Semver(0, 9, 0)); + + // Loading the vendor takes the installed cache, not the newer shipped profile. + PresetBundle out; + out.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(out.vendors.at("Acme").name == "Installed Acme"); +} + +TEST_CASE("a vendor whose cache covers it is loaded without parsing any JSON", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + PresetBundle src; + add_vendor(src, "Acme", "Cached Acme"); + const VendorProfile* vp = &src.vendors.at("Acme"); + add_system_preset(src.filaments, "Acme PLA @0.4", vp); + add_system_preset(src.printers, "Acme Printer 0.4", vp); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // The cache is the whole installation — no profile, no preset JSONs — and the + // caller asks for the vendor exactly as it would for a JSON install. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + CHECK(substitutions.empty()); + CHECK(presets_loaded == 2); + CHECK(out.vendors.at("Acme").name == "Cached Acme"); + + // Nothing was written back: the presets never came from a parse. + CHECK(!fs::exists(user / "Acme.json")); +} + +TEST_CASE("a vendor whose cache is stale falls back to parsing its JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + // An update installed the vendor at 2.0.0; the cache next to it was built from + // the profile before that, so it no longer covers what is on disk. + write_vendor_tree(user, "Acme", "2.0.0"); + PresetBundle src; + add_vendor(src, "Acme", "Cached Acme"); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(2, 0, 0)); + + // A one-off parse like this one leaves the stale cache alone: only a bundle + // told its parses are complete writes one. + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "1.0.0"); + + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_vendor_cache_version((user / "Acme.json").string())); +} + +TEST_CASE("a vendor with nothing installed is parsed from the shipped profiles", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(user); + fs::create_directories(rsrc); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + write_vendor_tree(rsrc, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Neither a cache nor a profile in the directory asked for, so the vendor comes + // out of resources — which is where a build that ships caches keeps the JSONs a + // rejected cache has to be re-parsed from. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(1, 0, 0)); + + // The cache such a parse writes lands in the directory asked for, stamped with + // the version of the profile it was actually built from. + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_vendor_cache_version((rsrc / "Acme.json").string())); +} + +TEST_CASE("a cache with a mismatched vendor name is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "VendorA"); + REQUIRE(save_one_vendor(src, cache.string(), "VendorA", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "VendorB", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache is rejected against an unparsable version", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "not-a-version","1.0.0")); + REQUIRE(out.vendors.empty()); // rejection happens before the body is touched +} + +TEST_CASE("obsolete preset names survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + src.obsolete_presets.filaments = {"Old filament"}; + src.obsolete_presets.printers = {"Old printer"}; + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", "1.0.0","1.0.0")); + REQUIRE(out.obsolete_presets.filaments == std::vector{"Old filament"}); + REQUIRE(out.obsolete_presets.printers == std::vector{"Old printer"}); +} + +TEST_CASE("printer hold aliases survive a cache round-trip", "[VendorCache]") +{ + // m_printer_hold_alias has no public getter: is_alias_exist() looks it up, + // but only after finding the alias in m_map_alias_to_profile_name, which + // is populated solely by the (protected) system-preset load path, so it + // cannot be probed here without that machinery. Round-trip it the same + // way the general parity test does instead: save, load, re-save, and + // require the second file to be byte-identical to the first — if load + // dropped or mangled the hold-alias map, the two saves would differ. Kept + // deliberately preset-free so this test isolates the hold-alias map from + // the fuller parity test below. + TempDir tmp; + const fs::path cache1 = tmp.path / "c1.opc"; + const fs::path cache2 = tmp.path / "c2.opc"; + const std::string vid = "Acme"; + const std::string printer_name = vid + " Printer 0.4"; + + PresetBundle src; + add_vendor(src, vid); + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({printer_name})); + src.filaments.set_printer_hold_alias("Acme PLA", probe); + + REQUIRE(save_one_vendor(src, cache1.string(), vid, "1.0.0")); + PresetBundle loaded; + REQUIRE(loaded.load_vendor_cache(cache1.string(), vid, "1.0.0","1.0.0")); + REQUIRE(save_one_vendor(loaded, cache2.string(), vid, "1.0.0")); + + auto slurp = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + REQUIRE(slurp(cache1) == slurp(cache2)); +} + +TEST_CASE("inheritance maps survive a cache round-trip", "[VendorCache]") +{ + // m_config_maps/m_filament_id_maps are public data members of PresetBundle + // (no accessor needed); they matter only for the Orca filament library's + // own cache, since they are the inheritance base other vendors resolve + // against, but save_vendor_cache/load_vendor_cache archive them + // unconditionally regardless of which vendor is being cached. + TempDir tmp; + const fs::path cache = tmp.path / "lib.opc"; + const std::string lib_vendor = "OrcaFilamentLibrary"; + + PresetBundle src; + DynamicPrintConfig cfg; + cfg.set_key_value("filament_type", new ConfigOptionStrings({"PLA"})); + src.m_config_maps["Generic PLA"] = cfg; + src.m_filament_id_maps["Generic PLA"] = "GFL00"; + + REQUIRE(save_one_vendor(src, cache.string(), lib_vendor, "1.0.0", "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), lib_vendor, "1.0.0", "1.0.0")); + REQUIRE(out.m_filament_id_maps == src.m_filament_id_maps); + REQUIRE(out.m_config_maps.count("Generic PLA") == 1); + CHECK(out.m_config_maps.at("Generic PLA").equals(src.m_config_maps.at("Generic PLA"))); +} + +TEST_CASE("a loaded cache re-serializes to byte-identical output", "[VendorCache]") +{ + // The strongest cheap parity check: if load dropped or mangled anything + // that save writes (including the inheritance maps and hold aliases), + // the second file differs. + TempDir tmp; + const fs::path cache1 = tmp.path / "c1.opc", cache2 = tmp.path / "c2.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + const VendorProfile* vp = &src.vendors.at("Acme"); + Preset& f = add_system_preset(src.filaments, "Acme Filament", vp); + f.alias = "AF"; + add_system_preset(src.printers, "Acme Printer 0.4 nozzle", vp); + src.obsolete_presets.prints = {"Old print"}; + src.m_config_maps["Generic PLA"] = DynamicPrintConfig(); + src.m_filament_id_maps["Generic PLA"] = "GFL00"; + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({"Acme Printer 0.4 nozzle"})); + src.filaments.set_printer_hold_alias("AF", probe); + + REQUIRE(src.save_vendor_cache(cache1.string(), "Acme", "1.0.0", "1.0.0")); + PresetBundle loaded; + REQUIRE(loaded.load_vendor_cache(cache1.string(), "Acme", "1.0.0", "1.0.0")); + REQUIRE(loaded.save_vendor_cache(cache2.string(), "Acme", "1.0.0", "1.0.0")); + auto slurp = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + REQUIRE(slurp(cache1) == slurp(cache2)); +} + +TEST_CASE("a cache that fails mid-body deserialization is rejected and leaves the bundle clean", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path valid_cache = tmp.path / "valid.opc"; + const fs::path corrupt_cache = tmp.path / "corrupt.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA @0.4", &src.vendors.at(vid)); + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({vid + " Printer 0.4"})); + src.filaments.set_printer_hold_alias(vid + " PLA alias", probe); + + REQUIRE(save_one_vendor(src, valid_cache.string(), vid, "1.0.0")); + // Truncate the tail (obsolete-preset vectors + m_errors, per save_vendor_cache's + // field order) so the header's size/CRC still validate but cereal runs out of + // bytes partway through the body. Grow the cut if a given size ever stops + // throwing (e.g. after an unrelated field-order change to the cache format). + size_t truncate_by = 40; + bool throws = false; + for (; truncate_by <= 200; truncate_by += 8) { + fs::copy_file(valid_cache, corrupt_cache, fs::copy_option::overwrite_if_exists); + truncate_payload_and_fix_header(corrupt_cache.string(), truncate_by); + PresetBundle probe_bundle; + if (!probe_bundle.load_vendor_cache(corrupt_cache.string(), vid, "1.0.0","1.0.0")) { + throws = true; + break; + } + } + REQUIRE(throws); + + PresetBundle out; + // Pre-populate the target bundle with prior-cycle state: Fix 1(b) must + // clear this in the catch block, not just leave it from a previous cycle. + Preset stale_probe(Preset::TYPE_FILAMENT, "stale-probe"); + stale_probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({"Stale Printer"})); + out.filaments.set_printer_hold_alias("Stale alias", stale_probe); + + REQUIRE(!out.load_vendor_cache(corrupt_cache.string(), vid, "1.0.0","1.0.0")); + CHECK(out.vendors.empty()); + + // No public getter exists for m_printer_hold_alias, so probe it the same + // way the round-trip tests do: serialize `out` and a never-touched fresh + // bundle under the same vendor/keys and require the bytes to match. Since + // save_collection archives the hold-alias map, any leftover "Stale alias" + // entry would make the two files differ. + const fs::path out_after = tmp.path / "out_after.opc"; + const fs::path clean_ref = tmp.path / "clean_ref.opc"; + PresetBundle clean; + REQUIRE(out.save_vendor_cache(out_after.string(), vid, "1.0.0","1.0.0")); + REQUIRE(clean.save_vendor_cache(clean_ref.string(), vid, "1.0.0","1.0.0")); + auto slurp2 = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + CHECK(slurp2(out_after) == slurp2(clean_ref)); + + // The recovery must leave a bundle a caller can still load a good cache into. + REQUIRE(out.load_vendor_cache(valid_cache.string(), vid, "1.0.0","1.0.0")); + CHECK(out.vendors.count(vid) == 1); + CHECK(presets_for(out.filaments, vid).size() == 1); +}