diff --git a/.gitattributes b/.gitattributes index 4cab1f4d26..441bdfe1eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Set the default behavior, in case people don't have core.autocrlf set. * text=auto + +# Shell scripts are run by Git Bash on Windows CI, which cannot read a script +# with CRLF line endings: it fails on the first line. Windows checkouts default +# to core.autocrlf=true, so keep these LF whatever the platform. +*.sh text eol=lf diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index a7652c3bd6..3a42983ce4 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -162,6 +162,14 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (macOS) + if: runner.os == 'macOS' && !inputs.macos-combine-only + working-directory: ${{ github.workspace }} + shell: bash + # The bundle was already packed from resources/, so the caches have to be + # installed into it here; the source tree keeps its JSONs for later jobs. + run: ./scripts/build_preset_cache.sh -b build/${{ inputs.arch }} build/${{ inputs.arch }}/OrcaSlicer/OrcaSlicer.app/Contents/Resources/profiles + - name: Pack macOS app bundle ${{ inputs.arch }} if: runner.os == 'macOS' && !inputs.macos-combine-only working-directory: ${{ github.workspace }} @@ -390,6 +398,13 @@ jobs: if ($arch -eq "arm64") { .\build_release_vs.bat slicer arm64 tests } else { .\build_release_vs.bat slicer tests } shell: pwsh + - name: Build system preset cache (Windows) + if: runner.os == 'Windows' + shell: cmd + # Shipped into both the already-installed tree (portable zip, MSIX) and + # the checkout cpack re-installs from when it builds the NSIS installer. + run: scripts\build_preset_cache.bat "%BUILD_DIR%" "resources\profiles" "%BUILD_DIR%\OrcaSlicer\resources\profiles" + - name: Pack unit tests Win if: runner.os == 'Windows' working-directory: ${{ github.workspace }} @@ -539,6 +554,20 @@ jobs: retention-days: 5 if-no-files-found: error + - name: Build system preset cache (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + # Both were packed from resources/ before the caches existed, so the + # AppImage is unpacked first and the caches shipped into it and into + # the package tree; the source tree keeps its JSONs for later steps. + appimage=$(find build -maxdepth 1 -name "OrcaSlicer_Linux_AppImage*.AppImage" | head -1) + chmod +x "$appimage" + "$appimage" --appimage-extract + ./scripts/build_preset_cache.sh -b build build/package/resources/profiles squashfs-root/resources/profiles + appimagetool=$(find build -name "appimagetool.AppImage" | head -1) + ARCH=$(uname -m) "$appimagetool" --appimage-extract-and-run squashfs-root "$appimage" + rm -rf squashfs-root # Ship the freshly-built validator so slice_check_linux (build_all.yml) # can slice-sweep the shipped profiles with this PR's engine. Taken from # the aarch64 leg so the sweep also exercises the arm build; x86_64 on diff --git a/.gitignore b/.gitignore index 916c7207b7..4d3ccb5c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ internal_docs/ # Python bytecode __pycache__/ *.pyc +*.opc diff --git a/build_linux.sh b/build_linux.sh index 72ea742f1a..6d65a10e41 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -567,6 +567,8 @@ if [[ -n "${BUILD_ORCA}" ]] || [[ -n "${BUILD_TESTS}" ]] ; then print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer echo "Building OrcaSlicer_profile_validator .." print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target OrcaSlicer_profile_validator + echo "Building generate_system_cache ..." + print_and_run cmake --build $BUILD_DIR --config "${BUILD_CONFIG}" --target generate_system_cache ./scripts/run_gettext.sh fi if [[ -n "${BUILD_TESTS}" ]] ; then diff --git a/docs/HLSD/preset-cache.md b/docs/HLSD/preset-cache.md new file mode 100644 index 0000000000..0923ce108f --- /dev/null +++ b/docs/HLSD/preset-cache.md @@ -0,0 +1,218 @@ +# System Preset Cache — High Level Design + +## Why it exists + +OrcaSlicer ships tens of thousands of system preset JSON files. Every launch used to +parse all of them: read each vendor profile, walk its machine, process and filament +sub-files, resolve inheritance, and build the preset collections from scratch. That +parse dominated startup, and it produced the same result every time, because system +presets only change when the app is updated or a profile update is installed. + +The preset cache replaces that parse with a read. Each vendor's fully-resolved presets +are serialized once — at build time, in CI — into a single binary file that the app +loads directly into memory. Nothing is recomputed at startup unless something changed. + +The cache is **only ever an optimization**. Every rule below exists to guarantee that a +cache is either provably equivalent to parsing the JSONs, or rejected. There is no +"mostly right" cache. + +## The unit is one vendor + +A cache covers exactly one vendor. `BBL.opc` sits beside `BBL.json` and holds +everything `BBL.json` and the `BBL/` sub-file tree would have produced. + +Per-vendor granularity is what makes the system practical: + +- A vendor whose profile is bumped invalidates only its own cache. The other 60-odd + vendors keep theirs. +- The setup wizard, which loads vendors one at a time, gets the same speedup as + startup without a second code path. +- A vendor with no cache, or a broken one, costs only that vendor a parse. + +A cache holds *system* presets only. User presets, project settings and modified +presets are never serialized — they have their own storage and their own lifecycle. + +## Where the files live + +| Location | Contents on a shipped build | Role | +|---|---|---| +| `resources/profiles/` | `.opc` alone — the profile and its preset JSONs both pruned | What the app ships with; the fallback everything falls back to | +| `/system/` | `.opc` alone, or `.json` + `/` after an update | What the user has installed | +| `/system/` (dev build) | `.json` + `/` + `.opc` written at runtime | A developer tree caches as it parses | + +Two forms of the same vendor therefore exist, and the system's central rule is that +**a vendor's cache is the whole of it**. Where a cache ships or is installed, no profile +and no preset JSONs sit beside it: the cache carries the presets, the vendor profile, +and the version stamp that says which release it came from. A vendor is "installed" if +either form is present, and its installed version is read from whichever form is there. + +What stays beside the caches in `resources/profiles/` is everything that is not a +preset: each vendor's directory of printer thumbnails, cover images, bed models and +hotend meshes, which are read from disk by path and were never part of the cache. Files +that are not vendors at all, `blacklist.json` chief among them, are untouched. + +The alternative — shipping both and treating the cache as a sidecar — was rejected. It +doubles the installed size, and it creates a class of bug where the two disagree and +the app's behavior depends on which one a given code path happened to read. + +## What a cache file is + +A fixed-size header followed by one binary stream. + +The header carries a magic number, the cache format version, the payload size and a +CRC32 of the payload. It exists so that a truncated download, a half-written file or a +file from an entirely different program is rejected in microseconds, before anything +tries to interpret it. + +The payload opens with the stamps that decide whether the cache may be used at all — +format version, schema fingerprint, vendor name, vendor version, filament library +version — and then the vendor's data: the vendor profiles, the five preset collections +(print, SLA print, filament, SLA material, printer), the config and filament-id lookup +maps, the obsolete-preset lists, and the count of errors the original parse hit. + +Two deliberate choices in the layout: + +- **Stamps come first**, so the question "what version is this vendor installed at?" + can be answered by reading the first kilobyte. The updater asks that question for + every vendor on every launch; reading tens of megabytes to answer it would give back + the startup time the cache saved. +- **Defaults are not stored.** Every collection reconstructs its default presets the + way the JSON path does, and the cache carries only what a parse would have added on + top. This keeps the cache a record of the vendor's data, not a memory image of the + program's state. + +## When a cache may be used + +A cache is accepted only if every gate below passes. Any failure means "parse the +JSONs instead" — never a hard error, never a partial load. + +**1. Integrity.** Magic number, plausible size, CRC32 over the payload. + +**2. Cache format version.** A single integer bumped by hand whenever the binary layout +changes in a way nothing else would catch: reordering or retyping a serialized field, +or changing what the cache's own stamps mean. + +**3. Schema fingerprint.** A checksum over the app version and the entire print-config +option schema — every option's key, type, wire ordinal and enum values. This is the +gate that makes the cache safe across development: adding a config option, changing its +type, or reordering the enum values of an existing one all change the fingerprint, so +caches from before the change are rejected without anyone having to remember to bump +anything. It also means a cache never crosses app versions. + +**4. Vendor identity and version.** The cache names the vendor it holds and the profile +version it was built from. It is accepted only if that version is at least as new as +the profile now on disk. Where no profile sits beside the cache — the shipped, +cache-only form — the comparison is skipped, because nothing on disk can be newer than +a cache that is the installation. + +**5. Filament library version.** Every vendor's filaments inherit from the shared Orca +filament library, so a vendor's cache is only valid against the library it was resolved +against. Bumping the library invalidates every vendor's cache, which is correct and +is why the library's version is stamped into all of them. + +A vendor profile with no parsable version is never cached and never served from a +cache. There would be no way to tell later whether the cache had gone stale, and a +cache nothing can invalidate is worse than no cache. + +## How a vendor is loaded + +When the app loads a vendor, it tries, in order: + +1. The cache in the directory it was asked to load from — normally `/system/`. +2. The shipped cache in `resources/profiles/`. +3. Parsing the JSONs — from the data directory if the profile is installed there, and + from `resources/profiles/` otherwise, which on a shipped build only has JSONs for a + vendor that has no cache. + +The second tier is what makes app upgrades work. After an upgrade, a cache the previous +version installed fails the fingerprint gate; the new build's own shipped cache answers +instead, and the user never sees a parse. The stale installed file is simply ignored +until the next profile update overwrites it. + +If a parse does happen and the vendor's profile carries a version, the app writes the +cache back beside where it looked for the vendor. That is how a developer build warms +itself up on second launch, and how a vendor delivered by a profile update becomes +cached without waiting for the next release. + +## How a vendor is installed + +Installing copies from `resources/profiles/` into `/system/`. A shipped build +offers only a cache and a source tree only JSONs, but a partially-generated tree can +have both, at different versions, so the installer picks the form that ships at the +**newer version** and installs only that one: + +- Cache newer or equal, and readable → copy the `.opc`, and delete any profile and + vendor directory a previous install left behind, so nothing can shadow it. +- Profile newer, or the cache unreadable or absent → copy the profile and the vendor's + preset JSONs exactly as the app did before caches existed, and delete any stale `.opc`. + +The result is that only one form of a vendor is ever present, and it is the newest one +the build has. This matters most for the update check, which compares what is installed +against what installing *would* lay down: if those two disagreed about which form +counts, a vendor could reinstall on every launch forever, or silently never update. + +Profile updates delivered over the air always arrive as JSONs, and they win — an +updated vendor's real profile lands in the data directory, the shipped cache is older +and gets rejected, and the vendor is parsed and re-cached. + +## How the caches are produced + +Cache generation is a build step, not something a user ever runs. + +One script per platform does the whole job, and CI calls it once on each. It builds a +small dev-utility that loads a profiles directory exactly as the app would, with cache +writing enabled, dropping a `.opc` beside every vendor profile it parses; then +it copies those caches into each packaged application it was pointed at and deletes +every preset JSON they replace — the vendor's own profile included. Only a vendor that +actually has a cache is pruned, so a vendor the generator skipped keeps its JSONs and is +simply parsed at startup. + +Because the schema fingerprint includes the app version, caches must be generated by +the same build that ships them. Generation runs after the build, in the same job. + +## Behavior when things go wrong + +The system is designed so that no cache problem is fatal: + +- **Corrupt, truncated or foreign file** — rejected at the header, vendor parsed. +- **Cache from another app version or schema** — rejected at the fingerprint, vendor + parsed or served from the shipped cache. +- **Stale cache** — rejected on the version stamps, vendor parsed and re-cached. +- **Failure part-way through reading** — the bundle is reset to a clean state before + falling back, so a half-loaded cache can never leak into the parsed result. +- **A vendor that can be neither read nor parsed** — logged, and left out. The setup + wizard drops that vendor from its list and opens with the rest; startup records the + error alongside the vendors that did load. One broken vendor never takes the app down. + +The one genuine limit: on a shipped build a vendor is its cache and nothing else, so a +rejected cache has nothing to fall back to for that vendor. This is by design — the +alternative is shipping every preset twice — and it is why the acceptance gates are +conservative and why CI generates the caches with the same build that ships them. The +recovery path is a profile update, which delivers real JSONs. + +It also means nothing may quietly assume a `.json` exists. Discovery, version +checks and the update decision all read whichever form is present, and a code path that +enumerates only `*.json` will find no vendors at all in a packaged build. + +## Maintenance rules + +- **Adding or changing a config option** needs nothing. The fingerprint covers it. +- **Changing what a cache serializes**, or the order it serializes it in, requires + bumping the cache format version by hand. +- **Bumping a vendor profile's version** invalidates that vendor's cache and nothing + else. Bumping the filament library invalidates all of them. +- **Caches are never committed.** They are build artifacts, generated per build, + ignored by git. + +## Where this lives in the tree + +| Area | Files | +|---|---| +| Cache format, read/write, load and save | `src/libslic3r/PresetBundle.{hpp,cpp}` | +| Per-preset serialization | `src/libslic3r/Preset.{hpp,cpp}` | +| Vendor discovery, installed/shipped versions, installation | `src/libslic3r/PresetBundle.cpp` | +| Update and reinstall decisions | `src/slic3r/Utils/PresetUpdater.cpp` | +| Setup wizard and printer-selection dialog | `src/slic3r/GUI/ConfigWizard.cpp`, `src/slic3r/GUI/WebGuideDialog.cpp` | +| Generator tool | `src/dev-utils/generate_system_cache.cpp` | +| Build and packaging script | `scripts/build_preset_cache.{sh,bat}` | +| Tests | `tests/libslic3r/test_vendor_cache.cpp` | diff --git a/scripts/build_preset_cache.bat b/scripts/build_preset_cache.bat new file mode 100644 index 0000000000..ac839b7c49 --- /dev/null +++ b/scripts/build_preset_cache.bat @@ -0,0 +1,122 @@ +@echo off +rem Build the per-vendor system preset caches (one .opc per vendor) by +rem running the generate_system_cache.exe dev tool against a profiles directory, +rem and make every profiles directory named on the command line ship-ready: +rem install the caches into it and delete the preset JSONs they replace, so a +rem build ships one copy of its presets instead of two. +rem +rem scripts\build_preset_cache.bat [build_dir] [target_dir ...] +rem +rem build_dir defaults to "build" +rem target_dir profiles directories to ship into. Caches are generated into +rem the source tree's resources\profiles, which is what every +rem packaging step copies from; a target may be that same +rem directory, which then only gets pruned. +rem +rem Shipping deletes, so it is a CI packaging step. A vendor's own .json +rem goes along with its preset JSONs: the cache carries the vendor profile and +rem the version it was built at, so discovery, version checks and installing all +rem read it there. Only a vendor that has a cache is pruned, so non-vendor JSONs +rem (blacklist.json) are left alone, as are the vendor directories themselves - +rem thumbnails, covers and bed models still live there. +rem +rem set CONFIG= to pin the build config for multi-config generators +rem (default: the config of the tool already in the build tree, else Release) +setlocal enabledelayedexpansion + +set "REPO_ROOT=%~dp0.." +set "BUILD_DIR=%~1" +if "%BUILD_DIR%"=="" set "BUILD_DIR=build" +if not exist "%BUILD_DIR%\" ( + echo ERROR: build tree not found: %BUILD_DIR% 1>&2 + exit /b 1 +) +if not "%~1"=="" shift + +rem Newest match wins: a stale binary silently produces a stale cache layout. +call :find_tool +if not defined CONFIG ( + for %%c in (Debug Release RelWithDebInfo MinSizeRel) do ( + echo !TOOL! | findstr /i "\\%%c\\" >nul && set "CONFIG=%%c" + ) +) +if not defined CONFIG set "CONFIG=Release" + +echo Building generate_system_cache in %BUILD_DIR% (%CONFIG%) +cmake --build "%BUILD_DIR%" --config %CONFIG% --target generate_system_cache +if errorlevel 1 ( + echo ERROR: could not build generate_system_cache - configure the build tree with -DORCA_TOOLS=ON: 1>&2 + echo cmake -S "%REPO_ROOT%" -B "%BUILD_DIR%" -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) +call :find_tool +if not defined TOOL ( + echo ERROR: generate_system_cache.exe not found under %BUILD_DIR% - build with -DORCA_TOOLS=ON 1>&2 + exit /b 1 +) + +set "PROFILES=%REPO_ROOT%\resources\profiles" +if not exist "%PROFILES%\" ( + echo ERROR: profiles directory not found: %PROFILES% 1>&2 + exit /b 1 +) +for %%d in ("%PROFILES%") do set "PROFILES=%%~fd" + +rem Add the slicer's runtime DLL directory to PATH so generate_system_cache.exe +rem can resolve its dependencies (TKernel.dll etc.) without a full install step. +set "DLL_DIR=" +for /f "delims=" %%f in ('dir /s /b "%BUILD_DIR%\TKernel.dll" 2^>nul') do ( + if not defined DLL_DIR set "DLL_DIR=%%~dpf" +) +if defined DLL_DIR set "PATH=%DLL_DIR%;%PATH%" + +echo Generating per-vendor preset caches in %PROFILES% +rem Start clean so vendors that went away - and caches written by older tool +rem versions - don't linger next to the freshly generated ones. +del /q "%PROFILES%\*.opc" 2>nul +del /q "%PROFILES%\*.cache" 2>nul +"%TOOL%" --path "%PROFILES%" --log_level 2 +if errorlevel 1 exit /b %errorlevel% + +:next_target +if "%~1"=="" exit /b 0 +call :ship "%~1" +if errorlevel 1 exit /b 1 +shift +goto :next_target + +:ship +set "TARGET=%~1" +if not exist "%TARGET%\" ( + echo ERROR: profiles directory not found: %TARGET% 1>&2 + exit /b 1 +) +for %%d in ("%TARGET%") do set "TARGET=%%~fd" +if /i not "%TARGET%"=="%PROFILES%" copy /y "%PROFILES%\*.opc" "%TARGET%\" >nul + +set /a SHIPPED=0 +set /a PRUNED=0 +for %%c in ("%PROFILES%\*.opc") do ( + set /a SHIPPED+=1 + set "VENDOR=%%~nc" + if exist "%TARGET%\!VENDOR!.json" ( + del /q "%TARGET%\!VENDOR!.json" + set /a PRUNED+=1 + ) + if exist "%TARGET%\!VENDOR!\" ( + for /f %%n in ('dir /s /b "%TARGET%\!VENDOR!\*.json" 2^>nul ^| find /c /v ""') do set /a PRUNED+=%%n + del /s /q "%TARGET%\!VENDOR!\*.json" >nul 2>&1 + rem Deepest first, so a directory the delete above emptied goes too; rd + rem refuses the ones still holding covers or meshes. + for /f "delims=" %%d in ('dir /s /b /ad "%TARGET%\!VENDOR!" 2^>nul ^| sort /r') do rd "%%d" 2>nul + ) +) +echo %TARGET%: !SHIPPED! caches, dropped !PRUNED! preset JSONs +exit /b 0 + +:find_tool +set "TOOL=" +for /f "delims=" %%f in ('dir /s /b /o-d "%BUILD_DIR%\generate_system_cache.exe" 2^>nul') do ( + if not defined TOOL set "TOOL=%%f" +) +exit /b 0 diff --git a/scripts/build_preset_cache.sh b/scripts/build_preset_cache.sh new file mode 100755 index 0000000000..33a436f151 --- /dev/null +++ b/scripts/build_preset_cache.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Build the per-vendor system preset caches (one .opc per vendor) by +# running the generate_system_cache dev tool against a profiles directory, and +# make every profiles directory named on the command line ship-ready: install +# the caches into it and delete the preset JSONs they replace, so a build ships +# one copy of its presets instead of two. +# +# ./scripts/build_preset_cache.sh # caches into resources/profiles +# ./scripts/build_preset_cache.sh -b build/arm64 # search this build tree for the tool +# ./scripts/build_preset_cache.sh [ ...] # and ship into these profiles dirs +# +# Caches are generated into the source tree's resources/profiles, which is what +# every packaging step copies from. Shipping deletes, so it is a CI packaging +# step: pass packaged output directories, or the checkout of a build that is +# about to be packaged from it. +# +# A vendor's own .json goes along with its preset JSONs: the cache +# carries the vendor profile and the version it was built at, so discovery, +# version checks and installing all read it there. A shipped vendor is its cache +# and nothing else. Only a vendor that has a cache is pruned, so an ungenerated +# vendor keeps its JSONs and is simply parsed at startup; non-vendor JSONs +# (blacklist.json) are left alone, as are the vendor directories themselves — +# thumbnails, covers and bed models still live there. +# +# -b build tree holding the tool +# (default: build/arm64, build/x86_64, or build — first that exists) +# -p profiles directory to generate caches into +# (default: /resources/profiles) +# -c build config for multi-config generators +# (default: the config of the tool already in the build tree, else +# the build tree's CMAKE_BUILD_TYPE) +# -n skip the rebuild and run the tool already in the build tree +# -l tool log level (default: 2) +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd -P)" +build_dir="" +profiles_dir="" +config="" +build_tool=1 +log_level=2 + +while getopts "b:p:c:l:nh" opt; do + case $opt in + b) build_dir="$OPTARG" ;; + p) profiles_dir="$OPTARG" ;; + c) config="$OPTARG" ;; + n) build_tool=0 ;; + l) log_level="$OPTARG" ;; + h) sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "$build_dir" ]; then + for candidate in "$repo_root/build/arm64" "$repo_root/build/x86_64" "$repo_root/build"; do + if [ -d "$candidate" ]; then build_dir="$candidate"; break; fi + done +fi +if [ -z "$build_dir" ] || [ ! -d "$build_dir" ]; then + echo "ERROR: build tree not found (pass -b )" >&2 + exit 1 +fi + +# Newest match wins: multi-config trees keep one binary per config, and a stale +# one silently produces a stale cache layout. +find_tool() { + local best="" f + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ -z "$best" ] || [ "$f" -nt "$best" ]; then best="$f"; fi + done < <(find "$build_dir" -name generate_system_cache -type f 2>/dev/null) + printf '%s' "$best" +} + +tool=$(find_tool) +if [ -z "$config" ]; then + case "$tool" in + */Debug/*) config=Debug ;; + */Release/*) config=Release ;; + */RelWithDebInfo/*) config=RelWithDebInfo ;; + */MinSizeRel/*) config=MinSizeRel ;; + *) config=$(sed -n 's/^CMAKE_BUILD_TYPE:[A-Z]*=\(.\+\)$/\1/p' "$build_dir/CMakeCache.txt" 2>/dev/null | head -1 || true) ;; + esac +fi + +if [ "$build_tool" = 1 ]; then + echo "Building generate_system_cache in $build_dir${config:+ ($config)}" + build_args=(--build "$build_dir" --target generate_system_cache) + if [ -n "$config" ]; then build_args+=(--config "$config"); fi + if ! cmake "${build_args[@]}"; then + echo "ERROR: could not build generate_system_cache — configure the build tree with -DORCA_TOOLS=ON:" >&2 + echo " cmake -S \"$repo_root\" -B \"$build_dir\" -DORCA_TOOLS=ON" >&2 + exit 1 + fi + tool=$(find_tool) +fi + +if [ -z "$tool" ]; then + echo "ERROR: generate_system_cache not found under $build_dir — build with -DORCA_TOOLS=ON" >&2 + exit 1 +fi + +if [ -z "$profiles_dir" ]; then profiles_dir="$repo_root/resources/profiles"; fi +if [ ! -d "$profiles_dir" ]; then + echo "ERROR: profiles directory not found: $profiles_dir" >&2 + exit 1 +fi +profiles_dir=$(cd "$profiles_dir" && pwd -P) + +# Start clean so vendors that went away — and caches written by older tool +# versions — don't linger next to the freshly generated ones. +echo "Generating per-vendor preset caches in $profiles_dir" +rm -f "$profiles_dir"/*.opc "$profiles_dir"/*.cache +"$tool" --path "$profiles_dir" --log_level "$log_level" + +for target in "$@"; do + resolved=$(cd "$target" 2>/dev/null && pwd -P) || { + echo "ERROR: profiles directory not found: $target" >&2 + exit 1 + } + if [ "$resolved" != "$profiles_dir" ]; then + cp "$profiles_dir"/*.opc "$resolved"/ + fi + + pruned=0 + shipped=0 + for cache in "$profiles_dir"/*.opc; do + vendor=$(basename "$cache" .opc) + shipped=$(( shipped + 1 )) + if [ -f "$resolved/$vendor.json" ]; then + rm -f "$resolved/$vendor.json" + pruned=$(( pruned + 1 )) + fi + [ -d "$resolved/$vendor" ] || continue + n=$(find "$resolved/$vendor" -name '*.json' | wc -l) + find "$resolved/$vendor" -name '*.json' -delete + find "$resolved/$vendor" -type d -empty -delete + pruned=$(( pruned + n )) + done + echo "$resolved: $shipped caches, dropped $pruned preset JSONs" +done diff --git a/src/dev-utils/CMakeLists.txt b/src/dev-utils/CMakeLists.txt index e3534a024a..2cfce6a7c5 100644 --- a/src/dev-utils/CMakeLists.txt +++ b/src/dev-utils/CMakeLists.txt @@ -20,6 +20,16 @@ if (SLIC3R_ENC_CHECK) ) endif() +if (ORCA_TOOLS) + set(_DEV_DEFS -DBOOST_ALL_NO_LIB -DBOOST_USE_WINAPI_VERSION=0x602 -DBOOST_SYSTEM_USE_UTF8) + + # generate_system_cache: pre-generates per-vendor .opc files under resources/profiles for CI bundling. + add_executable(generate_system_cache generate_system_cache.cpp) + target_link_libraries(generate_system_cache libslic3r boost_headeronly) + target_compile_definitions(generate_system_cache PRIVATE ${_DEV_DEFS}) + +endif() + # Function that adds source file encoding check to a target # using the above encoding-check binary diff --git a/src/dev-utils/generate_system_cache.cpp b/src/dev-utils/generate_system_cache.cpp new file mode 100644 index 0000000000..23bf66fbdc --- /dev/null +++ b/src/dev-utils/generate_system_cache.cpp @@ -0,0 +1,85 @@ +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/Utils.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace Slic3r; +namespace fs = boost::filesystem; +namespace po = boost::program_options; + +int main(int argc, char* argv[]) +{ + po::options_description desc("OrcaSlicer System Cache Generator\nUsage"); + // clang-format off + desc.add_options() + ("help,h", "Show help") +#ifdef __APPLE__ + ("path,p", po::value()->default_value("../../../../../../../resources/profiles"), "Path to profiles directory") +#else + ("path,p", po::value()->default_value("../../../resources/profiles"), "Path to profiles directory") +#endif + ("log_level,l", po::value()->default_value(2), "Log level (0=trace, 2=info, 4=error)"); + // clang-format on + + po::variables_map vm; + try { + po::store(po::parse_command_line(argc, argv, desc), vm); + if (vm.count("help")) { std::cout << desc << "\n"; return 0; } + po::notify(vm); + } catch (const po::error& e) { + std::cerr << "Error: " << e.what() << "\n" << desc << "\n"; + return 1; + } + + const std::string profiles_path = vm["path"].as(); + const int log_level = vm["log_level"].as(); + + if (!fs::exists(profiles_path) || !fs::is_directory(profiles_path)) { + std::cerr << "Error: '" << profiles_path << "' is not a valid directory\n"; + return 1; + } + + set_logging_level(log_level); + set_data_dir(profiles_path); + set_resources_dir(fs::path(profiles_path).parent_path().make_preferred().string()); + + const fs::path user_dir = fs::path(data_dir()) / PRESET_USER_DIR; + if (!fs::exists(user_dir)) + fs::create_directories(user_dir); + + AppConfig app_config; + app_config.set("preset_folder", "default"); + + auto preset_bundle = std::make_unique(); + preset_bundle->set_is_validation_mode(true); + preset_bundle->set_default_suppressed(true); + preset_bundle->set_generate_vendor_caches(true); + + std::cout << "Loading system presets from: " << profiles_path << "\n"; + + try { + // In validation mode data_dir() is the profiles directory set above, so the + // loader writes each .opc next to its .json as it parses it. + preset_bundle->load_presets(app_config, ForwardCompatibilitySubstitutionRule::EnableSilent); + } catch (const std::exception& ex) { + std::cerr << "Failed to load presets: " << ex.what() << "\n"; + return 1; + } + + size_t cache_count = 0; + for (auto& entry : fs::directory_iterator(profiles_path)) + if (boost::iends_with(entry.path().string(), ".opc")) + ++ cache_count; + if (cache_count == 0) { + std::cerr << "No vendor cache files were generated under " << profiles_path << "\n"; + return 1; + } + std::cout << "Generated " << cache_count << " vendor cache file(s) under " << profiles_path << "\n"; + return 0; +} diff --git a/src/libslic3r/Preset.cpp b/src/libslic3r/Preset.cpp index 775c31b563..4c50b09535 100644 --- a/src/libslic3r/Preset.cpp +++ b/src/libslic3r/Preset.cpp @@ -147,6 +147,17 @@ Semver get_version_from_json(std::string file_path) return Semver(); //throw ConfigurationError(format("Failed loading configuration file \"%1%\": %2%", file_path, err.what())); } + catch(...) { + return Semver(); + } +} + +std::string get_vendor_cache_version(const std::string& json_path) +{ + // The version a vendor's cache is stamped with. A profile without a parsable + // version cannot be judged for staleness, so it is simply never cached. + const Semver ver = get_version_from_json(json_path); + return ver.valid() ? ver.to_string() : std::string(); } //BBS: add a function to load the key-values from xxx.json diff --git a/src/libslic3r/Preset.hpp b/src/libslic3r/Preset.hpp index b88b5a5ed5..9955210e06 100644 --- a/src/libslic3r/Preset.hpp +++ b/src/libslic3r/Preset.hpp @@ -16,6 +16,14 @@ #include "Semver.hpp" #include "ProjectTask.hpp" +#include +#include +#include +#include +#include +#include +#include + //BBS: change system directories #define PRESET_SYSTEM_DIR "system" #define PRESET_USER_DIR "user" @@ -114,6 +122,10 @@ extern Semver get_version_from_json(std::string file_path); //BBS: add a function to load the key-values from xxx.json extern int get_values_from_json(std::string file_path, std::vector& keys, std::map& key_values); +// Returns the version a vendor JSON's preset cache is stamped with: its Semver +// string, or an empty string when the profile carries no usable version. +extern std::string get_vendor_cache_version(const std::string& json_path); + extern ConfigFileType guess_config_file_type(const boost::property_tree::ptree &tree); extern void extend_default_config_length(DynamicPrintConfig& config, const bool set_nil_to_default, const DynamicPrintConfig& defaults); @@ -131,6 +143,10 @@ public: PrinterVariant() {} PrinterVariant(const std::string &name) : name(name) {} std::string name; + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) { ar(name); } // PrinterVariant }; struct PrinterModel { @@ -139,7 +155,7 @@ public: std::string name; //BBS: this is internal id for the printer. Currently only used for searching in database std::string model_id; - PrinterTechnology technology; + PrinterTechnology technology = ptFFF; std::string family; std::vector variants; std::vector default_materials; @@ -162,6 +178,17 @@ public: } const PrinterVariant* variant(const std::string &name) const { return const_cast(this)->variant(name); } + + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // PrinterModel + { + ar(id, name, model_id, technology, family, variants, default_materials, + not_support_bed_types, bed_model, bed_texture, image_bed_type, + bottom_texture_end_name, use_double_extruder_default_texture, + bottom_texture_rect, bottom_texture_rect_longer, middle_texture_rect, + hotend_model); + } }; std::vector models; @@ -173,6 +200,14 @@ public: bool valid() const { return ! name.empty() && ! id.empty() && config_version.valid(); } + // All fields, declaration order — keep in sync; bump CACHE_VERSION on change. + template + void serialize(Archive& ar) // VendorProfile + { + ar(name, id, config_version, config_update_url, changelog_url, + models, default_filaments, default_sla_materials); + } + // Load VendorProfile from an ini file. // If `load_all` is false, only the header with basic info (name, version, URLs) is loaded. static VendorProfile from_ini(const boost::filesystem::path &path, bool load_all=true); @@ -425,12 +460,30 @@ public: // BBS: move constructor to public Preset(Type type, const std::string &name, bool is_default = false) : type(type), is_default(is_default), name(name) {} - -protected: + // Default constructor is public so cereal can default-construct elements when + // deserializing std::vector (std::allocator is not a cereal::access friend). Preset() = default; +protected: friend class PresetCollection; friend class PresetBundle; + friend class cereal::access; + + // Hand-written cereal serialization for the per-vendor binary cache. + // Lists every data member except the two raw pointers: + // - loading_substitutions: transient parse state, never cached + // - vendor: re-pointed on load from the vendor id stored alongside each preset + // Keep this list in sync with the member declarations, in declaration order; + // bump CACHE_VERSION in PresetBundle.cpp when it changes. + template + void serialize(Archive& ar) + { + ar(type, is_default, is_external, is_system, is_visible, is_dirty, + is_compatible, is_project_embedded, name, file, loaded, config, + alias, renamed_from, m_excluded_from, m_from_orca_filament_lib, + bundle_id, version, ini_str, setting_id, filament_id, user_id, + base_id, sync_info, description, updated_time, key_values); + } }; bool is_compatible_with_print (const PresetWithVendorProfile &preset, const PresetWithVendorProfile &active_print, const PresetWithVendorProfile &active_printer); diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index 01cbc43bc2..f093223842 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -1,7 +1,15 @@ #include +#include #include +#include #include "PresetBundle.hpp" + +#include +#include +#include +#include +#include #include "PrintConfig.hpp" #include "libslic3r.h" #include "I18N.hpp" @@ -564,6 +572,8 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, substitution_rule %1%, preferred printer_model_id %2%")%substitution_rule%preferred_selection.printer_model_id; + const auto startup_t0 = std::chrono::steady_clock::now(); + //BBS: change system config to json std::tie(substitutions, errors_cummulative) = this->load_system_presets_from_json(substitution_rule); @@ -589,6 +599,12 @@ PresetsConfigSubstitutions PresetBundle::load_presets(AppConfig &config, Forward set_calibrate_printer(""); + { + const auto total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - startup_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: all presets loaded in " << total_ms << " ms"; + } + //BBS: add config related logs BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" finished, returned substitutions %1%")%substitutions.size(); return substitutions; @@ -1001,6 +1017,8 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For bundles.m_bundles.clear(); bundles.WriteUnlock(); + const auto user_load_t0 = std::chrono::steady_clock::now(); + // Load bundle metadata from _local directory first fs::path local_dir(folder / PRESET_LOCAL_DIR); if (fs::exists(local_dir)) { @@ -1019,7 +1037,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.filament_presets.clear(); metadata.printer_presets.clear(); - // Add the profiles this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::LocalBundle, metadata.id)); @@ -1056,7 +1073,6 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For metadata.printer_presets.clear(); metadata.is_subscribed = true; - // Load presets from bundle (same logic as __local__) this->prints.load_presets(bundle_dir, PRESET_PRINT_NAME, substitutions, substitution_rule, [&](Preset& preset) { metadata.print_presets.push_back(preset.name); }, PresetOrigin(PresetOrigin::Kind::SubscribedBundle, metadata.id)); @@ -1077,34 +1093,41 @@ PresetsConfigSubstitutions PresetBundle::load_user_presets(std::string user, For } } - // BBS do not load sla_print - // BBS: change directoties by design - try { - std::string print_selected_preset_name = prints.get_selected_preset().name; - this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); - prints.select_preset_by_name(print_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + // BBS: change directories by design + + { + const auto json_t0 = std::chrono::steady_clock::now(); + try { + std::string sel = prints.get_selected_preset().name; + this->prints.load_presets(dir_user_presets, PRESET_PRINT_NAME, substitutions, substitution_rule); + prints.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = filaments.get_selected_preset().name; + this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); + filaments.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + try { + std::string sel = printers.get_selected_preset().name; + this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); + printers.select_preset_by_name(sel, false); + } catch (const std::runtime_error& err) { errors_cummulative += err.what(); } + if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + + const auto json_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - json_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user presets loaded from JSON in " << json_ms << " ms"; } - try { - std::string filament_selected_preset_name = filaments.get_selected_preset().name; - this->filaments.load_presets(dir_user_presets, PRESET_FILAMENT_NAME, substitutions, substitution_rule); - filaments.select_preset_by_name(filament_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); + + { + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - user_load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: user + bundle presets loaded in " << ms << " ms"; } - try { - std::string printer_selected_preset_name = printers.get_selected_preset().name; - this->printers.load_presets(dir_user_presets, PRESET_PRINTER_NAME, substitutions, substitution_rule); - printers.select_preset_by_name(printer_selected_preset_name, false); - } catch (const std::runtime_error &err) { - errors_cummulative += err.what(); - } - if (!errors_cummulative.empty()) throw Slic3r::RuntimeError(errors_cummulative); + this->update_multi_material_filament_presets(); this->update_compatible(PresetSelectCompatibleType::Never); - set_calibrate_printer(""); return PresetsConfigSubstitutions(); @@ -1210,13 +1233,10 @@ bool PresetBundle::apply_vendor_config( : std::map(); // Find vendors that need installation - const auto vendor_dir = (fs::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - std::vector install_bundles; for (const auto &it : new_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir / (it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -2225,6 +2245,166 @@ void PresetBundle::remove_users_preset(AppConfig &config, std::map vendor_names_in(const boost::filesystem::path& dir) +{ + std::set names; + for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { + const auto& path = dir_entry.path(); + if (Slic3r::is_json_file(path.string()) || path.extension() == ".opc") + names.insert(path.stem().string()); + } + return names; +} + +// A vendor's preset cache is the whole of its installation: it carries the presets, +// the vendor profile and the version they were built at, so where one ships nothing +// else needs copying. Unless the profile beside it claims a newer version — a cache +// generated before that profile was bumped is out of date, and a cache that cannot +// be read is no installation at all — and the vendor is installed the way it was +// before caches existed, as its profile and the preset JSONs it points at. Returns +// the version the cache is stamped with, or nothing when it is not the form to install. +static std::string installable_cache_version(const boost::filesystem::path& dir, const std::string& vendor) +{ + const auto cache_ver = Semver::parse(PresetBundle::peek_vendor_cache_version((dir / (vendor + ".opc")).string(), vendor)); + if (! cache_ver) + return {}; + const Semver profile_ver = get_version_from_json((dir / (vendor + ".json")).string()); + return profile_ver.valid() && *cache_ver < profile_ver ? std::string() : cache_ver->to_string(); +} + +Semver resource_vendor_version(const std::string& vendor) +{ + const boost::filesystem::path dir = boost::filesystem::path(resources_dir()) / "profiles"; + const auto ver = Semver::parse(installable_cache_version(dir, vendor)); + return ver ? *ver : get_version_from_json((dir / (vendor + ".json")).string()); +} + +bool install_vendor_bundles_from_resources( + const std::vector& bundle_names, + const std::string& resource_subdir, + const std::string& data_subdir) +{ + namespace fs = boost::filesystem; + + fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; + fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; + + BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; + + for (const auto &bundle : bundle_names) { + try { + // Install the JSON file + auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); + auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); + auto cache_in_rsrc = (rsrc_path / bundle).replace_extension(".opc"); + auto cache_in_vendors = (vendor_path / bundle).replace_extension(".opc"); + + // Either form of the vendor will do: a build may ship it as a cache alone. + if (!fs::exists(path_in_rsrc) && !fs::exists(cache_in_rsrc)) { + BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; + return false; + } + + // Create target directory if needed + if (!fs::exists(vendor_path)) + fs::create_directories(vendor_path); + + std::string error_message; + bool installed_cache = false; + if (! installable_cache_version(rsrc_path, bundle).empty()) { + installed_cache = copy_file(cache_in_rsrc.string(), cache_in_vendors.string(), error_message, false) == CopyFileResult::SUCCESS; + if (! installed_cache) + BOOST_LOG_TRIVIAL(warning) << "Failed to copy " << bundle << ".opc: " << error_message; + } else { + boost::system::error_code ec; + fs::remove(cache_in_vendors, ec); + } + + if (! installed_cache) { + CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); + if (cfr != CopyFileResult::SUCCESS) { + BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; + return false; + } + } else { + // Left in place, an earlier install's profile would shadow the cache. + boost::system::error_code ec; + fs::remove(path_in_vendors, ec); + } + + // Copy the vendor directory (if it exists) + auto dir_in_rsrc = rsrc_path / bundle; + auto dir_in_vendors = vendor_path / bundle; + + // Whatever is installed came from an earlier version of this vendor and + // would be parsed in place of the one being installed now. + if (fs::exists(dir_in_vendors)) + fs::remove_all(dir_in_vendors); + + if (! installed_cache && fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { + fs::create_directories(dir_in_vendors); + + // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) + // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf + auto file_filter = [](const std::string name) -> bool { + return boost::iends_with(name, ".stl") || + boost::iends_with(name, ".png") || + boost::iends_with(name, ".svg") || + boost::iends_with(name, ".jpeg") || + boost::iends_with(name, ".jpg") || + boost::iends_with(name, ".3mf"); + }; + + copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); + } + + BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; + + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); + return false; + } + } + + return true; +} + //BBS: add json related logic, load system presets from json std::pair PresetBundle::load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule) { @@ -2243,22 +2423,19 @@ std::pair PresetBundle::load_system_pre if (validation_mode) dir = (boost::filesystem::path(data_dir())).make_preferred(); + const auto load_t0 = std::chrono::steady_clock::now(); + + // The vendors below are loaded whole and against each other — the filament + // library first, then every other vendor with it as the base — so each parse + // is complete enough to be worth caching. + m_generate_vendor_caches = m_generate_vendor_caches || ! validation_mode; + PresetsConfigSubstitutions substitutions; std::string errors_cummulative; - bool first = true; - std::vector vendor_names; - // store all vendor names in vendor_names - for (auto& dir_entry : boost::filesystem::directory_iterator(dir)) { - std::string vendor_file = dir_entry.path().string(); - if (!Slic3r::is_json_file(vendor_file)) - continue; - - std::string vendor_name = dir_entry.path().filename().string(); - - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - vendor_names.push_back(vendor_name); - } + bool first = true; + // Sorted, so any duplicate-preset warning below comes out in the same order on + // every run. + const std::set vendor_names = vendor_names_in(dir); // Separate ORCA_FILAMENT_LIBRARY from other vendors. It must be loaded // first because other vendors' filaments may inherit from it via the // `base_bundle` lookup in parse_subfile. The remaining vendors are @@ -2274,8 +2451,17 @@ std::pair PresetBundle::load_system_pre } // Step 1: Load ORCA_FILAMENT_LIBRARY into `this` synchronously. - if (!orca_lib_vendor.empty()) { + if (! orca_lib_vendor.empty()) { try { + // Match a fresh launch before parsing: hold aliases and the error + // counter survive reset(), and would otherwise leak prior-cycle + // state into the library cache the load below writes. + this->prints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); + this->m_errors = 0; append(substitutions, this->load_vendor_configs_from_json(dir.string(), orca_lib_vendor, PresetBundle::LoadSystem, compatibility_rule).first); first = false; } catch (const std::runtime_error &err) { @@ -2298,10 +2484,10 @@ std::pair PresetBundle::load_system_pre for (size_t i = range.begin(); i < range.end(); ++i) { auto bundle = std::make_unique(); bundle->set_is_validation_mode(validation_mode); + bundle->set_generate_vendor_caches(m_generate_vendor_caches); try { auto result = bundle->load_vendor_configs_from_json( - dir.string(), other_vendors[i], PresetBundle::LoadSystem, - compatibility_rule, this); + dir.string(), other_vendors[i], PresetBundle::LoadSystem, compatibility_rule, this); parallel_substitutions[i] = std::move(result.first); parallel_bundles[i] = std::move(bundle); } catch (const std::runtime_error &err) { @@ -2346,6 +2532,11 @@ std::pair PresetBundle::load_system_pre } this->update_system_maps(); + + const auto load_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - load_t0).count(); + BOOST_LOG_TRIVIAL(info) << "PresetBundle: " << vendor_names.size() << " vendor(s) loaded in " << load_ms << " ms"; + //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(" finished, errors_cummulative %1%")%errors_cummulative; return std::make_pair(std::move(substitutions), errors_cummulative); @@ -4762,18 +4953,37 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool //BBS: Load a config bundle file from json std::pair PresetBundle::load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle) { // Enable substitutions for user config bundle, throw an exception when loading a system profile. ConfigSubstitutionContext substitution_context { compatibility_rule }; PresetsConfigSubstitutions substitutions; //BBS: add config related logs - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%path.c_str()%compatibility_rule; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(" enter, path %1%, compatibility_rule %2%")%dir.c_str()%compatibility_rule; if (flags.has(LoadConfigBundleAttribute::ResetUserProfile) || flags.has(LoadConfigBundleAttribute::LoadSystem)) // Reset this bundle, delete user profile files if SaveImported. this->reset(flags.has(LoadConfigBundleAttribute::SaveImported)); + // Orca: only a whole-vendor load has a cache — the vendor-only and filament-only + // scans want a slice of one. Validation reads the JSONs whatever is cached. + const boost::filesystem::path dir_path(dir); + const bool cacheable = flags.has(LoadConfigBundleAttribute::LoadSystem) && ! flags.has(LoadConfigBundleAttribute::LoadFilamentOnly); + if (cacheable && ! validation_mode && this->load_vendor_cache(dir_path, vendor_name)) { + size_t presets_loaded = 0; + for (const PresetCollection* coll : std::initializer_list{ + &this->prints, &this->sla_prints, &this->filaments, &this->sla_materials, &this->printers }) + presets_loaded += coll->m_presets.size() - coll->m_num_default_presets; + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << boost::format(", %1% served from its preset cache, %2% presets")%vendor_name%presets_loaded; + return std::make_pair(std::move(substitutions), presets_loaded); + } + + // Orca: a build that ships preset caches installs them without the preset + // JSONs, so a vendor left to be parsed is parsed from the profiles in + // resources. An update does deliver JSONs into `dir`, and those win. + const std::string path = (validation_mode || boost::filesystem::exists(dir_path / (vendor_name + ".json"))) + ? dir : (boost::filesystem::path(resources_dir()) / "profiles").string(); + // 1) load the vroot json and construct the vendor profile VendorProfile vendor_profile(vendor_name); std::string root_file = path + "/" + vendor_name + ".json"; @@ -5331,6 +5541,21 @@ std::pair PresetBundle::load_vendor_configs_ } } + // Orca: leave the vendor's cache in step with the profile just parsed, so the + // next run reads it instead. It is written where the vendor was looked for, + // even when the profile came from resources, and stamped with the version that + // profile claims — a profile without one cannot be judged for staleness later, + // and a cache nothing can invalidate is worse than none. + if (cacheable && m_generate_vendor_caches && vendor_profile.config_version.valid()) { + const std::string version = vendor_profile.config_version.to_string(); + // The library is its own reference point; every other vendor's cache holds + // filaments resolved against it, and is stamped with the version in effect. + const std::string lib_version = vendor_name == ORCA_FILAMENT_LIBRARY ? version : effective_lib_version(dir_path); + if (! lib_version.empty() && + ! this->save_vendor_cache((dir_path / (vendor_name + ".opc")).string(), vendor_name, version, lib_version)) + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache for " << vendor_name; + } + //BBS: add config related logs BOOST_LOG_TRIVIAL(debug) << __FUNCTION__ << boost::format(", finished, presets_loaded %1%")%presets_loaded; return std::make_pair(std::move(substitutions), presets_loaded); @@ -5953,4 +6178,312 @@ bool BundleMetadata::save_to_json(const std::string& path) const return false; } } +// ---- Preset cache file format (shared by the per-vendor cache) ---------- + +namespace { + +#pragma pack(push, 1) +struct CacheFileHeader { + uint32_t magic; + uint32_t version; + uint64_t data_size; + uint32_t crc32; +}; +#pragma pack(pop) +static_assert(sizeof(CacheFileHeader) == 20, "CacheFileHeader must be 20 bytes"); + +constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ" +// Bump when the wire format changes in a way the schema fingerprint cannot +// detect: reordering, removing, or retyping a field of a hand-written +// serialize() (Preset, VendorProfile and its nested types), or when the +// cache's own layout or the meaning of its stamps changes (e.g. the move from +// one whole-bundle cache to one cache per vendor). +constexpr uint32_t CACHE_VERSION = 4; + +// A cache stays usable as long as it was built from a vendor profile — and a +// filament library — at least as new as the ones now on disk. Profiles without +// a version cannot be judged this way and are never served from cache; where no +// profile sits beside the cache at all, nothing can be newer than it. +static bool cache_covers_version(const std::string& cached, const std::string& on_disk) +{ + if (on_disk == PresetBundle::CACHE_ANY_VERSION) + return true; + const auto cached_ver = Semver::parse(cached); + const auto on_disk_ver = Semver::parse(on_disk); + return cached_ver && on_disk_ver && *cached_ver >= *on_disk_ver; +} + +// Fingerprint of everything that determines the cache wire format: the app +// version and the DynamicPrintConfig option schema (key/type/ordinal/enum +// values — serialization_key_ordinal IS the config wire format). Any mismatch +// means bytes written by another build could deserialize into the wrong +// fields, so the cache is rejected wholesale before its body is read. +std::string compute_cache_schema_fingerprint() +{ + std::string schema; + schema += SLIC3R_VERSION; + schema += ';'; + for (const auto& [key, def] : print_config_def.options) { // std::map => stable order + schema += key; + schema += '#'; schema += std::to_string(int(def.type)); + schema += '@'; schema += std::to_string(def.serialization_key_ordinal); + for (const std::string& ev : def.enum_values) { schema += ','; schema += ev; } + schema += ';'; + } + boost::crc_32_type crc; + crc.process_bytes(schema.data(), schema.size()); + return std::to_string(crc.checksum()); +} + +} // anonymous namespace + +// static +bool PresetBundle::read_cache_blob(const std::string& path, std::string& out_blob) +{ + try { + boost::nowide::ifstream ifs(path, std::ios::binary); + if (!ifs.is_open()) + return false; + CacheFileHeader fhdr; + if (!ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr))) + return false; + if (fhdr.magic != CACHE_MAGIC) + return false; + if (fhdr.data_size == 0 || fhdr.data_size > 512u * 1024u * 1024u) + return false; + out_blob.assign(fhdr.data_size, '\0'); + if (!ifs.read(&out_blob[0], static_cast(fhdr.data_size))) + return false; + boost::crc_32_type crc; + crc.process_bytes(out_blob.data(), out_blob.size()); + if (crc.checksum() != fhdr.crc32) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: CRC mismatch: " << path; + return false; + } + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: read failed (" << path << "): " << e.what(); + return false; + } +} + +// static +void PresetBundle::write_cache_blob(const std::string& path, const std::string& blob) +{ + boost::crc_32_type crc; + crc.process_bytes(blob.data(), blob.size()); + try { + boost::filesystem::create_directories(boost::filesystem::path(path).parent_path()); + boost::nowide::ofstream ofs(path, std::ios::binary | std::ios::trunc); + if (!ofs.is_open()) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: cannot open for writing: " << path; + return; + } + CacheFileHeader fhdr; + fhdr.magic = CACHE_MAGIC; + fhdr.version = CACHE_VERSION; + fhdr.data_size = static_cast(blob.size()); + fhdr.crc32 = crc.checksum(); + ofs.write(reinterpret_cast(&fhdr), sizeof(fhdr)); + ofs.write(blob.data(), static_cast(blob.size())); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "SystemPresetsCache: write failed (" << path << "): " << e.what(); + } +} + +// ---- Per-vendor preset cache --------------------------------------------- + +// Serializes every non-default, non-external preset of one collection plus +// its m_printer_hold_alias. Defaults are constructor-derived state and are +// skipped; load re-installs the loaded presets after the constructor's +// defaults, exactly like the JSON parse path (reset-to-defaults + sorted +// append). +void PresetBundle::save_collection(cereal::BinaryOutputArchive& ar, const PresetCollection& coll) +{ + uint64_t count = 0; + for (const Preset& p : coll.m_presets) + if (! p.is_default && ! p.is_external) + ++ count; + ar(count); + for (const Preset& p : coll.m_presets) { + if (p.is_default || p.is_external) + continue; + std::string vendor_id = p.vendor ? p.vendor->id : std::string(); + ar(vendor_id, p); + } + // unordered containers -> sorted, for byte-deterministic output (parity test) + std::map> hold; + for (const auto& entry : coll.m_printer_hold_alias) + hold.emplace(entry.first, std::set(entry.second.begin(), entry.second.end())); + ar(hold); +} + +void PresetBundle::load_collection(cereal::BinaryInputArchive& ar, PresetCollection& coll, const VendorMap& vendors) +{ + coll.m_printer_hold_alias.clear(); + // Drop everything but the constructor-installed defaults, like PresetCollection::reset(). + coll.m_presets.erase(coll.m_presets.begin() + coll.m_num_default_presets, coll.m_presets.end()); + // Mirror PresetCollection::reset(), which follows the same truncation with + // select_preset(0): re-point selection/edited-preset state at the default + // preset before the loaded presets are appended below, so a cache hit onto + // an already-populated bundle (e.g. a reload) leaves selection state + // identical to the JSON-parse path instead of carrying over a stale index. + coll.select_preset(0); + uint64_t count = 0; + ar(count); + for (uint64_t i = 0; i < count; ++ i) { + std::string vendor_id; + Preset preset(coll.m_type, std::string()); + ar(vendor_id, preset); + if (! vendor_id.empty()) { + auto it = vendors.find(vendor_id); + if (it == vendors.end()) + throw std::runtime_error("vendor cache references unknown vendor: " + vendor_id); + preset.vendor = &it->second; + } + coll.m_presets.emplace_back(std::move(preset)); + } + std::map> hold; + ar(hold); + for (auto& entry : hold) + coll.m_printer_hold_alias.emplace(entry.first, std::unordered_set(entry.second.begin(), entry.second.end())); +} + +// ---- Per-vendor preset cache implementation ------------------------------ + +bool PresetBundle::save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const std::string& lib_version) const +{ + try { + std::ostringstream body(std::ios::binary); + { + cereal::BinaryOutputArchive ar(body); + ar(CACHE_VERSION); + ar(compute_cache_schema_fingerprint()); + ar(vendor_name, vendor_version, lib_version); + ar(this->vendors); + save_collection(ar, this->prints); + save_collection(ar, this->sla_prints); + save_collection(ar, this->filaments); + save_collection(ar, this->sla_materials); + save_collection(ar, this->printers); + ar(this->m_config_maps, this->m_filament_id_maps); + ar(this->obsolete_presets.prints, this->obsolete_presets.sla_prints, + this->obsolete_presets.filaments, this->obsolete_presets.sla_materials, + this->obsolete_presets.printers); + ar(this->m_errors); + } + write_cache_blob(cache_path, body.str()); + std::string verify; + return read_cache_blob(cache_path, verify); // magic + size + CRC + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: failed to save vendor cache " << cache_path << ": " << e.what(); + return false; + } +} + +// static +std::string PresetBundle::peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name) +{ + try { + boost::nowide::ifstream ifs(cache_path, std::ios::binary); + CacheFileHeader fhdr; + if (! ifs.read(reinterpret_cast(&fhdr), sizeof(fhdr)) || fhdr.magic != CACHE_MAGIC) + return {}; + // Only the head of the body is read, and its CRC left unverified: the stamps + // sit at the front, this answers "what version is this vendor at?" once per + // vendor on every update check, and reading tens of megabytes to do so is not + // worth it. A stamp that comes out garbled fails to parse as a version, which + // is the same answer as none. + std::string head(static_cast(std::min(fhdr.data_size, 1024)), '\0'); + if (! ifs.read(&head[0], static_cast(head.size()))) + return {}; + std::istringstream body(head, std::ios::binary); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + std::string fingerprint, vendor_name, vendor_version; + ar(fingerprint); + ar(vendor_name, vendor_version); + // The fingerprint is deliberately not checked: the version a cache carries + // is what this build installed, whether or not this build can still read it. + if (cache_version != CACHE_VERSION || vendor_name != expected_vendor_name) + return {}; + return vendor_version; + } catch (const std::exception&) { + return {}; + } +} + +bool PresetBundle::load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name) +{ + // Whichever cache answers is judged against the vendor as installed in `dir`: + // the profile there, or — with none, as when the cache is the whole of the + // installation — nothing, since nothing on disk can then be newer than it. + // Plus the filament library in effect, which a vendor's filaments were + // resolved against when its cache was built. + const boost::filesystem::path profile = dir / (vendor_name + ".json"); + const std::string version = boost::filesystem::exists(profile) ? get_vendor_cache_version(profile.string()) + : std::string(CACHE_ANY_VERSION); + const std::string lib_version = effective_lib_version(dir); + const boost::filesystem::path rsrc = boost::filesystem::path(resources_dir()) / "profiles"; + return this->load_vendor_cache((dir / (vendor_name + ".opc")).string(), vendor_name, version, lib_version) + || (dir != rsrc && this->load_vendor_cache((rsrc / (vendor_name + ".opc")).string(), vendor_name, version, lib_version)); +} + +bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const std::string& expected_vendor_version, const std::string& expected_lib_version) +{ + std::string blob; + if (! read_cache_blob(cache_path, blob)) + return false; + try { + std::istringstream body(blob, std::ios::binary); + cereal::BinaryInputArchive ar(body); + uint32_t cache_version = 0; + ar(cache_version); + if (cache_version != CACHE_VERSION) + return false; + std::string fingerprint; + ar(fingerprint); + if (fingerprint != compute_cache_schema_fingerprint()) + return false; + std::string vendor_name, vendor_version, lib_version; + ar(vendor_name, vendor_version, lib_version); + if (vendor_name != expected_vendor_name || + ! cache_covers_version(vendor_version, expected_vendor_version) || + ! cache_covers_version(lib_version, expected_lib_version)) + return false; + ar(this->vendors); + load_collection(ar, this->prints, this->vendors); + load_collection(ar, this->sla_prints, this->vendors); + load_collection(ar, this->filaments, this->vendors); + load_collection(ar, this->sla_materials, this->vendors); + load_collection(ar, this->printers, this->vendors); + ar(this->m_config_maps, this->m_filament_id_maps); + ar(this->obsolete_presets.prints, this->obsolete_presets.sla_prints, + this->obsolete_presets.filaments, this->obsolete_presets.sla_materials, + this->obsolete_presets.printers); + ar(this->m_errors); + return true; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "PresetBundle: rejecting vendor cache " << cache_path << ": " << e.what(); + // Restore a clean state so the caller can fall back to the JSON parse. + this->reset(false); + this->vendors.clear(); + this->m_config_maps.clear(); + this->m_filament_id_maps.clear(); + this->m_errors = 0; + // reset() does not clear m_printer_hold_alias, and a mid-body failure may + // have left collections it never reached (each load_collection clears its + // own collection's map only when it runs) with stale aliases. + this->prints.m_printer_hold_alias.clear(); + this->sla_prints.m_printer_hold_alias.clear(); + this->filaments.m_printer_hold_alias.clear(); + this->sla_materials.m_printer_hold_alias.clear(); + this->printers.m_printer_hold_alias.clear(); + return false; + } +} + } // namespace Slic3r diff --git a/src/libslic3r/PresetBundle.hpp b/src/libslic3r/PresetBundle.hpp index 685687975b..bde0bbbeba 100644 --- a/src/libslic3r/PresetBundle.hpp +++ b/src/libslic3r/PresetBundle.hpp @@ -6,6 +6,7 @@ #include "enum_bitmask.hpp" #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include + #define DEFAULT_USER_FOLDER_NAME "default" #define BUNDLE_STRUCTURE_JSON_NAME "bundle_structure.json" @@ -170,6 +172,44 @@ struct PresetBundleMetadata class PresetBundle { public: + // ---- Per-vendor preset cache -------------------------------------------- + // One cache file per vendor (plus the Orca filament library), stamped with + // the vendor's own profile version rather than a directory scan. + + // The cache is not something a caller loads from: a vendor is loaded with + // load_vendor_configs_from_json, which comes from the cache whenever one covers + // it. What is public here is what the cache's own tests drive directly. + + // Save this bundle's slice belonging to one vendor (vendor_name at + // vendor_version), built against the given filament library version. + bool save_vendor_cache(const std::string& cache_path, const std::string& vendor_name, + const std::string& vendor_version, const std::string& lib_version) const; + + // Expected version meaning "no profile sits beside this cache", so nothing can + // be newer than it and only its own integrity decides whether it is used. Not + // the same as an empty version, which means a profile is there but unreadable. + static constexpr const char* CACHE_ANY_VERSION = "*"; + + // Load a validated per-vendor cache into this bundle. Rejects (returns + // false) unless the cache version, schema fingerprint and vendor name match + // and the cache was built from a vendor profile and filament library at + // least as new as the expected ones. Profiles without a version (empty + // string) are never served from cache. + bool load_vendor_cache(const std::string& cache_path, const std::string& expected_vendor_name, + const std::string& expected_vendor_version, const std::string& expected_lib_version); + + // Read the profile version a cache was stamped with, without deserializing its + // presets. Empty if the file is unreadable, not a cache this build understands, + // or not this vendor's. This is how an installed vendor's version is known when + // only its cache is installed. + static std::string peek_vendor_cache_version(const std::string& cache_path, const std::string& expected_vendor_name); + + // Enable writing a per-vendor cache after a JSON parse (off by default). Only + // for bundles whose parses are complete — load_system_presets_from_json loads + // the filament library first and every other vendor against it, so its parses + // qualify; a wizard's one-off parse of a single vendor does not. + void set_generate_vendor_caches(bool enable) { m_generate_vendor_caches = enable; } + static DynamicPrintConfig construct_full_config(Preset &in_printer_preset, Preset &in_print_preset, const DynamicPrintConfig &project_config, @@ -444,8 +484,12 @@ public: /*std::pair load_configbundle( const std::string &path, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule);*/ //Orca: load config bundle from json, pass the base bundle to support cross vendor inheritance + // Orca: `dir` is where the vendor is looked for — its own directory, whether or + // not the profile JSONs are still there. A whole-vendor load comes from the + // vendor's preset cache whenever one covers the profile on disk, and is parsed + // from the JSONs (falling back to the ones in resources) only when none does. std::pair load_vendor_configs_from_json( - const std::string &path, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); + const std::string &dir, const std::string &vendor_name, LoadConfigBundleAttributes flags, ForwardCompatibilitySubstitutionRule compatibility_rule, const PresetBundle* base_bundle = nullptr); // Export a config bundle file containing all the presets and the names of the active presets. //void export_configbundle(const std::string &path, bool export_system_settings = false, bool export_physical_printers = false); @@ -521,7 +565,33 @@ public: // compatible_prints references a deleted (unknown) or renamed (old) preset name. bool check_preset_references() const; + // Merge one vendor's presets with the other vendor's presets, report duplicates. + // Public so per-vendor-cache consumers (e.g. the setup wizard) can assemble a + // bundle out of several per-vendor caches loaded into separate PresetBundle instances. + std::vector merge_presets(PresetBundle &&other); + private: + // Load one vendor from its preset cache: the one in `dir`, or — when that is + // missing or stale — the one shipped in resources/profiles, both judged against + // the vendor as installed in `dir` and against the filament library in effect. + // False, with this bundle left clean, when neither is usable and the vendor has + // to be parsed. This is how load_vendor_configs_from_json reads a cache. + bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name); + + // Read raw cache blob: verify magic, size, CRC. + static bool read_cache_blob(const std::string& path, std::string& out_blob); + // Write a cache blob with the standard 20-byte file header. + static void write_cache_blob(const std::string& path, const std::string& blob); + + // (De)serialization of one collection's slice for the per-vendor cache: + // every non-default, non-external preset (system or user) plus + // m_printer_hold_alias. See save_vendor_cache/load_vendor_cache. + static void save_collection(cereal::BinaryOutputArchive& ar, const PresetCollection& coll); + static void load_collection(cereal::BinaryInputArchive& ar, PresetCollection& coll, const VendorMap& vendors); + + // Whether to (re)write a per-vendor cache after a JSON parse. + bool m_generate_vendor_caches { false }; + // Orca: validation only - flag any printer with two or more compatible // filament presets sharing one filament_id (ambiguous AMS subtype match). bool check_duplicate_filament_subtypes() const; @@ -529,8 +599,6 @@ private: //std::pair load_system_presets(ForwardCompatibilitySubstitutionRule compatibility_rule); //BBS: add json related logic std::pair load_system_presets_from_json(ForwardCompatibilitySubstitutionRule compatibility_rule); - // Merge one vendor's presets with the other vendor's presets, report duplicates. - std::vector merge_presets(PresetBundle &&other); // Update the multicolor information for filaments. void update_filament_multi_color(); // Update renamed_from and alias maps of system profiles. @@ -565,6 +633,34 @@ private: ENABLE_ENUM_BITMASK_OPERATORS(PresetBundle::LoadConfigBundleAttribute) +// True if `vendor` is installed in data_dir()/system. A build that ships preset +// caches installs the cache alone, so it — not the profile — marks a vendor +// installed, and either one on its own counts. +extern bool is_vendor_installed(const std::string& vendor); + +// The version of the installed vendor: what its profile claims, or what its cache +// was stamped with where only the cache is installed. Invalid Semver if neither is. +extern Semver installed_vendor_version(const std::string& vendor); + +// The vendors `dir` holds, sorted: one is named by its profile or, in a build that +// ships preset caches instead of the raw profile JSONs, by its cache alone. +extern std::set vendor_names_in(const boost::filesystem::path& dir); + +// The version a build ships `vendor` at: whichever of its preset cache and its +// profile is newer, that being the one installing lays down. Invalid Semver if the +// build ships neither. +extern Semver resource_vendor_version(const std::string& vendor); + +// Install vendors from the resources directory into the data directory, each as +// its preset cache or as its profile and preset JSONs — whichever of the two the +// build ships at the newer version. Anything the previous install of that vendor +// left behind goes, so only the form just installed is there to be loaded. +// bundle_names: vendor names, without extension. +// Returns false on the first vendor that cannot be installed. +extern bool install_vendor_bundles_from_resources(const std::vector& bundle_names, + const std::string& resource_subdir = "profiles", + const std::string& data_subdir = "system"); + } // namespace Slic3r #endif /* slic3r_PresetBundle_hpp_ */ diff --git a/src/libslic3r/PrintConfig.hpp b/src/libslic3r/PrintConfig.hpp index 67841c6404..5f77baad86 100644 --- a/src/libslic3r/PrintConfig.hpp +++ b/src/libslic3r/PrintConfig.hpp @@ -2410,7 +2410,8 @@ namespace cereal { archive(serialization_key_ordinal); assert(serialization_key_ordinal > 0); auto it = Slic3r::print_config_def.by_serialization_key_ordinal.find(serialization_key_ordinal); - assert(it != Slic3r::print_config_def.by_serialization_key_ordinal.end()); + if (it == Slic3r::print_config_def.by_serialization_key_ordinal.end()) + throw std::runtime_error("VendorCache: unknown serialization_key_ordinal " + std::to_string(serialization_key_ordinal) + " - cache is stale"); config.set_key_value(it->second->opt_key, it->second->load_option_from_archive(archive)); } } diff --git a/src/libslic3r/Semver.hpp b/src/libslic3r/Semver.hpp index 4d64b1c7db..d3683b4eb8 100644 --- a/src/libslic3r/Semver.hpp +++ b/src/libslic3r/Semver.hpp @@ -190,6 +190,19 @@ public: os << self.to_string(); return os; } + + // cereal: round-trip through the standard 3-part string (major.minor.patch). + // to_string() uses a BBS 4-part format that semver_parse() cannot read back. + template + std::string save_minimal(const Archive&) const { return to_string_sf(); } + template + void load_minimal(const Archive&, const std::string& s) { + auto v = Semver::parse(s); + if (! v) + throw std::runtime_error("Semver: cannot parse serialized version: " + s); + *this = std::move(*v); + } + private: semver_t ver; diff --git a/src/libslic3r/Utils.hpp b/src/libslic3r/Utils.hpp index 62b2eeb78e..c52a120eff 100644 --- a/src/libslic3r/Utils.hpp +++ b/src/libslic3r/Utils.hpp @@ -722,15 +722,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, std::function filter = nullptr, bool merge_mode = false); -// Install vendor bundles from resources directory to data directory -// bundle_names: vector of vendor bundle names (without .json extension) -// resource_subdir: subdirectory under resources_dir() (default: "profiles") -// data_subdir: subdirectory under data_dir() (default: "system") -// Returns: true if all bundles installed successfully, false otherwise -bool install_vendor_bundles_from_resources(const std::vector& bundle_names, - const std::string& resource_subdir = "profiles", - const std::string& data_subdir = "system"); - // Orca: Since 1.7.9 Boost deprecated save_string_file and load_string_file, copy and modified from boost 1.7.8 void save_string_file(const boost::filesystem::path& p, const std::string& str); void load_string_file(const boost::filesystem::path& p, std::string& str); diff --git a/src/libslic3r/utils.cpp b/src/libslic3r/utils.cpp index 5f429f076a..4f4cc3450d 100644 --- a/src/libslic3r/utils.cpp +++ b/src/libslic3r/utils.cpp @@ -1724,76 +1724,6 @@ void copy_directory_recursively(const boost::filesystem::path& source, return; } -bool install_vendor_bundles_from_resources( - const std::vector& bundle_names, - const std::string& resource_subdir, - const std::string& data_subdir) -{ - namespace fs = boost::filesystem; - - fs::path rsrc_path = fs::path(Slic3r::resources_dir()) / resource_subdir; - fs::path vendor_path = fs::path(Slic3r::data_dir()) / data_subdir; - - BOOST_LOG_TRIVIAL(info) << "Installing " << bundle_names.size() << " bundles from resources..."; - - for (const auto &bundle : bundle_names) { - try { - // Install the JSON file - auto path_in_rsrc = (rsrc_path / bundle).replace_extension(".json"); - auto path_in_vendors = (vendor_path / bundle).replace_extension(".json"); - - if (!fs::exists(path_in_rsrc)) { - BOOST_LOG_TRIVIAL(warning) << "Bundle not found in resources: " << bundle; - return false; - } - - // Create target directory if needed - if (!fs::exists(vendor_path)) - fs::create_directories(vendor_path); - - // Copy JSON file - std::string error_message; - CopyFileResult cfr = copy_file(path_in_rsrc.string(), path_in_vendors.string(), error_message, false); - if (cfr != CopyFileResult::SUCCESS) { - BOOST_LOG_TRIVIAL(error) << "Failed to copy " << bundle << ".json: " << error_message; - return false; - } - - // Copy the vendor directory (if it exists) - auto dir_in_rsrc = rsrc_path / bundle; - auto dir_in_vendors = vendor_path / bundle; - - if (fs::exists(dir_in_rsrc) && fs::is_directory(dir_in_rsrc)) { - // Remove existing directory - if (fs::exists(dir_in_vendors)) - fs::remove_all(dir_in_vendors); - fs::create_directories(dir_in_vendors); - - // Copy with file filter (same as PresetUpdater::install_bundles_rsrc) - // Filter out certain file types: .stl, .png, .svg, .jpeg, .jpg, .3mf - auto file_filter = [](const std::string name) -> bool { - return boost::iends_with(name, ".stl") || - boost::iends_with(name, ".png") || - boost::iends_with(name, ".svg") || - boost::iends_with(name, ".jpeg") || - boost::iends_with(name, ".jpg") || - boost::iends_with(name, ".3mf"); - }; - - copy_directory_recursively(dir_in_rsrc, dir_in_vendors, file_filter); - } - - BOOST_LOG_TRIVIAL(info) << "Successfully installed bundle: " << bundle; - - } catch (const std::exception& e) { - BOOST_LOG_TRIVIAL(error) << "Exception installing bundle " << bundle << ": " << e.what(); - return false; - } - } - - return true; -} - void save_string_file(const boost::filesystem::path& p, const std::string& str) { boost::nowide::ofstream file; diff --git a/src/slic3r/Config/Snapshot.cpp b/src/slic3r/Config/Snapshot.cpp index 4b071994fc..a7135eac6f 100644 --- a/src/slic3r/Config/Snapshot.cpp +++ b/src/slic3r/Config/Snapshot.cpp @@ -432,14 +432,9 @@ const Snapshot& SnapshotDB::take_snapshot(const AppConfig &app_config, Snapshot: cfg.models_variants_installed.erase(it ++); else ++ it; - // Read the active config bundle, parse the config version. - PresetBundle bundle; - //BBS: change directoties by design - //bundle.load_configbundle((data_dir / PRESET_SYSTEM_DIR / (cfg.name + ".ini")).string(), PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - bundle.load_vendor_configs_from_json((data_dir/PRESET_SYSTEM_DIR).string(), cfg.name, PresetBundle::LoadConfigBundleAttribute::LoadVendorOnly, ForwardCompatibilitySubstitutionRule::EnableSilent); - for (const auto &vp : bundle.vendors) - if (vp.second.id == cfg.name) - cfg.version.config_version = vp.second.config_version; + // Orca: the version the vendor is installed at, read from its profile or — + // where the cache is the whole installation — from the cache's own stamp. + cfg.version.config_version = installed_vendor_version(cfg.name); snapshot.vendor_configs.emplace_back(std::move(cfg)); } diff --git a/src/slic3r/GUI/ConfigWizard.cpp b/src/slic3r/GUI/ConfigWizard.cpp index 0bbbc15f87..dba8699105 100644 --- a/src/slic3r/GUI/ConfigWizard.cpp +++ b/src/slic3r/GUI/ConfigWizard.cpp @@ -66,41 +66,41 @@ using Config::SnapshotDB; // Configuration data structures extensions needed for the wizard //BBS: set BBL as default -bool Bundle::load(fs::path source_path, bool ais_in_resources, bool ais_bbl_bundle) +bool Bundle::load(fs::path dir, const std::string &vendor_name, bool ais_in_resources, bool ais_bbl_bundle) { this->preset_bundle = std::make_unique(); this->is_in_resources = ais_in_resources; this->is_bbl_bundle = ais_bbl_bundle; - std::string path_string = source_path.string(); - std::string parent_path = source_path.parent_path().string(); //BBS: add json logic for vendor bundles - std::string vendor_name = source_path.filename().string(); - if (Slic3r::is_json_file(path_string)) { - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - } - else + // Orca: served from the vendor's preset cache where one covers it — which is + // how a shipped build carries its vendors — and parsed from the JSONs otherwise. + // A vendor that can be neither read nor parsed — a cache the build cannot use + // with the preset JSONs behind it pruned, say — is one the wizard cannot offer. + // Every other vendor still can be, so it is left out rather than thrown over. + size_t presets_loaded = 0; + try { + auto [config_substitutions, loaded] = preset_bundle->load_vendor_configs_from_json( + dir.string(), vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + UNUSED(config_substitutions); + // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. + assert(config_substitutions.empty()); + presets_loaded = loaded; + } catch (const std::exception &e) { + BOOST_LOG_TRIVIAL(fatal) << boost::format("Vendor bundle: `%1%`: cannot be loaded, leaving it out: %2%") % vendor_name % e.what(); return false; - - // Throw when parsing invalid configuration. Only valid configuration is supposed to be provided over the air. - //BBS: add json logic for vendor bundles - auto [config_substitutions, presets_loaded] = preset_bundle->load_vendor_configs_from_json( - parent_path, vendor_name, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); - UNUSED(config_substitutions); - // No substitutions shall be reported when loading a system config bundle, no substitutions are allowed. - assert(config_substitutions.empty()); + } auto first_vendor = preset_bundle->vendors.begin(); if (first_vendor == preset_bundle->vendors.end()) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No vendor information defined, cannot install.") % vendor_name; return false; } if (presets_loaded == 0) { - BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % path_string; + BOOST_LOG_TRIVIAL(error) << boost::format("Vendor bundle: `%1%`: No profile loaded.") % vendor_name; return false; - } + } - BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % path_string % presets_loaded; + BOOST_LOG_TRIVIAL(trace) << boost::format("Vendor bundle: `%1%`: %2% profiles loaded.") % vendor_name % presets_loaded; this->vendor_profile = &first_vendor->second; return true; } @@ -125,15 +125,10 @@ BundleMap BundleMap::load() //Orca: add custom as default //Orca: add json logic for vendor bundle - auto orca_bundle_path = (vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - auto orca_bundle_rsrc = false; - if (!boost::filesystem::exists(orca_bundle_path)) { - orca_bundle_path = (rsrc_vendor_dir / PresetBundle::ORCA_DEFAULT_BUNDLE).replace_extension(".json"); - orca_bundle_rsrc = true; - } { + const bool from_rsrc = ! is_vendor_installed(PresetBundle::ORCA_DEFAULT_BUNDLE); Bundle bbl_bundle; - if (bbl_bundle.load(std::move(orca_bundle_path), orca_bundle_rsrc, true)) + if (bbl_bundle.load(from_rsrc ? rsrc_vendor_dir : vendor_dir, PresetBundle::ORCA_DEFAULT_BUNDLE, from_rsrc, true)) res.emplace(PresetBundle::ORCA_DEFAULT_BUNDLE, std::move(bbl_bundle)); } @@ -141,18 +136,13 @@ BundleMap BundleMap::load() // and then additionally from resources/profiles. bool is_in_resources = false; for (auto dir : { &vendor_dir, &rsrc_vendor_dir }) { - for (const auto &dir_entry : boost::filesystem::directory_iterator(*dir)) { - //BBS: add json logic for vendor bundle - if (Slic3r::is_json_file(dir_entry.path().string())) { - std::string id = dir_entry.path().stem().string(); // stem() = filename() without the trailing ".json" part + for (const std::string &id : vendor_names_in(*dir)) { + // Don't load this bundle if we've already loaded it. + if (res.find(id) != res.end()) { continue; } - // Don't load this bundle if we've already loaded it. - if (res.find(id) != res.end()) { continue; } - - Bundle bundle; - if (bundle.load(dir_entry.path(), is_in_resources)) - res.emplace(std::move(id), std::move(bundle)); - } + Bundle bundle; + if (bundle.load(*dir, id, is_in_resources)) + res.emplace(id, std::move(bundle)); } is_in_resources = true; diff --git a/src/slic3r/GUI/ConfigWizard_private.hpp b/src/slic3r/GUI/ConfigWizard_private.hpp index 364d378b42..7b9674b216 100644 --- a/src/slic3r/GUI/ConfigWizard_private.hpp +++ b/src/slic3r/GUI/ConfigWizard_private.hpp @@ -71,9 +71,11 @@ struct Bundle Bundle() = default; Bundle(Bundle&& other); + // Load the vendor `vendor_name` as it is installed in `dir`, from its preset + // cache or its profile JSONs, whichever is usable. // Returns false if not loaded. Reason for that is logged as boost::log error. //BBS: set BBL as default - bool load(fs::path source_path, bool is_in_resources, bool is_bbl_bundle = false); + bool load(fs::path dir, const std::string &vendor_name, bool is_in_resources, bool is_bbl_bundle = false); const std::string& vendor_id() const { return vendor_profile->id; } }; diff --git a/src/slic3r/GUI/CreatePresetsDialog.cpp b/src/slic3r/GUI/CreatePresetsDialog.cpp index 1bd80d5f00..33f49c2a38 100644 --- a/src/slic3r/GUI/CreatePresetsDialog.cpp +++ b/src/slic3r/GUI/CreatePresetsDialog.cpp @@ -2201,25 +2201,14 @@ bool CreatePrinterPresetDialog::load_system_and_user_presets_with_curr_model(Pre } else { selected_vendor_id = m_printer_preset_vendor_selected.id; - if (boost::filesystem::exists(boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(); - } else if (boost::filesystem::exists(boost::filesystem::path(Slic3r::resources_dir()) / "profiles" / selected_vendor_id)) { - preset_path = (boost::filesystem::path(Slic3r::resources_dir()) / "profiles").string(); - } - - if (preset_path.empty()) { - BOOST_LOG_TRIVIAL(info) << "Preset path was not found"; - MessageDialog dlg(this, _L("Preset path was not found; please reselect vendor."), wxString(SLIC3R_APP_FULL_NAME) + " - " + _L("Info"), - wxYES_NO | wxYES_DEFAULT | wxCENTRE); - dlg.ShowModal(); - return false; - } - try { // Pass the app's preset bundle (which already holds OrcaFilamentLibrary) as the base // bundle so vendor filaments that inherit OFL bases resolve via the existing // cross-vendor inheritance path. - temp_preset_bundle.load_vendor_configs_from_json(preset_path, selected_vendor_id, + // Orca: served from the vendor's preset cache where one covers it — a shipped + // build carries that instead of the raw preset JSONs — and parsed otherwise. + temp_preset_bundle.load_vendor_configs_from_json((boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).string(), + selected_vendor_id, PresetBundle::LoadConfigBundleAttribute::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent, wxGetApp().preset_bundle); diff --git a/src/slic3r/GUI/WebGuideDialog.cpp b/src/slic3r/GUI/WebGuideDialog.cpp index 0d2f6c724b..a076fb635e 100644 --- a/src/slic3r/GUI/WebGuideDialog.cpp +++ b/src/slic3r/GUI/WebGuideDialog.cpp @@ -2,6 +2,7 @@ #include "ConfigWizard.hpp" #include +#include #include #include #include @@ -9,6 +10,7 @@ #include "I18N.hpp" #include "libslic3r/AppConfig.hpp" #include "libslic3r/Config.hpp" +#include "libslic3r/Preset.hpp" #include "libslic3r/PresetBundle.hpp" #include "slic3r/GUI/wxExtensions.hpp" #include "slic3r/GUI/GUI_App.hpp" @@ -41,8 +43,6 @@ using namespace nlohmann; namespace Slic3r { namespace GUI { -json m_ProfileJson; - static wxString update_custom_filaments() { json m_Res = json::object(); @@ -191,11 +191,10 @@ GuideFrame::GuideFrame(GUI_App *pGUI, long style) GuideFrame::~GuideFrame() { m_destroy = true; - if (m_load_task && m_load_task->joinable()) { + *m_cancel_token = true; // signal any queued CallAfter lambdas before join + if (m_load_task && m_load_task->joinable()) m_load_task->join(); - delete m_load_task; - m_load_task = nullptr; - } + m_load_task.reset(); if (m_browser) { delete m_browser; m_browser = nullptr; @@ -301,15 +300,65 @@ void GuideFrame::OnNavigationRequest(wxWebViewEvent &evt) /** * Callback invoked when a navigation request was accepted */ +void GuideFrame::init_guide_paths() +{ + m_ProfileJson = json::parse("{}"); + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + + vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); + rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); + orca_bundle_rsrc = true; + + if (boost::filesystem::exists(vendor_dir)) { + for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { + if (!boost::filesystem::is_directory(entry) && + boost::iequals(entry.path().extension().string(), ".json") && + !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { + orca_bundle_rsrc = false; + break; + } + } + } + + auto lib_json = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + m_OrcaFilaLibPath = boost::filesystem::exists(vendor_dir / lib_json) + ? (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string() + : (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); +} + +void GuideFrame::on_profile_loaded() +{ + // Must be called on the main thread. + SaveProfileData(); + const std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); + BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents:\n" << strAll; + json res; + res["command"] = "userguide_profile_load_finish"; + res["sequence_id"] = "10001"; + RunScript(wxString::Format("HandleStudio(%s)", res.dump(-1, ' ', true))); +} + void GuideFrame::OnNavigationComplete(wxWebViewEvent &evt) { //wxLogMessage("%s", "Navigation complete; url='" + evt.GetURL() + "'"); if (!bFirstComplete) { - m_load_task = new boost::thread(boost::bind(&GuideFrame::LoadProfileData, this)); - // boost::thread LoadProfileThread(boost::bind(&GuideFrame::LoadProfileData, this)); - //LoadProfileThread.detach(); - bFirstComplete = true; + try { + init_guide_paths(); + if (BuildProfileDataFromPresetBundle()) { + if (!m_destroy) + on_profile_loaded(); + } else { + // Presets not yet in memory — delegate to background thread. + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", init error: " << e.what(); + m_load_task = std::make_unique(boost::bind(&GuideFrame::LoadProfileData, this)); + } } m_browser->Show(); @@ -762,11 +811,9 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle bool check_unsaved_preset_changes = false; std::vector install_bundles; std::vector remove_bundles; - const auto vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); for (const auto &it : enabled_vendors) { if (it.second.size() > 0) { - auto vendor_file = vendor_dir/(it.first + ".json"); - if (!fs::exists(vendor_file)) { + if (!is_vendor_installed(it.first)) { install_bundles.emplace_back(it.first); } } @@ -777,8 +824,7 @@ bool GuideFrame::apply_config(AppConfig *app_config, PresetBundle *preset_bundle if (it.second.size() > 0) { if (enabled_vendors.find(it.first) != enabled_vendors.end()) continue; - auto vendor_file = vendor_dir/(it.first + ".json"); - if (fs::exists(vendor_file)) { + if (is_vendor_installed(it.first)) { remove_bundles.emplace_back(it.first); } } @@ -1127,99 +1173,256 @@ int GuideFrame::GetFilamentInfo( std::string VendorDirectory, json & pFilaList, return status; } -int GuideFrame::LoadProfileData() +bool GuideFrame::BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors) { try { - m_ProfileJson = json::parse("{}"); + // Models from vendor profiles + for (const auto& [vendor_id, vp] : bundle.vendors) { + for (const auto& model : vp.models) { + std::string nozzle_str; + for (const auto& v : model.variants) { + if (!nozzle_str.empty()) nozzle_str += ";"; + nozzle_str += v.name; + } + std::string materials_str; + for (const auto& m : model.default_materials) { + if (!materials_str.empty()) materials_str += ";"; + materials_str += m; + } + boost::filesystem::path cover_path = + (boost::filesystem::path(resources_dir()) / "profiles" / vp.id / (model.id + "_cover.png")) + .make_preferred(); + if (!boost::filesystem::exists(cover_path)) + cover_path = + (boost::filesystem::path(resources_dir()) / "web/image/printer" / (model.id + "_cover.png")) + .make_preferred(); + + json entry; + entry["model"] = model.id; + entry["name"] = model.name; + entry["vendor"] = vp.id; + entry["nozzle_diameter"] = nozzle_str; + entry["materials"] = materials_str; + entry["cover"] = cover_path.string(); + entry["nozzle_selected"] = ""; + entry["sub_path"] = ""; + m_ProfileJson["model"].push_back(entry); + } + } + + // Machine map: preset name -> {model, nozzle variant} + for (const Preset& p : bundle.printers()) { + if (!p.is_system || !p.vendor) continue; + const auto* printer_model = p.config.option("printer_model"); + const auto* printer_variant = p.config.option("printer_variant"); + if (!printer_model || printer_model->value.empty() || !printer_variant) continue; + + json mach; + mach["model"] = printer_model->value; + mach["nozzle"] = printer_variant->value; + m_ProfileJson["machine"][p.name] = mach; + } + + // Filament map from system filament presets (vendor/type already resolved in config) + for (const Preset& p : bundle.filaments()) { + if (!p.is_system || !p.vendor) continue; + const auto* fila_vendor = p.config.option("filament_vendor"); + const auto* fila_type = p.config.option("filament_type"); + const auto* compat_printers = p.config.option("compatible_printers"); + + std::string vendor = (fila_vendor && !fila_vendor->values.empty()) ? fila_vendor->values[0] : ""; + std::string type = (fila_type && !fila_type->values.empty()) ? fila_type->values[0] : ""; + + std::string model_list; + if (compat_printers) { + for (const std::string& pname : compat_printers->values) { + if (m_ProfileJson["machine"].contains(pname)) { + std::string m = m_ProfileJson["machine"][pname]["model"]; + std::string n = m_ProfileJson["machine"][pname]["nozzle"]; + model_list += "[" + m + "++" + n + "]"; + } + } + } + + json ff; + ff["name"] = p.name; + ff["sub_path"] = p.file; + ff["vendor"] = vendor; + ff["type"] = type; + ff["models"] = model_list; + ff["selected"] = 0; + m_ProfileJson["filament"][p.name] = ff; + } + + // Process list from visible system print presets + for (const Preset& p : bundle.prints()) { + if (!p.is_system || !p.vendor || !p.is_visible) continue; + json entry; + entry["name"] = p.name; + entry["sub_path"] = p.file; + m_ProfileJson["process"].push_back(entry); + } + + if (require_all_resource_vendors) { + // If rsrc_vendor_dir has vendor JSONs not covered by the current bundle, the + // bundle is incomplete (e.g. dev env where data_dir/system only has + // OrcaFilamentLibrary+Custom). Fall back so LoadProfileFamily reads both dirs. + try { + for (const auto& e : boost::filesystem::directory_iterator(rsrc_vendor_dir)) { + if (e.path().extension().string() != ".json") continue; + const std::string stem = e.path().stem().string(); + if (bundle.vendors.find(stem) == bundle.vendors.end()) { + BOOST_LOG_TRIVIAL(info) << "GuideFrame: vendor '" << stem + << "' in resources but not in preset_bundle — falling back to JSON loading"; + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + return false; + } + } + } catch (const std::exception&) {} + } + + BOOST_LOG_TRIVIAL(info) << "GuideFrame: built profile data (" + << m_ProfileJson["model"].size() << " models, " + << m_ProfileJson["machine"].size() << " machines, " + << m_ProfileJson["filament"].size() << " filaments)"; + return !m_ProfileJson["machine"].empty(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "GuideFrame::BuildProfileJson failed: " << e.what() + << " — falling back to JSON loading"; m_ProfileJson["model"] = json::array(); m_ProfileJson["machine"] = json::object(); m_ProfileJson["filament"] = json::object(); m_ProfileJson["process"] = json::array(); + return false; + } +} - vendor_dir = (boost::filesystem::path(Slic3r::data_dir()) / PRESET_SYSTEM_DIR).make_preferred(); - rsrc_vendor_dir = (boost::filesystem::path(resources_dir()) / "profiles").make_preferred(); +bool GuideFrame::BuildProfileDataFromPresetBundle() +{ + PresetBundle* pb = wxGetApp().preset_bundle; + if (!pb || pb->vendors.empty()) + return false; + return BuildProfileJson(*pb, /*require_all_resource_vendors=*/true); +} - // Orca: add custom as default - // Orca: add json logic for vendor bundle - orca_bundle_rsrc = true; +bool GuideFrame::BuildProfileDataFromVendors() +{ + try { + // Same vendor set and precedence as the JSON scan in LoadProfileData: a + // vendor in the user's system dir shadows the bundled one of that name. + // A vendor is named by its profile or, where a build ships preset caches + // instead, by its cache alone — so both forms name one here. + std::map vendor_files; + auto collect = [&vendor_files](const boost::filesystem::path& dir) { + boost::system::error_code ec; + if (!boost::filesystem::exists(dir, ec)) + return; + for (const auto& e : boost::filesystem::directory_iterator(dir, ec)) + if (Slic3r::is_json_file(e.path().string()) || e.path().extension() == ".opc") + vendor_files.emplace(e.path().stem().string(), e.path()); // first wins + }; + collect(vendor_dir); + collect(rsrc_vendor_dir); - // search if there exists a .json file in vendor_dir folder, if exists, set orca_bundle_rsrc to false - for (const auto& entry : boost::filesystem::directory_iterator(vendor_dir)) { - if (!boost::filesystem::is_directory(entry) && boost::iequals(entry.path().extension().string(), ".json") && !boost::iequals(entry.path().stem().string(), PresetBundle::ORCA_FILAMENT_LIBRARY)) { - orca_bundle_rsrc = false; - break; - } - } - - // load the default filament library first - std::set loaded_vendors; - auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); - if (boost::filesystem::exists(vendor_dir / filament_library_name)) { - m_OrcaFilaLibPath = (vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); - } else { - m_OrcaFilaLibPath = (rsrc_vendor_dir / PresetBundle::ORCA_FILAMENT_LIBRARY).string(); - LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); - } - loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - - //load custom bundle from user data path - boost::filesystem::directory_iterator endIter; - for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if(strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } + // Each vendor comes from its preset cache where one covers it, which is what + // makes this worth doing instead of the scan below; the filament library goes + // first because the others' filaments inherit from it, and resolving those on + // the vendors the cache does not cover needs it already loaded. + PresetBundle bundle; + auto load_vendor = [this](PresetBundle& into, const std::string& vendor, const PresetBundle* base) { + into.load_vendor_configs_from_json(vendor_dir.string(), vendor, PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent, base); + }; + const std::string filament_library(PresetBundle::ORCA_FILAMENT_LIBRARY); + if (vendor_files.count(filament_library)) + load_vendor(bundle, filament_library, nullptr); + for (const auto& entry : vendor_files) { if (m_destroy) - return 0; + return false; // as in the scan below: a vendor without a cache is parsed, and that takes time + const std::string& vendor = entry.first; + // A cache is only ever written for a versioned vendor; a JSON has to be + // asked, so that an unversioned one (blacklist.json) carrying no presets + // is passed over. + if (vendor == filament_library || + (entry.second.extension() != ".opc" && get_vendor_cache_version(entry.second.string()).empty())) + continue; + PresetBundle tmp; + load_vendor(tmp, vendor, &bundle); + bundle.merge_presets(std::move(tmp)); + } + if (bundle.vendors.empty()) + return false; + return BuildProfileJson(bundle, /*require_all_resource_vendors=*/false); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << " failed: " << e.what(); + m_ProfileJson["model"] = json::array(); + m_ProfileJson["machine"] = json::object(); + m_ProfileJson["filament"] = json::object(); + m_ProfileJson["process"] = json::array(); + return false; + } +} + +int GuideFrame::LoadProfileData() +{ + // Background thread: the fast path in OnNavigationComplete failed (presets not yet loaded). + // Loading order (fastest to slowest): + // 1. Load every vendor, from its preset cache wherever one covers it + // 2. Read all vendor JSONs by hand + try { + if (!BuildProfileDataFromVendors()) { + // Last resort — read all vendor JSONs + std::set loaded_vendors; + auto filament_library_name = boost::filesystem::path(PresetBundle::ORCA_FILAMENT_LIBRARY).replace_extension(".json"); + if (boost::filesystem::exists(vendor_dir / filament_library_name)) + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (vendor_dir / filament_library_name).string()); + else + LoadProfileFamily(PresetBundle::ORCA_FILAMENT_LIBRARY, (rsrc_vendor_dir / filament_library_name).string()); + loaded_vendors.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); + + boost::filesystem::directory_iterator endIter; + for (boost::filesystem::directory_iterator iter(vendor_dir); iter != endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (m_destroy) return 0; + } + + boost::filesystem::directory_iterator others_endIter; + for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { + if (!boost::filesystem::is_directory(*iter)) { + wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); + strVendor = strVendor.AfterLast('\\'); + strVendor = strVendor.AfterLast('/'); + wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); + if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) + continue; + LoadProfileFamily(w2s(strVendor), iter->path().string()); + loaded_vendors.insert(w2s(strVendor)); + } + if (m_destroy) return 0; + } } - boost::filesystem::directory_iterator others_endIter; - for (boost::filesystem::directory_iterator iter(rsrc_vendor_dir); iter != others_endIter; iter++) { - if (!boost::filesystem::is_directory(*iter)) { - wxString strVendor = from_u8(iter->path().string()).BeforeLast('.'); - strVendor = strVendor.AfterLast('\\'); - strVendor = strVendor.AfterLast('/'); - wxString strExtension = from_u8(iter->path().string()).AfterLast('.').Lower(); - if (strExtension.CmpNoCase("json") != 0 || loaded_vendors.find(w2s(strVendor)) != loaded_vendors.end()) - continue; - - LoadProfileFamily(w2s(strVendor), iter->path().string()); - loaded_vendors.insert(w2s(strVendor)); - } - if (m_destroy) - return 0; - } - - wxGetApp().CallAfter([this] { - if (!m_destroy) { - //sync to appconfig first to populate current selections - SaveProfileData(); - - //sync to web after selections are populated - std::string strAll = m_ProfileJson.dump(-1, ' ', false, json::error_handler_t::ignore); - - BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ", finished, json contents: " << std::endl << strAll; - json m_Res = json::object(); - m_Res["command"] = "userguide_profile_load_finish"; - m_Res["sequence_id"] = "10001"; - wxString strJS = wxString::Format("HandleStudio(%s)", m_Res.dump(-1, ' ', true)); - - RunScript(strJS); - } + // Capture the cancel token by value (shared_ptr) so the lambda doesn't + // touch `this` if GuideFrame is destroyed before the event fires. + auto tok = m_cancel_token; + wxGetApp().CallAfter([this, tok] { + if (!*tok) + on_profile_loaded(); }); - } catch (std::exception& e) { - // wxLogMessage("GUIDE: load_profile_error %s ", e.what()); - // wxMessageBox(e.what(), "", MB_OK); - BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what() << std::endl; + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ", error: " << e.what(); } filament_info_cache.clear(); diff --git a/src/slic3r/GUI/WebGuideDialog.hpp b/src/slic3r/GUI/WebGuideDialog.hpp index fcdb0841db..71e2409ae5 100644 --- a/src/slic3r/GUI/WebGuideDialog.hpp +++ b/src/slic3r/GUI/WebGuideDialog.hpp @@ -30,10 +30,14 @@ #include "libslic3r/PresetBundle.hpp" #include "slic3r/Utils/PresetUpdater.hpp" +#include +#include #include #include +#include + namespace Slic3r { namespace GUI { class GuideFrame : public DPIDialog @@ -78,6 +82,11 @@ public: int LoadProfileData(); int SaveProfileData(); int LoadProfileFamily(std::string strVendor, std::string strFilePath); + void init_guide_paths(); + void on_profile_loaded(); + bool BuildProfileJson(const PresetBundle& bundle, bool require_all_resource_vendors); + bool BuildProfileDataFromPresetBundle(); + bool BuildProfileDataFromVendors(); int SaveProfile(); int GetFilamentInfo( std::string VendorDirectory,json & pFilaList, std::string filepath, std::string &sVendor, std::string &sType); @@ -112,8 +121,11 @@ private: //First Load bool bFirstComplete{false}; - bool m_destroy{false}; - boost::thread* m_load_task{ nullptr }; + std::atomic m_destroy{false}; + // Shared cancel token captured by CallAfter lambdas so they don't touch + // `this` after the destructor has run and the object is freed. + std::shared_ptr> m_cancel_token{std::make_shared>(false)}; + std::unique_ptr m_load_task; // User Config bool PrivacyUse; @@ -123,6 +135,7 @@ private: bool InstallNetplugin; bool network_plugin_ready {false}; + json m_ProfileJson; json m_OrcaFilaList; std::string m_OrcaFilaLibPath; diff --git a/src/slic3r/Utils/PresetUpdater.cpp b/src/slic3r/Utils/PresetUpdater.cpp index 9ec57fd653..0e94d00ceb 100644 --- a/src/slic3r/Utils/PresetUpdater.cpp +++ b/src/slic3r/Utils/PresetUpdater.cpp @@ -1042,47 +1042,48 @@ void PresetUpdater::priv::check_installed_vendor_profiles() const std::set bundles; // Orca: always install filament library bundles.insert(PresetBundle::ORCA_FILAMENT_LIBRARY); - for (auto &dir_entry : boost::filesystem::directory_iterator(rsrc_path)) { - const auto &path = dir_entry.path(); - std::string file_path = path.string(); - if (is_json_file(file_path)) { - const auto path_in_vendor = vendor_path / path.filename(); - std::string vendor_name = path.filename().string(); - // Remove the .json suffix. - vendor_name.erase(vendor_name.size() - 5); - if (bundles.find(vendor_name) != bundles.end())continue; + // A vendor is named by its profile or, where the build ships preset caches + // instead of the raw profile JSONs, by its cache alone. + for (const std::string &vendor_name : vendor_names_in(rsrc_path)) { + if (bundles.find(vendor_name) != bundles.end())continue; - const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE - || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); - if (enabled_config_update) { - if ( fs::exists(path_in_vendor)) { - if (is_vendor_enabled) { - Semver resource_ver = get_version_from_json(file_path); - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + const auto is_vendor_enabled = (vendor_name == PresetBundle::ORCA_DEFAULT_BUNDLE) // always update configs from resource to vendor for ORCA_DEFAULT_BUNDLE + || (enabled_vendors.find(vendor_name) != enabled_vendors.end()); + if (enabled_config_update) { + if (is_vendor_installed(vendor_name)) { + if (is_vendor_enabled) { + // Orca: whichever form of the vendor resources ships at the newer + // version is the one installing lays down, and the one to judge + // what is installed against. + Semver resource_ver = resource_vendor_version(vendor_name); + // Orca: a vendor installed as a preset cache has no profile + // beside it; the version it was installed at is in the cache. + Semver vendor_ver = installed_vendor_version(vendor_name); - bool version_match = ((resource_ver.maj() == vendor_ver.maj()) && (resource_ver.min() == vendor_ver.min())); + bool version_match = ((resource_ver.maj() == vendor_ver.maj()) && (resource_ver.min() == vendor_ver.min())); - if (!version_match || (vendor_ver < resource_ver)) { - BOOST_LOG_TRIVIAL(info) << "[Orca Updater]:found vendor "< 0) { @@ -1162,11 +1163,12 @@ Updates PresetUpdater::priv::get_config_updates(const Semver &old_slic3r_version auto filament_in_cache = (cache_profile_path / vendor_name / PRESET_FILAMENT_NAME); auto machine_in_cache = (cache_profile_path / vendor_name / PRESET_PRINTER_NAME); - if (( fs::exists(path_in_vendor)) + if (is_vendor_installed(vendor_name) || fs::exists(print_in_cache) || fs::exists(filament_in_cache) || fs::exists(machine_in_cache)) { - Semver vendor_ver = get_version_from_json(path_in_vendor.string()); + // Orca: a vendor installed as a preset cache carries its version there. + Semver vendor_ver = installed_vendor_version(vendor_name); std::map key_values; std::vector keys(3); diff --git a/tests/libslic3r/CMakeLists.txt b/tests/libslic3r/CMakeLists.txt index 7524c27479..66a181873b 100644 --- a/tests/libslic3r/CMakeLists.txt +++ b/tests/libslic3r/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(${_TEST_NAME}_tests test_preset_bundle_loading.cpp test_preset_setting_id.cpp test_preset_diff.cpp + test_vendor_cache.cpp test_elephant_foot_compensation.cpp test_geometry.cpp test_multimaterial_segmentation.cpp diff --git a/tests/libslic3r/test_preset_bundle_loading.cpp b/tests/libslic3r/test_preset_bundle_loading.cpp index 351535dd9d..d2db84f077 100644 --- a/tests/libslic3r/test_preset_bundle_loading.cpp +++ b/tests/libslic3r/test_preset_bundle_loading.cpp @@ -146,7 +146,7 @@ TEST_CASE("Current vendor type tolerates missing printer model", "[Preset][Bundl { PresetBundle bundle; - VendorProfile orca_vendor("ORCA"); + VendorProfile orca_vendor; orca_vendor.id = "ORCA"; VendorProfile::PrinterModel model; model.name = "Orca Test"; orca_vendor.models.emplace_back(model); diff --git a/tests/libslic3r/test_vendor_cache.cpp b/tests/libslic3r/test_vendor_cache.cpp new file mode 100644 index 0000000000..c3b90fcde8 --- /dev/null +++ b/tests/libslic3r/test_vendor_cache.cpp @@ -0,0 +1,1036 @@ +#include + +#include +#include +#include +#include + +#include "libslic3r/PresetBundle.hpp" +#include "libslic3r/Preset.hpp" +#include "libslic3r/PrintConfig.hpp" +#include "libslic3r/Utils.hpp" + +using namespace Slic3r; +namespace fs = boost::filesystem; + +namespace { + +struct TempDir { + fs::path path; + TempDir() { + path = fs::temp_directory_path() / fs::unique_path("orca-cache-test-%%%%-%%%%"); + fs::create_directories(path); + } + ~TempDir() { boost::system::error_code ec; fs::remove_all(path, ec); } +}; + +std::string write_vendor_json(const fs::path& dir, const std::string& vendor_id, + const std::string& version = "1.0.0") +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"version":")" << version << R"(","name":")" << vendor_id << R"("})"; + return p.string(); +} + +// One vendor profile with a single process preset beside it, as an install or an +// update lays it down: /.json plus //process/standard.json. +void write_vendor_tree(const fs::path& dir, const std::string& vendor, const std::string& version) +{ + fs::create_directories(dir / vendor / "process"); + std::ofstream((dir / (vendor + ".json")).string()) + << R"({"version":")" << version << R"(","name":")" << vendor + << R"(","process_list":[{"name":"0.20mm Standard @)" << vendor << R"(","sub_path":"process/standard.json"}]})"; + std::ofstream((dir / vendor / "process" / "standard.json").string()) + << R"({"type":"process","name":"0.20mm Standard @)" << vendor + << R"(","from":"system","instantiation":"true","layer_height":"0.2"})"; +} + +std::string write_versionless_vendor_json(const fs::path& dir, const std::string& vendor_id) +{ + const fs::path p = dir / (vendor_id + ".json"); + std::ofstream f(p.string()); + f << R"({"name":")" << vendor_id << R"("})"; + return p.string(); +} + +void corrupt_blob_byte(const std::string& path) +{ + std::fstream f(path, std::ios::in | std::ios::out | std::ios::binary); + f.seekp(30); + char b = 0; f.read(&b, 1); + f.seekp(30); + b ^= 0xFF; + f.write(&b, 1); +} + +// Patch cache_version (blob[0..3], i.e. file offset 20) and recompute CRC so the +// file passes the CRC check but fails the cache_version check in load_vendor_cache. +void patch_cache_version(const std::string& path, uint32_t wrong_version) +{ + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + if (data.size() < 24) return; + std::memcpy(&data[20], &wrong_version, 4); + boost::crc_32_type crc; + crc.process_bytes(&data[20], data.size() - 20); + const uint32_t new_crc = crc.checksum(); + std::memcpy(&data[16], &new_crc, 4); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); +} + +// Truncates the cache's PAYLOAD (everything after the 20-byte header) by +// `truncate_by` bytes and recomputes data_size/crc32 in the header, exactly +// as write_cache_blob computes them, so read_cache_blob's size and CRC checks +// still pass but cereal runs out of bytes partway through deserializing the +// body — exercising load_vendor_cache's catch block instead of its early +// (pre-body) rejection paths. +void truncate_payload_and_fix_header(const std::string& path, size_t truncate_by) +{ + constexpr size_t header_size = 20; // magic(4) + version(4) + data_size(8) + crc32(4) + std::ifstream in(path, std::ios::binary); + std::vector data(std::istreambuf_iterator(in), {}); + in.close(); + REQUIRE(data.size() > header_size + truncate_by); + const size_t new_payload_size = data.size() - header_size - truncate_by; + const uint64_t data_size_field = static_cast(new_payload_size); + boost::crc_32_type crc; + crc.process_bytes(&data[header_size], new_payload_size); + const uint32_t crc_field = crc.checksum(); + std::memcpy(&data[8], &data_size_field, sizeof(data_size_field)); // data_size offset + std::memcpy(&data[16], &crc_field, sizeof(crc_field)); // crc32 offset + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(header_size + new_payload_size)); +} + +void add_vendor(PresetBundle& bundle, const std::string& vendor_id, + const std::string& name = "", Semver ver = Semver(1, 0, 0)) +{ + VendorProfile vp; vp.id = vendor_id; + vp.name = name.empty() ? vendor_id + " Corp" : name; + vp.config_version = ver; + bundle.vendors.emplace(vendor_id, vp); +} + +Preset& add_system_preset(PresetCollection& coll, const std::string& name, + const VendorProfile* vp) +{ + Preset& p = coll.load_preset("", name, DynamicPrintConfig(coll.default_preset().config), false); + p.is_system = true; + p.vendor = vp; + return p; +} + +static bool save_one_vendor(const PresetBundle& bundle, const std::string& path, + const std::string& vendor, const std::string& vendor_version, + const std::string& lib_version = "1.0.0") +{ + return bundle.save_vendor_cache(path, vendor, vendor_version, lib_version); +} + +// resources_dir()/data_dir() are process-wide, so restore them however the test +// leaves — including through a failed REQUIRE — to stay green under --order rand. +struct ScopedDirs { + std::string prev_data{data_dir()}, prev_rsrc{resources_dir()}; + ScopedDirs(const fs::path& data, const fs::path& rsrc) + { + set_data_dir(data.string()); + set_resources_dir(rsrc.string()); + } + ~ScopedDirs() { set_data_dir(prev_data); set_resources_dir(prev_rsrc); } +}; + +// Helper: filter a collection by vendor_id. +std::vector presets_for(const PresetCollection& coll, const std::string& vendor_id) +{ + std::vector out; + for (const Preset& p : coll()) + if (p.is_system && p.vendor && p.vendor->id == vendor_id) + out.push_back(&p); + return out; +} + +} // namespace + +namespace Slic3r { +inline bool operator==(const VendorProfile::PrinterVariant& a, const VendorProfile::PrinterVariant& b) { return a.name == b.name; } +inline bool operator==(const VendorProfile::PrinterModel& a, const VendorProfile::PrinterModel& b) +{ + return a.id == b.id && a.name == b.name && a.model_id == b.model_id && a.technology == b.technology + && a.family == b.family && a.variants == b.variants && a.default_materials == b.default_materials + && a.not_support_bed_types == b.not_support_bed_types && a.bed_model == b.bed_model + && a.bed_texture == b.bed_texture && a.image_bed_type == b.image_bed_type + && a.bottom_texture_end_name == b.bottom_texture_end_name + && a.use_double_extruder_default_texture == b.use_double_extruder_default_texture + && a.bottom_texture_rect == b.bottom_texture_rect + && a.bottom_texture_rect_longer == b.bottom_texture_rect_longer + && a.middle_texture_rect == b.middle_texture_rect && a.hotend_model == b.hotend_model; +} +} // namespace Slic3r + +static bool vendor_deep_equal(const VendorProfile& a, const VendorProfile& b) +{ + return a.name == b.name && a.id == b.id && a.config_version == b.config_version + && a.config_update_url == b.config_update_url && a.changelog_url == b.changelog_url + && a.models == b.models && a.default_filaments == b.default_filaments + && a.default_sla_materials == b.default_sla_materials; +} + +static bool preset_deep_equal(const Preset& a, const Preset& b) +{ + return a.type == b.type && a.is_default == b.is_default && a.is_external == b.is_external + && a.is_system == b.is_system && a.is_visible == b.is_visible && a.is_dirty == b.is_dirty + && a.is_compatible == b.is_compatible && a.is_project_embedded == b.is_project_embedded + && a.name == b.name && a.file == b.file && a.loaded == b.loaded + && a.config.equals(b.config) + && a.alias == b.alias && a.renamed_from == b.renamed_from + && a.m_excluded_from == b.m_excluded_from && a.m_from_orca_filament_lib == b.m_from_orca_filament_lib + && a.bundle_id == b.bundle_id && a.version == b.version && a.ini_str == b.ini_str + && a.setting_id == b.setting_id && a.filament_id == b.filament_id && a.user_id == b.user_id + && a.base_id == b.base_id && a.sync_info == b.sync_info && a.description == b.description + && a.updated_time == b.updated_time && a.key_values == b.key_values; +} + +TEST_CASE("a saved cache loads back with names, aliases and filament ids intact", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + const VendorProfile* vp = &src.vendors.at(vid); + + Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", vp); + fp.alias = "Acme PLA"; + fp.filament_id = "GFL_acme_pla"; + add_system_preset(src.printers, vid + " Printer 0.4", vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 1); + CHECK(fi[0]->name == vid + " PLA @0.4"); + CHECK(fi[0]->alias == "Acme PLA"); + CHECK(fi[0]->filament_id == "GFL_acme_pla"); + REQUIRE(pr.size() == 1); + CHECK(pr[0]->name == vid + " Printer 0.4"); +} + +TEST_CASE("loading a missing cache file returns false", "[VendorCache]") +{ + TempDir tmp; + PresetBundle out; + REQUIRE(!out.load_vendor_cache((tmp.path / "nonexistent.opc").string(), "Acme", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache with a corrupted byte is rejected by the CRC check", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + corrupt_blob_byte(cache.string()); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("two vendors produce two independent cache files", "[VendorCache]") +{ + TempDir tmp; + const fs::path cacheA = tmp.path / "vendorA.opc"; + const fs::path cacheB = tmp.path / "vendorB.opc"; + + PresetBundle srcA; + add_vendor(srcA, "VendorA"); + add_system_preset(srcA.filaments, "VendorA PLA", &srcA.vendors.at("VendorA")); + REQUIRE(save_one_vendor(srcA, cacheA.string(), "VendorA", "1.0.0")); + + PresetBundle srcB; + add_vendor(srcB, "VendorB"); + add_system_preset(srcB.filaments, "VendorB PLA", &srcB.vendors.at("VendorB")); + REQUIRE(save_one_vendor(srcB, cacheB.string(), "VendorB", "1.0.0")); + + // Corrupt only vendor B's file; vendor A's must be unaffected. + corrupt_blob_byte(cacheB.string()); + + PresetBundle outA; + REQUIRE(outA.load_vendor_cache(cacheA.string(), "VendorA", "1.0.0","1.0.0")); + REQUIRE(outA.vendors.count("VendorA") == 1); + REQUIRE(presets_for(outA.filaments, "VendorA").size() == 1); + + PresetBundle outB; + REQUIRE(!outB.load_vendor_cache(cacheB.string(), "VendorB", "1.0.0","1.0.0")); + REQUIRE(outB.vendors.empty()); +} + +TEST_CASE("vendor profile fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + VendorProfile vp; vp.id = vid; + vp.name = "Acme Corporation"; + vp.config_version = Semver(2, 5, 1); + VendorProfile::PrinterModel model; + model.id = "AcmePro"; + model.name = "Acme Pro"; + VendorProfile::PrinterVariant v0_4; v0_4.name = "0.4"; + model.variants.push_back(v0_4); + vp.models.push_back(model); + src.vendors.emplace(vid, vp); + REQUIRE(save_one_vendor(src, cache.string(), vid, "2.5.1")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "2.5.1","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + const VendorProfile& gvp = out.vendors.at(vid); + REQUIRE(vendor_deep_equal(gvp, src.vendors.at(vid))); + // Spot-check the fields the old test asserted directly, so a + // vendor_deep_equal regression still points at what actually broke. + CHECK(gvp.id == vid); + CHECK(gvp.name == "Acme Corporation"); + REQUIRE(gvp.models.size() == 1); + CHECK(gvp.models[0].id == "AcmePro"); + CHECK(gvp.models[0].name == "Acme Pro"); + REQUIRE(gvp.models[0].variants.size() == 1); + CHECK(gvp.models[0].variants[0].name == "0.4"); +} + +TEST_CASE("config option values survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + Preset& fp = add_system_preset(src.filaments, vid + " PETG @0.4", &src.vendors.at(vid)); + fp.config.set_key_value("filament_type", new ConfigOptionStrings({"PETG"})); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + auto fi = presets_for(out.filaments, vid); + REQUIRE(fi.size() == 1); + const auto* ft = fi[0]->config.option("filament_type"); + REQUIRE(ft != nullptr); + REQUIRE(ft->values.size() >= 1); + CHECK(ft->values[0] == "PETG"); +} + +TEST_CASE("multiple presets in one collection all round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + const VendorProfile* vp = &src.vendors.at(vid); + + const std::vector fi_names = {vid + " PLA", vid + " PETG", vid + " ABS"}; + const std::vector pr_names = {vid + " Printer 0.4", vid + " Printer 0.6"}; + for (const auto& n : fi_names) add_system_preset(src.filaments, n, vp); + for (const auto& n : pr_names) add_system_preset(src.printers, n, vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + auto fi = presets_for(out.filaments, vid); + auto pr = presets_for(out.printers, vid); + REQUIRE(fi.size() == 3); + REQUIRE(pr.size() == 2); + + std::set fi_got, pr_got; + for (const auto* p : fi) fi_got.insert(p->name); + for (const auto* p : pr) pr_got.insert(p->name); + for (const auto& n : fi_names) CHECK(fi_got.count(n) == 1); + for (const auto& n : pr_names) CHECK(pr_got.count(n) == 1); +} + +TEST_CASE("a truncated cache file is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "truncated.opc"; + { + std::ofstream f(cache.string(), std::ios::binary); + const char data[] = {0x4F, 0x52, 0x43}; + f.write(data, sizeof(data)); + } + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache with the wrong magic number is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + { + std::fstream f(cache.string(), std::ios::in | std::ios::out | std::ios::binary); + const uint32_t bad = 0xDEADBEEFu; + f.write(reinterpret_cast(&bad), sizeof(bad)); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("a vendor with no presets saves and loads cleanly", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + VendorProfile vp; vp.id = vid; + vp.name = "Acme Corporation"; + vp.config_version = Semver(1, 0, 0); + src.vendors.emplace(vid, vp); + + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.count(vid) == 1); + CHECK(out.vendors.at(vid).id == vid); + CHECK(out.vendors.at(vid).name == "Acme Corporation"); + CHECK(presets_for(out.filaments, vid).empty()); + CHECK(presets_for(out.printers, vid).empty()); + CHECK(presets_for(out.prints, vid).empty()); +} + +TEST_CASE("all preset metadata fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + Preset& fp = add_system_preset(src.filaments, vid + " PLA @0.4", &src.vendors.at(vid)); + fp.setting_id = "sid-test-001"; + fp.description = "A test filament preset"; + fp.bundle_id = "bundle-xyz"; + fp.user_id = "user-abc"; + fp.base_id = "base-123"; + fp.sync_info = "update"; + fp.updated_time = 1700000000LL; + fp.key_values = {{"color", "red"}, {"diameter", "1.75"}}; + fp.ini_str = "[filament]\nnozzle_temperature = 230\n"; + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + + const Preset* reloaded = out.filaments.find_preset(vid + " PLA @0.4", false); + REQUIRE(reloaded != nullptr); + const Preset* original = src.filaments.find_preset(vid + " PLA @0.4", false); + REQUIRE(original != nullptr); + REQUIRE(preset_deep_equal(*reloaded, *original)); +} + +TEST_CASE("a cache with the wrong cache version is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + patch_cache_version(cache.string(), 0xFFFFFFFFu); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("a cache truncated mid-blob is rejected", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA", &src.vendors.at(vid)); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + { + std::ifstream in(cache.string(), std::ios::binary); + std::vector buf(30); // 20-byte header + 10 bytes of blob + in.read(buf.data(), 30); + in.close(); + std::ofstream out(cache.string(), std::ios::binary | std::ios::trunc); + out.write(buf.data(), 30); + } + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); +} + +TEST_CASE("get_vendor_cache_version returns the version field, or nothing", "[VendorCache]") +{ + TempDir tmp; + const std::string versioned_path = write_vendor_json(tmp.path, "Acme", "2.5.1"); + const std::string versionless_path = write_versionless_vendor_json(tmp.path, "Beta"); + + // Semver::to_string() re-splits patch into two BBS sub-fields (patch/100, + // patch%100), so derive the expected string from the same Semver the JSON + // was written with rather than hardcoding its form. + const auto expected_ver = Semver::parse("2.5.1"); + REQUIRE(expected_ver.has_value()); + CHECK(get_vendor_cache_version(versioned_path) == expected_ver->to_string()); + + // A profile without a version has nothing to validate a cache against. + CHECK(get_vendor_cache_version(versionless_path).empty()); +} + +TEST_CASE("printer model bed texture fields survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path cache = tmp.path / "vendor.opc"; + + PresetBundle src; + add_vendor(src, vid); + VendorProfile::PrinterModel model; + model.id = "N1"; + model.name = "Neat One"; + model.bottom_texture_rect_longer = "5,5,50,10"; + src.vendors.at(vid).models.push_back(model); + REQUIRE(save_one_vendor(src, cache.string(), vid, "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), vid, "1.0.0","1.0.0")); + REQUIRE(out.vendors.at(vid).models.size() == 1); + REQUIRE(vendor_deep_equal(out.vendors.at(vid), src.vendors.at(vid))); + CHECK(out.vendors.at(vid).models[0].bottom_texture_rect_longer == "5,5,50,10"); +} + +TEST_CASE("a cache older than the vendor profile on disk is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.1","1.0.0")); +} + +TEST_CASE("a cache newer than the vendor profile on disk is used", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + add_system_preset(src.filaments, "Acme PLA", &src.vendors.at("Acme")); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.2.0", "2.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "1.0.0")); + CHECK(presets_for(out.filaments, "Acme").size() == 1); +} + +TEST_CASE("a cache built against an older filament library is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0", "2.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "2.0.1")); +} + +TEST_CASE("a profile with no usable version is never served from cache", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + + PresetBundle out; + // An unversioned vendor profile has no version to compare against. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "", "1.0.0")); + // Neither has an unversioned filament library. + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "")); + // And a cache carrying no version of its own is unusable either way. + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "", "")); + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "1.0.0", "1.0.0")); + REQUIRE(out.vendors.empty()); +} + +TEST_CASE("a vendor's cache is its whole installation", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources"; + const fs::path data = tmp.path / "data"; + fs::create_directories(rsrc / "profiles" / "Acme" / "machine"); + write_vendor_json(rsrc / "profiles", "Acme"); + std::ofstream((rsrc / "profiles" / "Acme" / "machine" / "printer.json").string()) << "{}"; + + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, (rsrc / "profiles" / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(data, rsrc); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + // The cache carries the presets, the vendor profile and the version they were + // built at, so it is installed on its own. + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); + CHECK(is_vendor_installed("Acme")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // A vendor with no cache is installed as its profile and preset JSONs instead, + // parsing them being the only way left to load it — and the cache the previous + // install left behind has to go, or it would shadow the profile just installed. + fs::remove(rsrc / "profiles" / "Acme.opc"); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(!fs::exists(data / "system" / "Acme.opc")); + CHECK(fs::exists(data / "system" / "Acme" / "machine" / "printer.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + // Installing the cache again takes the profile and its preset JSONs back out. + REQUIRE(save_one_vendor(src, (rsrc / "profiles" / "Acme.opc").string(), "Acme", "1.0.0")); + REQUIRE(install_vendor_bundles_from_resources({"Acme"})); + CHECK(fs::exists(data / "system" / "Acme.opc")); + CHECK(!fs::exists(data / "system" / "Acme.json")); + CHECK(!fs::exists(data / "system" / "Acme")); +} + +TEST_CASE("a vendor shipped as a cache alone is installed and loaded from it", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + // A packaged build: every vendor is its cache, with no profile of any kind + // beside it — not even the filament library's, and every other vendor's cache + // is judged against the library in effect. + const std::string lib(PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle shipped_lib; + add_vendor(shipped_lib, lib, "Shipped Library"); + REQUIRE(save_one_vendor(shipped_lib, (rsrc / (lib + ".opc")).string(), lib, "1.0.0")); + PresetBundle shipped; + add_vendor(shipped, "Acme", "Shipped Acme"); + REQUIRE(save_one_vendor(shipped, (rsrc / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The version the build ships the vendor at comes from the cache, there being + // no profile to read it from. + CHECK(resource_vendor_version("Acme") == Semver(1, 0, 0)); + + // Nothing installed yet, so the library the shipped cache is judged against is + // the shipped one — itself a cache. + PresetBundle before; + before.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(before.vendors.at("Acme").name == "Shipped Acme"); + + REQUIRE(install_vendor_bundles_from_resources({lib, "Acme"})); + CHECK(fs::exists(user / "Acme.opc")); + CHECK(!fs::exists(user / "Acme.json")); + CHECK(installed_vendor_version("Acme") == Semver(1, 0, 0)); + + PresetBundle after; + after.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(after.vendors.at("Acme").name == "Shipped Acme"); +} + +TEST_CASE("a vendor cache installed in the data dir shadows the shipped one", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + + write_vendor_tree(rsrc, "Acme", "1.0.0"); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle shipped; + add_vendor(shipped, "Acme", "Shipped Acme"); + REQUIRE(save_one_vendor(shipped, (rsrc / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + // The vendor profiles name the vendor "Acme"; the caches name it after where + // they came from, so the loaded name says which source answered. + auto loaded_name = [&user](PresetBundle& bundle) { + bundle.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + return bundle.vendors.at("Acme").name; + }; + + // Nothing installed yet: the shipped cache answers. + PresetBundle from_rsrc; + CHECK(loaded_name(from_rsrc) == "Shipped Acme"); + + // Installing a newer vendor profile makes the shipped cache too old for it, even + // though that cache still matches the profile sitting beside it in resources. + write_vendor_tree(user, "Acme", "2.0.0"); + write_vendor_json(user, PresetBundle::ORCA_FILAMENT_LIBRARY); + PresetBundle stale; + CHECK(loaded_name(stale) == "Acme"); + + // The cache installed alongside it does answer, and wins over the shipped one. + PresetBundle installed; + add_vendor(installed, "Acme", "Installed Acme"); + REQUIRE(save_one_vendor(installed, (user / "Acme.opc").string(), "Acme", "2.0.0")); + PresetBundle from_user; + CHECK(loaded_name(from_user) == "Installed Acme"); +} + +TEST_CASE("a cache installed with no profile beside it is used whatever its version", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, "Acme"); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + // Installed at an older version than the one now shipped in resources. Nothing + // sits beside it claiming to be newer, so the cache is what the vendor is. + PresetBundle src; + add_vendor(src, "Acme", "Installed Acme"); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "0.9.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "0.9.0"); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Other").empty()); + CHECK(installed_vendor_version("Acme") == Semver(0, 9, 0)); + + // Loading the vendor takes the installed cache, not the newer shipped profile. + PresetBundle out; + out.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(out.vendors.at("Acme").name == "Installed Acme"); +} + +TEST_CASE("a vendor whose cache covers it is loaded without parsing any JSON", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + PresetBundle src; + add_vendor(src, "Acme", "Cached Acme"); + const VendorProfile* vp = &src.vendors.at("Acme"); + add_system_preset(src.filaments, "Acme PLA @0.4", vp); + add_system_preset(src.printers, "Acme Printer 0.4", vp); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // The cache is the whole installation — no profile, no preset JSONs — and the + // caller asks for the vendor exactly as it would for a JSON install. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::Disable); + CHECK(substitutions.empty()); + CHECK(presets_loaded == 2); + CHECK(out.vendors.at("Acme").name == "Cached Acme"); + + // Nothing was written back: the presets never came from a parse. + CHECK(!fs::exists(user / "Acme.json")); +} + +TEST_CASE("a vendor whose cache is stale falls back to parsing its JSONs", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(rsrc); + fs::create_directories(user); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + + // An update installed the vendor at 2.0.0; the cache next to it was built from + // the profile before that, so it no longer covers what is on disk. + write_vendor_tree(user, "Acme", "2.0.0"); + PresetBundle src; + add_vendor(src, "Acme", "Cached Acme"); + REQUIRE(save_one_vendor(src, (user / "Acme.opc").string(), "Acme", "1.0.0")); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(2, 0, 0)); + + // A one-off parse like this one leaves the stale cache alone: only a bundle + // told its parses are complete writes one. + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") == "1.0.0"); + + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_vendor_cache_version((user / "Acme.json").string())); +} + +TEST_CASE("a vendor with nothing installed is parsed from the shipped profiles", "[VendorCache]") +{ + TempDir tmp; + const fs::path rsrc = tmp.path / "resources" / "profiles"; + const fs::path user = tmp.path / "data" / PRESET_SYSTEM_DIR; + fs::create_directories(user); + fs::create_directories(rsrc); + write_vendor_json(rsrc, PresetBundle::ORCA_FILAMENT_LIBRARY); + write_vendor_tree(rsrc, "Acme", "1.0.0"); + + ScopedDirs dirs(tmp.path / "data", tmp.path / "resources"); + + // Neither a cache nor a profile in the directory asked for, so the vendor comes + // out of resources — which is where a build that ships caches keeps the JSONs a + // rejected cache has to be re-parsed from. + PresetBundle out; + auto [substitutions, presets_loaded] = out.load_vendor_configs_from_json( + user.string(), "Acme", PresetBundle::LoadSystem, ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(presets_loaded == 1); + CHECK(out.vendors.at("Acme").config_version == Semver(1, 0, 0)); + + // The cache such a parse writes lands in the directory asked for, stamped with + // the version of the profile it was actually built from. + PresetBundle caching; + caching.set_generate_vendor_caches(true); + caching.load_vendor_configs_from_json(user.string(), "Acme", PresetBundle::LoadSystem, + ForwardCompatibilitySubstitutionRule::EnableSilent); + CHECK(PresetBundle::peek_vendor_cache_version((user / "Acme.opc").string(), "Acme") + == get_vendor_cache_version((rsrc / "Acme.json").string())); +} + +TEST_CASE("a cache with a mismatched vendor name is rejected", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "VendorA"); + REQUIRE(save_one_vendor(src, cache.string(), "VendorA", "1.0.0")); + + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "VendorB", "1.0.0","1.0.0")); +} + +TEST_CASE("a cache is rejected against an unparsable version", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + PresetBundle out; + REQUIRE(!out.load_vendor_cache(cache.string(), "Acme", "not-a-version","1.0.0")); + REQUIRE(out.vendors.empty()); // rejection happens before the body is touched +} + +TEST_CASE("obsolete preset names survive a cache round-trip", "[VendorCache]") +{ + TempDir tmp; + const fs::path cache = tmp.path / "vendor.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + src.obsolete_presets.filaments = {"Old filament"}; + src.obsolete_presets.printers = {"Old printer"}; + REQUIRE(save_one_vendor(src, cache.string(), "Acme", "1.0.0")); + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), "Acme", "1.0.0","1.0.0")); + REQUIRE(out.obsolete_presets.filaments == std::vector{"Old filament"}); + REQUIRE(out.obsolete_presets.printers == std::vector{"Old printer"}); +} + +TEST_CASE("printer hold aliases survive a cache round-trip", "[VendorCache]") +{ + // m_printer_hold_alias has no public getter: is_alias_exist() looks it up, + // but only after finding the alias in m_map_alias_to_profile_name, which + // is populated solely by the (protected) system-preset load path, so it + // cannot be probed here without that machinery. Round-trip it the same + // way the general parity test does instead: save, load, re-save, and + // require the second file to be byte-identical to the first — if load + // dropped or mangled the hold-alias map, the two saves would differ. Kept + // deliberately preset-free so this test isolates the hold-alias map from + // the fuller parity test below. + TempDir tmp; + const fs::path cache1 = tmp.path / "c1.opc"; + const fs::path cache2 = tmp.path / "c2.opc"; + const std::string vid = "Acme"; + const std::string printer_name = vid + " Printer 0.4"; + + PresetBundle src; + add_vendor(src, vid); + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({printer_name})); + src.filaments.set_printer_hold_alias("Acme PLA", probe); + + REQUIRE(save_one_vendor(src, cache1.string(), vid, "1.0.0")); + PresetBundle loaded; + REQUIRE(loaded.load_vendor_cache(cache1.string(), vid, "1.0.0","1.0.0")); + REQUIRE(save_one_vendor(loaded, cache2.string(), vid, "1.0.0")); + + auto slurp = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + REQUIRE(slurp(cache1) == slurp(cache2)); +} + +TEST_CASE("inheritance maps survive a cache round-trip", "[VendorCache]") +{ + // m_config_maps/m_filament_id_maps are public data members of PresetBundle + // (no accessor needed); they matter only for the Orca filament library's + // own cache, since they are the inheritance base other vendors resolve + // against, but save_vendor_cache/load_vendor_cache archive them + // unconditionally regardless of which vendor is being cached. + TempDir tmp; + const fs::path cache = tmp.path / "lib.opc"; + const std::string lib_vendor = "OrcaFilamentLibrary"; + + PresetBundle src; + DynamicPrintConfig cfg; + cfg.set_key_value("filament_type", new ConfigOptionStrings({"PLA"})); + src.m_config_maps["Generic PLA"] = cfg; + src.m_filament_id_maps["Generic PLA"] = "GFL00"; + + REQUIRE(save_one_vendor(src, cache.string(), lib_vendor, "1.0.0", "1.0.0")); + + PresetBundle out; + REQUIRE(out.load_vendor_cache(cache.string(), lib_vendor, "1.0.0", "1.0.0")); + REQUIRE(out.m_filament_id_maps == src.m_filament_id_maps); + REQUIRE(out.m_config_maps.count("Generic PLA") == 1); + CHECK(out.m_config_maps.at("Generic PLA").equals(src.m_config_maps.at("Generic PLA"))); +} + +TEST_CASE("a loaded cache re-serializes to byte-identical output", "[VendorCache]") +{ + // The strongest cheap parity check: if load dropped or mangled anything + // that save writes (including the inheritance maps and hold aliases), + // the second file differs. + TempDir tmp; + const fs::path cache1 = tmp.path / "c1.opc", cache2 = tmp.path / "c2.opc"; + PresetBundle src; + add_vendor(src, "Acme"); + const VendorProfile* vp = &src.vendors.at("Acme"); + Preset& f = add_system_preset(src.filaments, "Acme Filament", vp); + f.alias = "AF"; + add_system_preset(src.printers, "Acme Printer 0.4 nozzle", vp); + src.obsolete_presets.prints = {"Old print"}; + src.m_config_maps["Generic PLA"] = DynamicPrintConfig(); + src.m_filament_id_maps["Generic PLA"] = "GFL00"; + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({"Acme Printer 0.4 nozzle"})); + src.filaments.set_printer_hold_alias("AF", probe); + + REQUIRE(src.save_vendor_cache(cache1.string(), "Acme", "1.0.0", "1.0.0")); + PresetBundle loaded; + REQUIRE(loaded.load_vendor_cache(cache1.string(), "Acme", "1.0.0", "1.0.0")); + REQUIRE(loaded.save_vendor_cache(cache2.string(), "Acme", "1.0.0", "1.0.0")); + auto slurp = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + REQUIRE(slurp(cache1) == slurp(cache2)); +} + +TEST_CASE("a cache that fails mid-body deserialization is rejected and leaves the bundle clean", "[VendorCache]") +{ + TempDir tmp; + const std::string vid = "Acme"; + const fs::path valid_cache = tmp.path / "valid.opc"; + const fs::path corrupt_cache = tmp.path / "corrupt.opc"; + + PresetBundle src; + add_vendor(src, vid); + add_system_preset(src.filaments, vid + " PLA @0.4", &src.vendors.at(vid)); + Preset probe(Preset::TYPE_FILAMENT, "probe"); + probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({vid + " Printer 0.4"})); + src.filaments.set_printer_hold_alias(vid + " PLA alias", probe); + + REQUIRE(save_one_vendor(src, valid_cache.string(), vid, "1.0.0")); + // Truncate the tail (obsolete-preset vectors + m_errors, per save_vendor_cache's + // field order) so the header's size/CRC still validate but cereal runs out of + // bytes partway through the body. Grow the cut if a given size ever stops + // throwing (e.g. after an unrelated field-order change to the cache format). + size_t truncate_by = 40; + bool throws = false; + for (; truncate_by <= 200; truncate_by += 8) { + fs::copy_file(valid_cache, corrupt_cache, fs::copy_option::overwrite_if_exists); + truncate_payload_and_fix_header(corrupt_cache.string(), truncate_by); + PresetBundle probe_bundle; + if (!probe_bundle.load_vendor_cache(corrupt_cache.string(), vid, "1.0.0","1.0.0")) { + throws = true; + break; + } + } + REQUIRE(throws); + + PresetBundle out; + // Pre-populate the target bundle with prior-cycle state: Fix 1(b) must + // clear this in the catch block, not just leave it from a previous cycle. + Preset stale_probe(Preset::TYPE_FILAMENT, "stale-probe"); + stale_probe.config.set_key_value("compatible_printers", new ConfigOptionStrings({"Stale Printer"})); + out.filaments.set_printer_hold_alias("Stale alias", stale_probe); + + REQUIRE(!out.load_vendor_cache(corrupt_cache.string(), vid, "1.0.0","1.0.0")); + CHECK(out.vendors.empty()); + + // No public getter exists for m_printer_hold_alias, so probe it the same + // way the round-trip tests do: serialize `out` and a never-touched fresh + // bundle under the same vendor/keys and require the bytes to match. Since + // save_collection archives the hold-alias map, any leftover "Stale alias" + // entry would make the two files differ. + const fs::path out_after = tmp.path / "out_after.opc"; + const fs::path clean_ref = tmp.path / "clean_ref.opc"; + PresetBundle clean; + REQUIRE(out.save_vendor_cache(out_after.string(), vid, "1.0.0","1.0.0")); + REQUIRE(clean.save_vendor_cache(clean_ref.string(), vid, "1.0.0","1.0.0")); + auto slurp2 = [](const fs::path& p) { + std::ifstream ifs(p.string(), std::ios::binary); + return std::string(std::istreambuf_iterator(ifs), {}); + }; + CHECK(slurp2(out_after) == slurp2(clean_ref)); + + // The recovery must leave a bundle a caller can still load a good cache into. + REQUIRE(out.load_vendor_cache(valid_cache.string(), vid, "1.0.0","1.0.0")); + CHECK(out.vendors.count(vid) == 1); + CHECK(presets_for(out.filaments, vid).size() == 1); +}