From 2f67e460473fcd304a9faf68ed8f4c8f99f81e1b Mon Sep 17 00:00:00 2001 From: ExPikaPaka Date: Tue, 28 Jul 2026 12:53:05 +0200 Subject: [PATCH] fix(codegen): drop grpcio-tools, stop committing generated code The codegen only ever needed a protoc binary, but every entry point installed grpcio-tools to get one. That drags in the grpcio C extension, which has no Windows/ARM64 wheel and falls back to building from source there, so the ARM64 job died with "Failed building wheel for grpcio" -> "protoc not found". tools/codegen_toolchain.py resolves the toolchain instead: protoc from $PROTOC, PATH, a cache, grpc_tools when already installed, or a pinned checksum-verified protoc release unpacked into .codegen-tools/; protobuf and pyyaml from the calling interpreter or a cached virtualenv it re-execs into (distro Pythons refuse `pip install` under PEP 668). All four build scripts and all three CI jobs are now just `python tools/run_codegen.py`, with no pip lines around it. tools/config_metadata_pb2.py was the one generated file checked into git. The orca.* option extensions are now read out of the descriptor set, which already carries config_metadata.proto via --include_imports, so nothing is generated into the tree -- and the protobuf>=6.33.5,<7 CI pin goes away with it, since it only existed to satisfy gencode's hard ValidateProtobufRuntimeVersion check. Generated C++ verified byte-identical under upb and the pure-Python protobuf runtime (what win/arm64 installs), and under both grpc_tools' and standalone protoc. Also fixed: - build_release_macos.sh still passed -DPython3_EXECUTABLE=, pointing the bundled *embed* interpreter at the codegen environment -- the same confusion fae4b124 fixed on the CMake side. - Tab.cpp #includes TabLayout_generated.cpp but had no dependency on codegen_config, so an incremental build after a .proto edit could compile it while the file was being rewritten. libslic3r already had this guard. - ConfigCodegen.cmake now probes with `codegen_toolchain.py --check` (which never downloads or installs), prefers a host interpreter over the embed one, and lets a fresh clone generate at configure time instead of erroring out. Co-Authored-By: Claude Opus 5 --- .github/workflows/build_all.yml | 6 +- .github/workflows/build_orca.yml | 32 +--- .gitignore | 3 + CMakeLists.txt | 3 +- build_linux.sh | 3 +- build_release_macos.sh | 8 +- build_release_vs.bat | 3 +- build_release_vs2022.bat | 3 +- cmake/modules/ConfigCodegen.cmake | 56 ++++--- docs/PrintConfig_Codegen_Design.md | 53 +++--- src/slic3r/CMakeLists.txt | 10 ++ tools/codegen_toolchain.py | 260 +++++++++++++++++++++++++++++ tools/config_codegen.py | 31 ++-- tools/config_metadata.py | 67 ++++++++ tools/config_metadata_pb2.py | 51 ------ tools/run_codegen.py | 45 ++--- 16 files changed, 446 insertions(+), 188 deletions(-) create mode 100644 tools/codegen_toolchain.py create mode 100644 tools/config_metadata.py delete mode 100644 tools/config_metadata_pb2.py diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index 2d509d1eb1..27a7c88c93 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -283,10 +283,8 @@ jobs: scripts/flatpak/com.orcaslicer.OrcaSlicer.yml shell: bash - name: Generate config sources - run: | - python3 -m venv /tmp/codegen_venv - /tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q - /tmp/codegen_venv/bin/python tools/run_codegen.py + # Generated on the host: the flatpak build itself runs offline. + run: python3 tools/run_codegen.py shell: bash - uses: flatpak/flatpak-github-actions/flatpak-builder@master with: diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index a9e9a90710..acc23ceb83 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -57,34 +57,18 @@ jobs: useLocalCache: true # <--= Use the local cache (default is 'false'). useCloudCache: true - - name: Install codegen tools and generate config sources - # grpcio-tools has no Windows/ARM64 wheel for current Python and fails to build its - # grpcio C-extension from source there. The codegen only needs a protoc binary plus the - # protobuf runtime and pyyaml, so install those directly (works on x64 and arm64 Windows; - # the x64 protoc runs under emulation on arm64). protobuf must be >= the version the - # committed config_metadata_pb2.py was generated with (see its runtime-version check). - run: | - pip install "protobuf>=6.33.5,<7" pyyaml - $protocVer = "28.3" - $url = "https://github.com/protocolbuffers/protobuf/releases/download/v$protocVer/protoc-$protocVer-win64.zip" - Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\protoc.zip" - Expand-Archive -Path "$env:RUNNER_TEMP\protoc.zip" -DestinationPath "$env:RUNNER_TEMP\protoc" -Force - $env:PATH = "$env:RUNNER_TEMP\protoc\bin;$env:PATH" - python tools/run_codegen.py + # run_codegen.py resolves its own toolchain (a pinned protoc plus protobuf/pyyaml in a + # cached virtualenv), so nothing has to be pip-installed around it. That is what makes it + # work on windows-11-arm, where `pip install grpcio-tools` has no wheel and fails building + # the grpcio C extension from source. + - name: Generate config sources if: runner.os == 'Windows' + run: python tools/run_codegen.py shell: pwsh - - name: Install codegen tools and generate config sources - run: | - if [ "$(uname)" = "Linux" ]; then - pip3 install grpcio-tools pyyaml - python3 tools/run_codegen.py - else - python3 -m venv /tmp/codegen_venv - /tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q - /tmp/codegen_venv/bin/python tools/run_codegen.py - fi + - name: Generate config sources if: runner.os != 'Windows' + run: python3 tools/run_codegen.py shell: bash - name: Install CMake 3.31.x (Windows ARM64) # windows-11-arm ships CMake 4.x, which removed pre-3.5 policy diff --git a/.gitignore b/.gitignore index d27705b0d2..015248068f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,8 +46,11 @@ test.js internal_docs/ *.flatpak /flatpak-repo/ +# Config codegen: descriptor set, generated C++ and the downloaded protoc / +# bootstrap virtualenv. All of it is produced by tools/run_codegen.py. config.desc src/slic3r/GUI/generated/ +.codegen-tools/ # Python bytecode __pycache__/ *.pyc diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f28e81aa9..b6d565ec7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1084,7 +1084,8 @@ endfunction() # Config codegen — generates src/slic3r/GUI/generated/*.cpp from src/PrintConfigs/*.proto. # Must run before compiling libslic3r (PrintConfig.cpp #includes the generated files). -# Requires: pip install grpcio-tools OR standalone protoc in PATH. +# The toolchain (protoc, protobuf, pyyaml) is resolved by tools/run_codegen.py itself; +# set PROTOC= to use an installed protoc instead of the pinned download. include(cmake/modules/ConfigCodegen.cmake) # libslic3r, OrcaSlicer GUI and the OrcaSlicer executable. diff --git a/build_linux.sh b/build_linux.sh index 5f7df268d7..6ffb799513 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -557,7 +557,8 @@ if [[ -n "${BUILD_ORCA}" ]] || [[ -n "${BUILD_TESTS}" ]] ; then fi echo "Generating config sources from proto..." - pip install grpcio-tools -q + # Resolves protoc and its Python packages itself; nothing is installed into the + # system interpreter (distro Pythons refuse that under PEP 668 anyway). python3 tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; } print_and_run cmake -S . -B $BUILD_DIR "${CMAKE_C_CXX_COMPILER_CLANG[@]}" "${CMAKE_LLD_LINKER_ARGS[@]}" "${CMAKE_CCACHE_ARGS[@]}" -G "Ninja Multi-Config" \ diff --git a/build_release_macos.sh b/build_release_macos.sh index 3a83136145..6ae9f8c1a2 100755 --- a/build_release_macos.sh +++ b/build_release_macos.sh @@ -241,10 +241,9 @@ function verify_python_runtime() { function build_slicer() { echo "Generating config sources from proto..." - python3 -m venv /tmp/codegen_venv - /tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q - /tmp/codegen_venv/bin/python tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; } - CODEGEN_PYTHON="/tmp/codegen_venv/bin/python3" + # Resolves protoc and its Python packages itself (into .codegen-tools/), so nothing + # is installed into the system interpreter. + python3 tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; } # iterate over two architectures: x86_64 and arm64 for _ARCH in x86_64 arm64; do @@ -270,7 +269,6 @@ function build_slicer() { -DCMAKE_OSX_ARCHITECTURES="${_ARCH}" \ -DCMAKE_OSX_DEPLOYMENT_TARGET="${OSX_DEPLOYMENT_TARGET}" \ -DCMAKE_IGNORE_PREFIX_PATH="${CMAKE_IGNORE_PREFIX_PATH}" \ - -DPython3_EXECUTABLE="${CODEGEN_PYTHON}" \ ${CMAKE_POLICY_COMPAT} fi cmake --build . --config "$BUILD_CONFIG" --target "$SLICER_BUILD_TARGET" diff --git a/build_release_vs.bat b/build_release_vs.bat index e9ae7ca527..be52402513 100644 --- a/build_release_vs.bat +++ b/build_release_vs.bat @@ -147,7 +147,8 @@ echo "building Orca Slicer..." cd %WP% echo "generating config sources from proto..." -pip install grpcio-tools -q +REM run_codegen.py resolves protoc and its python packages itself (grpcio-tools has +REM no wheel on ARM64 and fails to build there). python tools/run_codegen.py if errorlevel 1 ( echo "ERROR: config codegen failed" diff --git a/build_release_vs2022.bat b/build_release_vs2022.bat index 777f6f5a40..644cccf0ae 100644 --- a/build_release_vs2022.bat +++ b/build_release_vs2022.bat @@ -68,7 +68,8 @@ echo "building Orca Slicer..." cd %WP% echo "generating config sources from proto..." -pip install grpcio-tools -q +REM run_codegen.py resolves protoc and its python packages itself (grpcio-tools has +REM no wheel on ARM64 and fails to build there). python tools/run_codegen.py if errorlevel 1 ( echo "ERROR: config codegen failed" diff --git a/cmake/modules/ConfigCodegen.cmake b/cmake/modules/ConfigCodegen.cmake index 681c300f64..c627897323 100644 --- a/cmake/modules/ConfigCodegen.cmake +++ b/cmake/modules/ConfigCodegen.cmake @@ -2,7 +2,8 @@ # # Generates C++ source files from protobuf schema definitions. # Generated files live in src/slic3r/GUI/generated/ and are gitignored. -# Run 'python tools/run_codegen.py' (requires grpcio-tools or protoc) to regenerate. +# Run 'python tools/run_codegen.py' to regenerate; it resolves protoc and the Python +# packages it needs itself (see tools/codegen_toolchain.py). # # Targets: # codegen_config - Custom target to regenerate C++ from .proto files @@ -21,24 +22,26 @@ set(_generated_marker "${CMAKE_SOURCE_DIR}/src/slic3r/GUI/generated/PrintConfigD # The main CMakeLists forces Python3_EXECUTABLE to the *bundled embed* interpreter (used for # the in-app Python plugin runtime). That interpreter has neither protoc nor the protobuf / # pyyaml packages, and for cross-compiled targets it may not even be the host architecture — so -# it cannot run tools/run_codegen.py. In CI the generated sources are produced by a dedicated -# pre-build "Install codegen tools and generate config sources" step; developers run -# tools/run_codegen.py themselves. Only wire up (re)generation when the interpreter really has -# the toolchain, otherwise fall back to the already-generated files. +# it cannot run tools/run_codegen.py. Prefer a host interpreter, which tools/run_codegen.py can +# bootstrap the toolchain into. # -# ORCA_CODEGEN_PYTHON lets a build explicitly point at an interpreter that has the tools -# (independent of the embed interpreter). Falls back to Python3_EXECUTABLE. +# ORCA_CODEGEN_PYTHON lets a build explicitly point at the interpreter to use. if(NOT ORCA_CODEGEN_PYTHON) - set(ORCA_CODEGEN_PYTHON "${Python3_EXECUTABLE}") + find_program(_orca_host_python NAMES python3 python) + if(_orca_host_python) + set(ORCA_CODEGEN_PYTHON "${_orca_host_python}") + else() + set(ORCA_CODEGEN_PYTHON "${Python3_EXECUTABLE}") + endif() endif() +# --check answers "can this interpreter regenerate right now?" (protobuf + pyyaml importable +# and a protoc already resolvable). It never downloads or installs, so wiring up the build-time +# regeneration below never turns a build into a network operation. set(_codegen_usable FALSE) if(ORCA_CODEGEN_PYTHON) - # Needs the protobuf python runtime (config_metadata_pb2) plus a protoc (standalone or - # grpc_tools.protoc). Probe the interpreter for both. execute_process( - COMMAND ${ORCA_CODEGEN_PYTHON} -c - "import shutil, importlib.util, google.protobuf; import sys; sys.exit(0 if (shutil.which('protoc') or importlib.util.find_spec('grpc_tools')) else 1)" + COMMAND ${ORCA_CODEGEN_PYTHON} "${CMAKE_SOURCE_DIR}/tools/codegen_toolchain.py" --check RESULT_VARIABLE _codegen_probe OUTPUT_QUIET ERROR_QUIET ) @@ -47,27 +50,28 @@ if(ORCA_CODEGEN_PYTHON) endif() endif() -# If generated files are missing (fresh clone) and we can run codegen, do it now at configure -# time so a plain cmake configure + build works without a separate pre-build step. +# If generated files are missing (fresh clone), run the codegen now at configure time so a plain +# cmake configure + build works without a separate pre-build step. Unlike the build-time +# regeneration above this is allowed to fetch the toolchain — there is nothing to build without it. if(NOT EXISTS "${_generated_marker}") - if(_codegen_usable) + set(_codegen_result 1) + if(ORCA_CODEGEN_PYTHON) message(STATUS "Config codegen: generated files missing — running codegen now...") execute_process( COMMAND ${ORCA_CODEGEN_PYTHON} "${CMAKE_SOURCE_DIR}/tools/run_codegen.py" --no-validate WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE _codegen_result ) - if(NOT _codegen_result EQUAL 0) - message(FATAL_ERROR "Config codegen failed. Install grpcio-tools: pip install grpcio-tools") - endif() - message(STATUS "Config codegen: generated files created successfully") - else() - message(FATAL_ERROR - "Config codegen: generated files are missing and no usable codegen toolchain was found.\n" - "Generate them first with an interpreter that has protoc + protobuf, e.g.:\n" - " pip install grpcio-tools pyyaml && python tools/run_codegen.py\n" - "or pass -DORCA_CODEGEN_PYTHON=.") endif() + if(NOT _codegen_result EQUAL 0) + message(FATAL_ERROR + "Config codegen: generated files are missing and could not be generated.\n" + "Run it manually with a host Python:\n" + " python tools/run_codegen.py\n" + "or pass -DORCA_CODEGEN_PYTHON=. Offline builds can point at an installed\n" + "protoc with PROTOC=.") + endif() + message(STATUS "Config codegen: generated files created successfully") endif() set(CONFIG_PROTO_DIR "${CMAKE_SOURCE_DIR}/src/PrintConfigs") @@ -101,7 +105,7 @@ set(CONFIG_PROTO_FILES ) if(_codegen_usable) - # Single command: run_codegen.py handles protoc/grpcio-tools detection internally. + # Single command: run_codegen.py resolves protoc itself. # Proto files → generated .cpp files. Runs automatically when any .proto changes. add_custom_command( OUTPUT ${CONFIG_GENERATED_SOURCES} diff --git a/docs/PrintConfig_Codegen_Design.md b/docs/PrintConfig_Codegen_Design.md index 400110dc84..72b5b0612c 100644 --- a/docs/PrintConfig_Codegen_Design.md +++ b/docs/PrintConfig_Codegen_Design.md @@ -275,9 +275,9 @@ Settings are split into three `.proto` files by preset type. Each setting become | File | Contents | |------|----------| -| `src/PrintConfigs/generated/print.proto` | ~477 print/process settings | -| `src/PrintConfigs/generated/filament.proto` | ~103 filament settings | -| `src/PrintConfigs/generated/printer.proto` | ~42 printer settings | +| `src/PrintConfigs/print.proto` | ~477 print/process settings | +| `src/PrintConfigs/filament.proto` | ~103 filament settings | +| `src/PrintConfigs/printer.proto` | ~42 printer settings | Each file also carries message-level `virtual_preset_keys` declarations (see §5.2.3). @@ -364,13 +364,19 @@ The codegen reads these and merges them (deduplicated, sorted) with the field-de ```cmake add_custom_command( OUTPUT ${GENERATED_SOURCES} - COMMAND protoc --descriptor_set_out=config.desc src/PrintConfigs/generated/*.proto - COMMAND python3 tools/config_codegen.py config.desc ${GENERATED_DIR} - DEPENDS src/PrintConfigs/generated/*.proto tools/config_codegen.py + COMMAND ${ORCA_CODEGEN_PYTHON} tools/run_codegen.py --no-validate + DEPENDS src/PrintConfigs/*.proto src/PrintConfigs/layout.yaml tools/config_codegen.py ) ``` -Generated files are checked into the repo (not gitignored) so builds work without `protoc`. CI validates that committed generated files match what the generator produces. +Generated files are **not** checked into the repo — `src/slic3r/GUI/generated/` and `config.desc` are gitignored, and every build produces them. Each build script and CI job runs `tools/run_codegen.py` before configuring; a fresh clone that goes straight to `cmake` gets them generated at configure time. + +`tools/run_codegen.py` resolves its own toolchain (`tools/codegen_toolchain.py`), so no caller has to install anything: + +- **protoc** — `$PROTOC`, then `PATH`, then a cached copy, then a pinned checksum-verified release downloaded into `.codegen-tools/` (also gitignored) +- **protobuf + pyyaml** — the calling interpreter if it has them, otherwise a virtualenv under `.codegen-tools/` that the script re-execs into + +Set `PROTOC=` to build offline or against a distro protoc. ### 4.5 Provider Customization @@ -431,7 +437,7 @@ Custom options get field numbers > 1000 to avoid conflicts. 1. Add a field to the appropriate `.proto` file (`print.proto`, `filament.proto`, or `printer.proto`) with all relevant annotations 2. Run `python tools/run_codegen.py` -3. Commit the `.proto` change and the updated generated files together +3. Commit the `.proto` change — the generated files are gitignored build output, never committed ### Adding a virtual preset key @@ -461,26 +467,25 @@ python tools/annotate_protos.py [--dry-run] src/PrintConfigs/ ├── config_metadata.proto # Custom field/message option extensions ├── layout.yaml # UI tab/page/group structure (Tab.cpp layout) -└── generated/ - ├── print.proto # ~477 print/process settings - ├── filament.proto # ~103 filament settings - └── printer.proto # ~42 printer/machine settings +├── print.proto # ~477 print/process settings +├── filament.proto # ~103 filament settings +└── printer.proto # ~42 printer/machine settings tools/ -├── parse_printconfig.py # Bootstrap: PrintConfig.cpp → .proto +├── run_codegen.py # Full pipeline script — the entry point everything calls +├── codegen_toolchain.py # Resolves protoc / protobuf / pyyaml ├── config_codegen.py # Proto descriptor → C++ codegen -├── validate_codegen.py # Generated vs original validation -├── run_codegen.py # Full pipeline script -├── annotate_protos.py # Inject (invalidates)/(list_membership) from C++ -├── move_proto_fields.py # Utility: move fields between proto files -└── config_metadata_pb2.py # Generated Python bindings for extensions +├── config_metadata.py # Reads the orca.* option extensions from the descriptor set +└── validate_codegen.py # Generated vs original validation -codegen/ -└── generated/ - ├── PrintConfigDef_generated.cpp # init_fff_params() body — #included by PrintConfig.cpp - ├── Preset_options_generated.cpp # s_Preset_*_options — #included by Preset.cpp - ├── Invalidation_generated.cpp # s_print_steps_map + s_object_steps_map — #included by Print.cpp - └── OptionKeys_generated.cpp # s_extruder_option_keys, s_filament_option_keys +src/slic3r/GUI/generated/ # gitignored — regenerated by every build +├── PrintConfigDef_generated.cpp # init_fff_params() body — #included by PrintConfig.cpp +├── Preset_options_generated.cpp # s_Preset_*_options — #included by Preset.cpp +├── Invalidation_generated.cpp # s_print_steps_map + s_object_steps_map — #included by Print.cpp +├── OptionKeys_generated.cpp # s_extruder_option_keys, s_filament_option_keys +└── TabLayout_generated.cpp # Tab page/group layout from layout.yaml + +.codegen-tools/ # gitignored — downloaded protoc + bootstrap virtualenv cmake/modules/ └── ConfigCodegen.cmake # CMake integration (build-time regeneration) diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 4d589b90f6..61f8ff1b48 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -831,6 +831,16 @@ else () endif () target_include_directories(libslic3r_gui PRIVATE Utils ${CMAKE_CURRENT_BINARY_DIR}) +# Tab.cpp #includes GUI/generated/TabLayout_generated.cpp, so it must not be compiled +# while the codegen is rewriting it (same contract as libslic3r's generated includes). +if(TARGET codegen_config) + add_dependencies(libslic3r_gui codegen_config) + set_source_files_properties( + "${CMAKE_CURRENT_SOURCE_DIR}/GUI/Tab.cpp" + PROPERTIES OBJECT_DEPENDS "${CONFIG_GENERATED_SOURCES}" + ) +endif() + if (WIN32) target_include_directories(libslic3r_gui SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/WebView2/include) target_link_libraries(libslic3r_gui Advapi32) diff --git a/tools/codegen_toolchain.py b/tools/codegen_toolchain.py new file mode 100644 index 0000000000..8917e50e24 --- /dev/null +++ b/tools/codegen_toolchain.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +Toolchain resolution for the config codegen. + +The codegen needs exactly three things: a protoc binary, the protobuf Python +runtime and pyyaml. Every entry point used to install `grpcio-tools` to get +protoc, which drags in the grpcio C extension: it has no Windows/ARM64 wheel and +falls back to building from source there, which is what broke the ARM64 build +("Failed building wheel for grpcio" -> "protoc not found"). Nothing in the +codegen uses gRPC. + +Resolution order, so callers only ever run `python tools/run_codegen.py`: + + protoc $PROTOC -> PATH -> cached download -> grpc_tools (if installed) -> + pinned, checksum-verified protoc release downloaded into + .codegen-tools/ (gitignored) + runtime the current interpreter, else a cached virtualenv under + .codegen-tools/venv that the entry point re-execs into + +Set PROTOC=/path/to/protoc (or put protoc on PATH) to build offline. +""" + +import hashlib +import importlib.util +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CACHE_DIR = ROOT / ".codegen-tools" + +# Pinned so every machine generates with the same compiler, and checksummed +# because we execute what we download. To bump: change the version and refresh +# every hash from https://github.com/protocolbuffers/protobuf/releases/tag/v +PROTOC_VERSION = "28.3" +PROTOC_ARCHIVE_SHA256 = { + "win64": "ce64f49bdeddef49ce4bd313a8f59bcf92fcf67b5831efbf66170386d2e66948", + "linux-x86_64": "0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f", + "linux-aarch_64": "1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a", + "osx-universal_binary": "52df502b263da20f3311b23b5c6553d10cc25c6ebb85df381d80a2806b6a698b", +} + +# pip name -> import name +PYTHON_PACKAGES = {"protobuf": "google.protobuf", "pyyaml": "yaml"} + +# Guards against an endless re-exec loop if the virtualenv still can't import. +_BOOTSTRAP_ENV = "ORCA_CODEGEN_BOOTSTRAPPED" + +_PROTOC_EXE = "protoc.exe" if os.name == "nt" else "protoc" + + +def _protoc_dir(): + return CACHE_DIR / f"protoc-{PROTOC_VERSION}" + + +def _archive_key(): + """Release asset for this host, or None if protobuf ships no build for it.""" + if sys.platform == "win32": + # There is no win/arm64 release; the x64 build runs under Windows' + # emulation, which is how the ARM64 CI job gets a protoc. + return "win64" + if sys.platform == "darwin": + return "osx-universal_binary" + if sys.platform.startswith("linux"): + machine = platform.machine().lower() + if machine in ("x86_64", "amd64"): + return "linux-x86_64" + if machine in ("aarch64", "arm64"): + return "linux-aarch_64" + return None + + +def _download_protoc(): + """Fetch and unpack the pinned protoc. Returns the binary path, or None.""" + key = _archive_key() + if key is None: + print(f" ERROR: no pinned protoc release for {sys.platform}/{platform.machine()}.") + print(" Install protoc from your package manager and re-run, or set PROTOC=.") + return None + + url = (f"https://github.com/protocolbuffers/protobuf/releases/download/" + f"v{PROTOC_VERSION}/protoc-{PROTOC_VERSION}-{key}.zip") + print(f" Downloading protoc {PROTOC_VERSION} ({key})...") + CACHE_DIR.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=CACHE_DIR) as tmp: + archive = Path(tmp) / "protoc.zip" + try: + with urllib.request.urlopen(url, timeout=120) as response: + archive.write_bytes(response.read()) + except OSError as exc: + print(f" ERROR: download failed: {exc}") + print(" Install protoc manually and re-run, or set PROTOC=.") + return None + + actual = hashlib.sha256(archive.read_bytes()).hexdigest() + expected = PROTOC_ARCHIVE_SHA256[key] + if actual != expected: + print(f" ERROR: protoc archive checksum mismatch for {key} {PROTOC_VERSION}:") + print(f" expected {expected}") + print(f" actual {actual}") + return None + + # Unpack the whole archive, not just bin/: protoc resolves the well-known + # imports (google/protobuf/descriptor.proto) from ../include next to it. + staging = Path(tmp) / "unpacked" + with zipfile.ZipFile(archive) as zf: + zf.extractall(staging) + + target = _protoc_dir() + if target.exists(): + shutil.rmtree(target) + # Move into place only once complete, so an interrupted run never leaves + # a half-extracted toolchain that later runs would happily use. + shutil.move(str(staging), str(target)) + + binary = target / "bin" / _PROTOC_EXE + if not binary.exists(): + print(f" ERROR: protoc archive did not contain bin/{_PROTOC_EXE}") + return None + binary.chmod(binary.stat().st_mode | 0o755) + return binary + + +def find_protoc(allow_download=True): + """Return the protoc command as a list, or None if it can't be resolved.""" + override = os.environ.get("PROTOC") + if override: + return [override] + + on_path = shutil.which("protoc") + if on_path: + return [on_path] + + cached = _protoc_dir() / "bin" / _PROTOC_EXE + if cached.exists(): + return [str(cached)] + + # Honour a pre-existing grpcio-tools install rather than downloading. + if importlib.util.find_spec("grpc_tools") is not None: + return [sys.executable, "-m", "grpc_tools.protoc"] + + if allow_download: + binary = _download_protoc() + if binary is not None: + return [str(binary)] + return None + + +def missing_packages(): + """pip names of the required Python packages this interpreter can't import.""" + missing = [] + for pip_name, module in PYTHON_PACKAGES.items(): + try: + found = importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + found = False + if not found: + missing.append(pip_name) + return missing + + +def _venv_python(): + venv_dir = CACHE_DIR / "venv" + if os.name == "nt": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +def bootstrap_python(): + """The cached virtualenv interpreter, if it exists and has the packages.""" + python = _venv_python() + if not python.exists(): + return None + probe = subprocess.run([str(python), "-c", "import google.protobuf, yaml"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return python if probe.returncode == 0 else None + + +def _ensure_venv(missing): + """Create/reuse .codegen-tools/venv with the required packages installed.""" + python = _venv_python() + + if not python.exists(): + print(f" Creating codegen virtualenv in {python.parent.parent}...") + if subprocess.run([sys.executable, "-m", "venv", str(python.parent.parent)]).returncode != 0 \ + or not python.exists(): + return None + + print(f" Installing {', '.join(missing)} into the codegen virtualenv...") + result = subprocess.run([str(python), "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", *missing]) + return python if result.returncode == 0 else None + + +def ensure_python_runtime(): + """ + Guarantee protobuf + pyyaml are importable. + + If they aren't, re-run the calling script in a cached virtualenv that has + them and exit with its status. Keeping the packages out of the caller's + interpreter is what lets the build scripts work on distros that refuse + `pip install` into a system Python (PEP 668). + """ + missing = missing_packages() + if not missing: + return + + if os.environ.get(_BOOTSTRAP_ENV): + # We are already inside the bootstrapped environment: installing again + # would just fail the same way. + print(f" ERROR: {', '.join(missing)} still missing after bootstrap.") + sys.exit(1) + + # Reuse a complete virtualenv as-is: builds call this every time, and pip + # would otherwise hit the network on each one. + python = bootstrap_python() or _ensure_venv(missing) + if python is None: + print(f" ERROR: could not provide {', '.join(missing)}.") + print(f" Install them for this interpreter: {sys.executable} -m pip install " + f"{' '.join(missing)}") + sys.exit(1) + + env = dict(os.environ, **{_BOOTSTRAP_ENV: "1"}) + sys.exit(subprocess.run([str(python), *sys.argv], env=env).returncode) + + +def toolchain_ready(): + """ + True if the codegen can run without downloading or installing anything -- + either from this interpreter or by re-execing into an existing bootstrap + virtualenv. This is what decides whether a build regenerates on proto edits. + """ + if missing_packages() and bootstrap_python() is None: + return False + return find_protoc(allow_download=False) is not None + + +def main(): + # --check is what ConfigCodegen.cmake probes with: it answers whether this + # interpreter can regenerate during a build, and must not download, install + # or create anything while doing so. + if "--check" in sys.argv[1:]: + return 0 if toolchain_ready() else 1 + + missing = missing_packages() + print(f"python: {sys.executable}") + print(f"packages: {'all present' if not missing else 'missing ' + ', '.join(missing)}") + protoc = find_protoc() + print(f"protoc: {' '.join(protoc) if protoc else 'NOT FOUND'}") + return 0 if protoc else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/config_codegen.py b/tools/config_codegen.py index ef40394e50..a90488c88e 100644 --- a/tools/config_codegen.py +++ b/tools/config_codegen.py @@ -10,11 +10,10 @@ Usage: protoc --proto_path=src/PrintConfigs --descriptor_set_out=config.desc \ --include_imports src/PrintConfigs/*.proto - # Step 2: Generate Python bindings (one-time, or when config_metadata.proto changes) - protoc --proto_path=src/PrintConfigs --python_out=tools/ config_metadata.proto + # Step 2: Run codegen + python tools/config_codegen.py config.desc src/slic3r/GUI/generated/ - # Step 3: Run codegen - python tools/config_codegen.py config.desc codegen/generated/ + (tools/run_codegen.py does both, and resolves protoc for you) Outputs: - PrintConfigDef_generated.cpp (init_fff_params body) @@ -29,20 +28,22 @@ import re import argparse from pathlib import Path -# Add tools/ to path so we can import generated config_metadata_pb2 +# Add tools/ to path so we can import the sibling helper modules sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) try: from google.protobuf import descriptor_pb2 - # Import the generated bindings - this registers extensions globally - import config_metadata_pb2 as meta_pb2 + from config_metadata import load_descriptor_set except ImportError as e: print(f"ERROR: {e}") - print("Ensure google-protobuf is installed: pip install protobuf") - print("And that config_metadata_pb2.py exists in tools/") - print("Generate it with: protoc --proto_path=src/PrintConfigs --python_out=tools/ config_metadata.proto") + print("Ensure protobuf is installed: pip install protobuf") + print("Or just run the pipeline, which bootstraps it: python tools/run_codegen.py") sys.exit(1) +# The orca.* option extensions and enum constants, bound by main() from the +# descriptor set (see config_metadata.py for why they aren't imported). +meta_pb2 = None + # Proto FieldDescriptorProto.Type enum values TYPE_DOUBLE = 1 @@ -179,9 +180,8 @@ def parse_field_options(field_desc_proto): Re-parse FieldOptions from a FieldDescriptorProto with extensions registered. This is needed because the FileDescriptorSet parser doesn't know about our custom extensions, so they end up as unknown fields. Re-parsing with the - extensions registered (via config_metadata_pb2 import) resolves them. + extensions registered (see config_metadata.load_descriptor_set) resolves them. """ - from google.protobuf import descriptor_pb2 opts = field_desc_proto.options if not opts.ByteSize(): return descriptor_pb2.FieldOptions() @@ -981,11 +981,8 @@ def main(): print(f"ERROR: Descriptor file not found: {desc_path}") sys.exit(1) - with open(desc_path, 'rb') as f: - raw = f.read() - - file_descriptor_set = descriptor_pb2.FileDescriptorSet() - file_descriptor_set.ParseFromString(raw) + global meta_pb2 + file_descriptor_set, meta_pb2 = load_descriptor_set(desc_path) print(f"Loaded {len(file_descriptor_set.file)} proto files") for fd in file_descriptor_set.file: diff --git a/tools/config_metadata.py b/tools/config_metadata.py new file mode 100644 index 0000000000..642a7d83a0 --- /dev/null +++ b/tools/config_metadata.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +Access to the orca.* option extensions declared in config_metadata.proto. + +The extensions are read straight out of the compiled descriptor set: protoc runs +with --include_imports, so config_metadata.proto travels inside the .desc file +that the codegen already consumes. Registering it in the default descriptor pool +before the descriptor set is parsed is what makes the custom options resolve +instead of landing in unknown fields. + +Doing it this way keeps generated code out of git. The alternative -- a checked-in +config_metadata_pb2.py -- also pinned the protobuf runtime to whichever protoc +produced it, because gencode embeds a hard ValidateProtobufRuntimeVersion() check. +""" + +from google.protobuf import descriptor_pb2, descriptor_pool + +METADATA_PROTO = "config_metadata.proto" + + +class Metadata: + """ + Stand-in for the generated config_metadata_pb2 module. + + Exposes each orca extension as an attribute holding its FieldDescriptor + (usable as `options.Extensions[meta.label]`) and each enum value as an int + constant (`meta.MODE_SIMPLE`, `meta.STEP_SLICE`, ...), matching how the + generated module was used. + """ + + def __init__(self, file_descriptor): + for name, extension in file_descriptor.extensions_by_name.items(): + setattr(self, name, extension) + for enum in file_descriptor.enum_types_by_name.values(): + for value in enum.values: + setattr(self, value.name, value.number) + + +def load_descriptor_set(path): + """ + Read a protoc descriptor set and return (FileDescriptorSet, Metadata). + + The file is parsed twice on purpose: the first pass only locates the embedded + config_metadata.proto so its extensions can be registered, the second one + parses with those extensions known. + """ + with open(path, 'rb') as f: + raw = f.read() + + probe = descriptor_pb2.FileDescriptorSet() + probe.ParseFromString(raw) + metadata_file = next((f for f in probe.file if f.name == METADATA_PROTO), None) + if metadata_file is None: + raise RuntimeError( + f"{path} does not contain {METADATA_PROTO} -- protoc must be run with " + "--include_imports") + + pool = descriptor_pool.Default() + try: + file_descriptor = pool.FindFileByName(METADATA_PROTO) + except KeyError: + pool.Add(metadata_file) + file_descriptor = pool.FindFileByName(METADATA_PROTO) + + descriptor_set = descriptor_pb2.FileDescriptorSet() + descriptor_set.ParseFromString(raw) + return descriptor_set, Metadata(file_descriptor) diff --git a/tools/config_metadata_pb2.py b/tools/config_metadata_pb2.py deleted file mode 100644 index 2a5c74ad86..0000000000 --- a/tools/config_metadata_pb2.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: config_metadata.proto -# Protobuf Python Version: 6.33.5 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 5, - '', - 'config_metadata.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x63onfig_metadata.proto\x12\x04orca\x1a google/protobuf/descriptor.proto\"0\n\x0e\x46loatOrPercent\x12\r\n\x05value\x18\x01 \x01(\x01\x12\x0f\n\x07percent\x18\x02 \x01(\x08\"\x1f\n\x07Point2D\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01*S\n\nConfigMode\x12\x0f\n\x0bMODE_SIMPLE\x10\x00\x12\x11\n\rMODE_ADVANCED\x10\x01\x12\x10\n\x0cMODE_DEVELOP\x10\x02\x12\x0f\n\x0bMODE_EXPERT\x10\x03*G\n\nPresetType\x12\x10\n\x0cPRESET_PRINT\x10\x00\x12\x13\n\x0fPRESET_FILAMENT\x10\x01\x12\x12\n\x0ePRESET_PRINTER\x10\x02*\xaa\x01\n\x10InvalidationStep\x12\x15\n\x11STEP_GCODE_EXPORT\x10\x00\x12\x13\n\x0fSTEP_SKIRT_BRIM\x10\x01\x12\x13\n\x0fSTEP_WIPE_TOWER\x10\x02\x12\x0e\n\nSTEP_SLICE\x10\x03\x12\x13\n\x0fSTEP_PERIMETERS\x10\x04\x12\x0f\n\x0bSTEP_INFILL\x10\x05\x12\x10\n\x0cSTEP_SUPPORT\x10\x06\x12\r\n\tSTEP_NONE\x10\x07*\x81\x01\n\x14OptionListMembership\x12\r\n\tLIST_NONE\x10\x00\x12\x1d\n\x19LIST_EXTRUDER_OPTION_KEYS\x10\x01\x12\x1d\n\x19LIST_FILAMENT_OPTION_KEYS\x10\x02\x12\x1c\n\x18LIST_VARIANT_OPTION_KEYS\x10\x03*\\\n\nCoTypeHint\x12\x16\n\x12\x43O_TYPE_HINT_UNSET\x10\x00\x12\r\n\tcoPercent\x10\x01\x12\x0e\n\ncoPercents\x10\x02\x12\n\n\x06\x63oEnum\x10\x03\x12\x0b\n\x07\x63oEnums\x10\x04*\x96\x01\n\x07GuiType\x12\x12\n\x0eGUI_TYPE_UNSET\x10\x00\x12\x0f\n\x0bi_enum_open\x10\x01\x12\x0f\n\x0b\x66_enum_open\x10\x02\x12\t\n\x05\x63olor\x10\x03\x12\x0f\n\x0bselect_open\x10\x04\x12\n\n\x06slider\x10\x05\x12\n\n\x06legend\x10\x06\x12\x0e\n\none_string\x10\x07\x12\x11\n\rplugin_picker\x10\x08:.\n\x05label\x12\x1d.google.protobuf.FieldOptions\x18\xd1\x86\x03 \x01(\t:3\n\nfull_label\x12\x1d.google.protobuf.FieldOptions\x18\xd2\x86\x03 \x01(\t:0\n\x07tooltip\x12\x1d.google.protobuf.FieldOptions\x18\xd3\x86\x03 \x01(\t:1\n\x08\x63\x61tegory\x12\x1d.google.protobuf.FieldOptions\x18\xd4\x86\x03 \x01(\t:1\n\x08sidetext\x12\x1d.google.protobuf.FieldOptions\x18\xd5\x86\x03 \x01(\t:2\n\tmin_value\x12\x1d.google.protobuf.FieldOptions\x18\xd6\x86\x03 \x01(\x01:2\n\tmax_value\x12\x1d.google.protobuf.FieldOptions\x18\xd7\x86\x03 \x01(\x01:4\n\x0bmax_literal\x12\x1d.google.protobuf.FieldOptions\x18\xd8\x86\x03 \x01(\x01:?\n\x04mode\x12\x1d.google.protobuf.FieldOptions\x18\xd9\x86\x03 \x01(\x0e\x32\x10.orca.ConfigMode:3\n\nratio_over\x12\x1d.google.protobuf.FieldOptions\x18\xda\x86\x03 \x01(\t:2\n\tmultiline\x12\x1d.google.protobuf.FieldOptions\x18\xdd\x86\x03 \x01(\x08:3\n\nfull_width\x12\x1d.google.protobuf.FieldOptions\x18\xde\x86\x03 \x01(\x08:/\n\x06height\x12\x1d.google.protobuf.FieldOptions\x18\xdf\x86\x03 \x01(\x05:A\n\x06preset\x12\x1d.google.protobuf.FieldOptions\x18\xdb\x86\x03 \x01(\x0e\x32\x10.orca.PresetType:L\n\x0binvalidates\x12\x1d.google.protobuf.FieldOptions\x18\xdc\x86\x03 \x03(\x0e\x32\x16.orca.InvalidationStep:T\n\x0flist_membership\x12\x1d.google.protobuf.FieldOptions\x18\xe2\x86\x03 \x03(\x0e\x32\x1a.orca.OptionListMembership:4\n\x0blegacy_name\x12\x1d.google.protobuf.FieldOptions\x18\xe0\x86\x03 \x01(\t:4\n\x0bis_nullable\x12\x1d.google.protobuf.FieldOptions\x18\xe1\x86\x03 \x01(\x08:@\n\x08gui_type\x12\x1d.google.protobuf.FieldOptions\x18\xe3\x86\x03 \x01(\x0e\x32\r.orca.GuiType:2\n\tgui_flags\x12\x1d.google.protobuf.FieldOptions\x18\xe4\x86\x03 \x01(\t::\n\x11\x65num_keys_map_ref\x12\x1d.google.protobuf.FieldOptions\x18\xe5\x86\x03 \x01(\t:/\n\x06no_cli\x12\x1d.google.protobuf.FieldOptions\x18\xe6\x86\x03 \x01(\x08:1\n\x08readonly\x12\x1d.google.protobuf.FieldOptions\x18\xe7\x86\x03 \x01(\x08:G\n\x0c\x63o_type_hint\x12\x1d.google.protobuf.FieldOptions\x18\xe8\x86\x03 \x01(\x0e\x32\x10.orca.CoTypeHint:6\n\rdefault_value\x12\x1d.google.protobuf.FieldOptions\x18\xe9\x86\x03 \x01(\t:4\n\x0bhas_default\x12\x1d.google.protobuf.FieldOptions\x18\xec\x86\x03 \x01(\x08:;\n\x12\x65num_value_entries\x12\x1d.google.protobuf.FieldOptions\x18\xea\x86\x03 \x03(\t:;\n\x12\x65num_label_entries\x12\x1d.google.protobuf.FieldOptions\x18\xeb\x86\x03 \x03(\t:1\n\x08tab_type\x12\x1d.google.protobuf.FieldOptions\x18\xed\x86\x03 \x01(\t:1\n\x08tab_page\x12\x1d.google.protobuf.FieldOptions\x18\xee\x86\x03 \x01(\t:5\n\x0ctab_optgroup\x12\x1d.google.protobuf.FieldOptions\x18\xef\x86\x03 \x01(\t:>\n\x13virtual_preset_keys\x12\x1f.google.protobuf.MessageOptions\x18\xe1\xd4\x03 \x03(\tb\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'config_metadata_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_CONFIGMODE']._serialized_start=148 - _globals['_CONFIGMODE']._serialized_end=231 - _globals['_PRESETTYPE']._serialized_start=233 - _globals['_PRESETTYPE']._serialized_end=304 - _globals['_INVALIDATIONSTEP']._serialized_start=307 - _globals['_INVALIDATIONSTEP']._serialized_end=477 - _globals['_OPTIONLISTMEMBERSHIP']._serialized_start=480 - _globals['_OPTIONLISTMEMBERSHIP']._serialized_end=609 - _globals['_COTYPEHINT']._serialized_start=611 - _globals['_COTYPEHINT']._serialized_end=703 - _globals['_GUITYPE']._serialized_start=706 - _globals['_GUITYPE']._serialized_end=856 - _globals['_FLOATORPERCENT']._serialized_start=65 - _globals['_FLOATORPERCENT']._serialized_end=113 - _globals['_POINT2D']._serialized_start=115 - _globals['_POINT2D']._serialized_end=146 -# @@protoc_insertion_point(module_scope) diff --git a/tools/run_codegen.py b/tools/run_codegen.py index b235afeae0..7940d5fab1 100644 --- a/tools/run_codegen.py +++ b/tools/run_codegen.py @@ -6,17 +6,23 @@ Convenience script: runs the codegen pipeline. 2. Generate C++ from descriptors (config_codegen.py) 3. Validate output against original +The toolchain (protoc, protobuf, pyyaml) is resolved by codegen_toolchain.py, so +this script is the single entry point every build script and CI job calls -- no +`pip install` lines needed around it. + Usage: python tools/run_codegen.py # full pipeline python tools/run_codegen.py --validate-only # just validate """ import argparse -import shutil import subprocess import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import codegen_toolchain # noqa: E402 + ROOT = Path(__file__).resolve().parent.parent PROTO_DIR = ROOT / "src" / "PrintConfigs" CODEGEN_OUT = ROOT / "src" / "slic3r" / "GUI" / "generated" @@ -24,22 +30,6 @@ DESC_FILE = ROOT / "config.desc" LAYOUT_YAML = PROTO_DIR / "layout.yaml" -def _ensure_pyyaml(): - """Install pyyaml if not present — needed for tab layout generation.""" - try: - import yaml # noqa: F401 - return True - except ImportError: - print(" Installing pyyaml (required for tab layout generation)...") - result = subprocess.run( - [sys.executable, "-m", "pip", "install", "pyyaml", "-q"], - capture_output=True) - if result.returncode != 0: - print(" ERROR: failed to install pyyaml") - return False - return True - - def run(cmd, **kwargs): print(f" $ {' '.join(str(c) for c in cmd)}") result = subprocess.run(cmd, **kwargs) @@ -49,19 +39,6 @@ def run(cmd, **kwargs): return True -def _protoc_cmd(): - """Return the protoc command list. Prefers standalone protoc, falls back to grpc_tools.""" - if shutil.which("protoc"): - return ["protoc"] - try: - import grpc_tools.protoc # noqa: F401 - return [sys.executable, "-m", "grpc_tools.protoc"] - except ImportError: - pass - print(" ERROR: protoc not found. Install protoc or run: pip install grpcio-tools") - return None - - def step_compile(): print("\n=== Step 1: Compile .proto -> descriptor set ===") proto_files = [f for f in PROTO_DIR.glob("*.proto") if not f.name.endswith("_gen.proto") and f.name != "config_metadata.proto"] @@ -69,7 +46,7 @@ def step_compile(): print(" ERROR: No .proto files found") return False - protoc = _protoc_cmd() + protoc = codegen_toolchain.find_protoc() if protoc is None: return False @@ -82,7 +59,6 @@ def step_compile(): def step_generate(): print("\n=== Step 2: Generate C++ from descriptors + layout.yaml ===") - _ensure_pyyaml() # tab layout generation requires pyyaml return run([sys.executable, str(ROOT / "tools" / "config_codegen.py"), str(DESC_FILE), str(CODEGEN_OUT)]) @@ -106,8 +82,11 @@ def main(): help="Skip validation step (used by cmake build)") args = parser.parse_args() + # Re-execs into a virtualenv with protobuf/pyyaml if this interpreter lacks them. + codegen_toolchain.ensure_python_runtime() + if args.validate_only: - # Compile + lint the protos, then check the committed generated files are current. + # Compile + lint the protos, then check the generated files against PrintConfig.cpp. sys.exit(0 if (step_compile() and step_lint() and step_validate()) else 1) for name, fn in [("Compile", step_compile), ("Generate", step_generate)]: