fix(codegen): drop grpcio-tools, stop committing generated code

The codegen only ever needed a protoc binary, but every entry point installed
grpcio-tools to get one. That drags in the grpcio C extension, which has no
Windows/ARM64 wheel and falls back to building from source there, so the ARM64
job died with "Failed building wheel for grpcio" -> "protoc not found".

tools/codegen_toolchain.py resolves the toolchain instead: protoc from $PROTOC,
PATH, a cache, grpc_tools when already installed, or a pinned checksum-verified
protoc release unpacked into .codegen-tools/; protobuf and pyyaml from the
calling interpreter or a cached virtualenv it re-execs into (distro Pythons
refuse `pip install` under PEP 668). All four build scripts and all three CI
jobs are now just `python tools/run_codegen.py`, with no pip lines around it.

tools/config_metadata_pb2.py was the one generated file checked into git. The
orca.* option extensions are now read out of the descriptor set, which already
carries config_metadata.proto via --include_imports, so nothing is generated
into the tree -- and the protobuf>=6.33.5,<7 CI pin goes away with it, since it
only existed to satisfy gencode's hard ValidateProtobufRuntimeVersion check.
Generated C++ verified byte-identical under upb and the pure-Python protobuf
runtime (what win/arm64 installs), and under both grpc_tools' and standalone
protoc.

Also fixed:
- build_release_macos.sh still passed -DPython3_EXECUTABLE=<codegen venv>,
  pointing the bundled *embed* interpreter at the codegen environment -- the
  same confusion fae4b124 fixed on the CMake side.
- Tab.cpp #includes TabLayout_generated.cpp but had no dependency on
  codegen_config, so an incremental build after a .proto edit could compile it
  while the file was being rewritten. libslic3r already had this guard.
- ConfigCodegen.cmake now probes with `codegen_toolchain.py --check` (which
  never downloads or installs), prefers a host interpreter over the embed one,
  and lets a fresh clone generate at configure time instead of erroring out.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ExPikaPaka
2026-07-28 12:53:05 +02:00
parent fae4b1241b
commit 2f67e46047
16 changed files with 446 additions and 188 deletions

View File

@@ -283,10 +283,8 @@ jobs:
scripts/flatpak/com.orcaslicer.OrcaSlicer.yml
shell: bash
- name: Generate config sources
run: |
python3 -m venv /tmp/codegen_venv
/tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q
/tmp/codegen_venv/bin/python tools/run_codegen.py
# Generated on the host: the flatpak build itself runs offline.
run: python3 tools/run_codegen.py
shell: bash
- uses: flatpak/flatpak-github-actions/flatpak-builder@master
with:

View File

@@ -57,34 +57,18 @@ jobs:
useLocalCache: true # <--= Use the local cache (default is 'false').
useCloudCache: true
- name: Install codegen tools and generate config sources
# grpcio-tools has no Windows/ARM64 wheel for current Python and fails to build its
# grpcio C-extension from source there. The codegen only needs a protoc binary plus the
# protobuf runtime and pyyaml, so install those directly (works on x64 and arm64 Windows;
# the x64 protoc runs under emulation on arm64). protobuf must be >= the version the
# committed config_metadata_pb2.py was generated with (see its runtime-version check).
run: |
pip install "protobuf>=6.33.5,<7" pyyaml
$protocVer = "28.3"
$url = "https://github.com/protocolbuffers/protobuf/releases/download/v$protocVer/protoc-$protocVer-win64.zip"
Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\protoc.zip"
Expand-Archive -Path "$env:RUNNER_TEMP\protoc.zip" -DestinationPath "$env:RUNNER_TEMP\protoc" -Force
$env:PATH = "$env:RUNNER_TEMP\protoc\bin;$env:PATH"
python tools/run_codegen.py
# run_codegen.py resolves its own toolchain (a pinned protoc plus protobuf/pyyaml in a
# cached virtualenv), so nothing has to be pip-installed around it. That is what makes it
# work on windows-11-arm, where `pip install grpcio-tools` has no wheel and fails building
# the grpcio C extension from source.
- name: Generate config sources
if: runner.os == 'Windows'
run: python tools/run_codegen.py
shell: pwsh
- name: Install codegen tools and generate config sources
run: |
if [ "$(uname)" = "Linux" ]; then
pip3 install grpcio-tools pyyaml
python3 tools/run_codegen.py
else
python3 -m venv /tmp/codegen_venv
/tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q
/tmp/codegen_venv/bin/python tools/run_codegen.py
fi
- name: Generate config sources
if: runner.os != 'Windows'
run: python3 tools/run_codegen.py
shell: bash
- name: Install CMake 3.31.x (Windows ARM64)
# windows-11-arm ships CMake 4.x, which removed pre-3.5 policy

3
.gitignore vendored
View File

@@ -46,8 +46,11 @@ test.js
internal_docs/
*.flatpak
/flatpak-repo/
# Config codegen: descriptor set, generated C++ and the downloaded protoc /
# bootstrap virtualenv. All of it is produced by tools/run_codegen.py.
config.desc
src/slic3r/GUI/generated/
.codegen-tools/
# Python bytecode
__pycache__/
*.pyc

View File

@@ -1084,7 +1084,8 @@ endfunction()
# Config codegen — generates src/slic3r/GUI/generated/*.cpp from src/PrintConfigs/*.proto.
# Must run before compiling libslic3r (PrintConfig.cpp #includes the generated files).
# Requires: pip install grpcio-tools OR standalone protoc in PATH.
# The toolchain (protoc, protobuf, pyyaml) is resolved by tools/run_codegen.py itself;
# set PROTOC=<path> to use an installed protoc instead of the pinned download.
include(cmake/modules/ConfigCodegen.cmake)
# libslic3r, OrcaSlicer GUI and the OrcaSlicer executable.

View File

@@ -557,7 +557,8 @@ if [[ -n "${BUILD_ORCA}" ]] || [[ -n "${BUILD_TESTS}" ]] ; then
fi
echo "Generating config sources from proto..."
pip install grpcio-tools -q
# Resolves protoc and its Python packages itself; nothing is installed into the
# system interpreter (distro Pythons refuse that under PEP 668 anyway).
python3 tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; }
print_and_run cmake -S . -B $BUILD_DIR "${CMAKE_C_CXX_COMPILER_CLANG[@]}" "${CMAKE_LLD_LINKER_ARGS[@]}" "${CMAKE_CCACHE_ARGS[@]}" -G "Ninja Multi-Config" \

View File

@@ -241,10 +241,9 @@ function verify_python_runtime() {
function build_slicer() {
echo "Generating config sources from proto..."
python3 -m venv /tmp/codegen_venv
/tmp/codegen_venv/bin/pip install grpcio-tools pyyaml -q
/tmp/codegen_venv/bin/python tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; }
CODEGEN_PYTHON="/tmp/codegen_venv/bin/python3"
# Resolves protoc and its Python packages itself (into .codegen-tools/), so nothing
# is installed into the system interpreter.
python3 tools/run_codegen.py || { echo "ERROR: config codegen failed"; exit 1; }
# iterate over two architectures: x86_64 and arm64
for _ARCH in x86_64 arm64; do
@@ -270,7 +269,6 @@ function build_slicer() {
-DCMAKE_OSX_ARCHITECTURES="${_ARCH}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET="${OSX_DEPLOYMENT_TARGET}" \
-DCMAKE_IGNORE_PREFIX_PATH="${CMAKE_IGNORE_PREFIX_PATH}" \
-DPython3_EXECUTABLE="${CODEGEN_PYTHON}" \
${CMAKE_POLICY_COMPAT}
fi
cmake --build . --config "$BUILD_CONFIG" --target "$SLICER_BUILD_TARGET"

View File

@@ -147,7 +147,8 @@ echo "building Orca Slicer..."
cd %WP%
echo "generating config sources from proto..."
pip install grpcio-tools -q
REM run_codegen.py resolves protoc and its python packages itself (grpcio-tools has
REM no wheel on ARM64 and fails to build there).
python tools/run_codegen.py
if errorlevel 1 (
echo "ERROR: config codegen failed"

View File

@@ -68,7 +68,8 @@ echo "building Orca Slicer..."
cd %WP%
echo "generating config sources from proto..."
pip install grpcio-tools -q
REM run_codegen.py resolves protoc and its python packages itself (grpcio-tools has
REM no wheel on ARM64 and fails to build there).
python tools/run_codegen.py
if errorlevel 1 (
echo "ERROR: config codegen failed"

View File

@@ -2,7 +2,8 @@
#
# Generates C++ source files from protobuf schema definitions.
# Generated files live in src/slic3r/GUI/generated/ and are gitignored.
# Run 'python tools/run_codegen.py' (requires grpcio-tools or protoc) to regenerate.
# Run 'python tools/run_codegen.py' to regenerate; it resolves protoc and the Python
# packages it needs itself (see tools/codegen_toolchain.py).
#
# Targets:
# codegen_config - Custom target to regenerate C++ from .proto files
@@ -21,24 +22,26 @@ set(_generated_marker "${CMAKE_SOURCE_DIR}/src/slic3r/GUI/generated/PrintConfigD
# The main CMakeLists forces Python3_EXECUTABLE to the *bundled embed* interpreter (used for
# the in-app Python plugin runtime). That interpreter has neither protoc nor the protobuf /
# pyyaml packages, and for cross-compiled targets it may not even be the host architecture — so
# it cannot run tools/run_codegen.py. In CI the generated sources are produced by a dedicated
# pre-build "Install codegen tools and generate config sources" step; developers run
# tools/run_codegen.py themselves. Only wire up (re)generation when the interpreter really has
# the toolchain, otherwise fall back to the already-generated files.
# it cannot run tools/run_codegen.py. Prefer a host interpreter, which tools/run_codegen.py can
# bootstrap the toolchain into.
#
# ORCA_CODEGEN_PYTHON lets a build explicitly point at an interpreter that has the tools
# (independent of the embed interpreter). Falls back to Python3_EXECUTABLE.
# ORCA_CODEGEN_PYTHON lets a build explicitly point at the interpreter to use.
if(NOT ORCA_CODEGEN_PYTHON)
set(ORCA_CODEGEN_PYTHON "${Python3_EXECUTABLE}")
find_program(_orca_host_python NAMES python3 python)
if(_orca_host_python)
set(ORCA_CODEGEN_PYTHON "${_orca_host_python}")
else()
set(ORCA_CODEGEN_PYTHON "${Python3_EXECUTABLE}")
endif()
endif()
# --check answers "can this interpreter regenerate right now?" (protobuf + pyyaml importable
# and a protoc already resolvable). It never downloads or installs, so wiring up the build-time
# regeneration below never turns a build into a network operation.
set(_codegen_usable FALSE)
if(ORCA_CODEGEN_PYTHON)
# Needs the protobuf python runtime (config_metadata_pb2) plus a protoc (standalone or
# grpc_tools.protoc). Probe the interpreter for both.
execute_process(
COMMAND ${ORCA_CODEGEN_PYTHON} -c
"import shutil, importlib.util, google.protobuf; import sys; sys.exit(0 if (shutil.which('protoc') or importlib.util.find_spec('grpc_tools')) else 1)"
COMMAND ${ORCA_CODEGEN_PYTHON} "${CMAKE_SOURCE_DIR}/tools/codegen_toolchain.py" --check
RESULT_VARIABLE _codegen_probe
OUTPUT_QUIET ERROR_QUIET
)
@@ -47,27 +50,28 @@ if(ORCA_CODEGEN_PYTHON)
endif()
endif()
# If generated files are missing (fresh clone) and we can run codegen, do it now at configure
# time so a plain cmake configure + build works without a separate pre-build step.
# If generated files are missing (fresh clone), run the codegen now at configure time so a plain
# cmake configure + build works without a separate pre-build step. Unlike the build-time
# regeneration above this is allowed to fetch the toolchain — there is nothing to build without it.
if(NOT EXISTS "${_generated_marker}")
if(_codegen_usable)
set(_codegen_result 1)
if(ORCA_CODEGEN_PYTHON)
message(STATUS "Config codegen: generated files missing — running codegen now...")
execute_process(
COMMAND ${ORCA_CODEGEN_PYTHON} "${CMAKE_SOURCE_DIR}/tools/run_codegen.py" --no-validate
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
RESULT_VARIABLE _codegen_result
)
if(NOT _codegen_result EQUAL 0)
message(FATAL_ERROR "Config codegen failed. Install grpcio-tools: pip install grpcio-tools")
endif()
message(STATUS "Config codegen: generated files created successfully")
else()
message(FATAL_ERROR
"Config codegen: generated files are missing and no usable codegen toolchain was found.\n"
"Generate them first with an interpreter that has protoc + protobuf, e.g.:\n"
" pip install grpcio-tools pyyaml && python tools/run_codegen.py\n"
"or pass -DORCA_CODEGEN_PYTHON=<python-with-tools>.")
endif()
if(NOT _codegen_result EQUAL 0)
message(FATAL_ERROR
"Config codegen: generated files are missing and could not be generated.\n"
"Run it manually with a host Python:\n"
" python tools/run_codegen.py\n"
"or pass -DORCA_CODEGEN_PYTHON=<python>. Offline builds can point at an installed\n"
"protoc with PROTOC=<path>.")
endif()
message(STATUS "Config codegen: generated files created successfully")
endif()
set(CONFIG_PROTO_DIR "${CMAKE_SOURCE_DIR}/src/PrintConfigs")
@@ -101,7 +105,7 @@ set(CONFIG_PROTO_FILES
)
if(_codegen_usable)
# Single command: run_codegen.py handles protoc/grpcio-tools detection internally.
# Single command: run_codegen.py resolves protoc itself.
# Proto files → generated .cpp files. Runs automatically when any .proto changes.
add_custom_command(
OUTPUT ${CONFIG_GENERATED_SOURCES}

View File

@@ -275,9 +275,9 @@ Settings are split into three `.proto` files by preset type. Each setting become
| File | Contents |
|------|----------|
| `src/PrintConfigs/generated/print.proto` | ~477 print/process settings |
| `src/PrintConfigs/generated/filament.proto` | ~103 filament settings |
| `src/PrintConfigs/generated/printer.proto` | ~42 printer settings |
| `src/PrintConfigs/print.proto` | ~477 print/process settings |
| `src/PrintConfigs/filament.proto` | ~103 filament settings |
| `src/PrintConfigs/printer.proto` | ~42 printer settings |
Each file also carries message-level `virtual_preset_keys` declarations (see §5.2.3).
@@ -364,13 +364,19 @@ The codegen reads these and merges them (deduplicated, sorted) with the field-de
```cmake
add_custom_command(
OUTPUT ${GENERATED_SOURCES}
COMMAND protoc --descriptor_set_out=config.desc src/PrintConfigs/generated/*.proto
COMMAND python3 tools/config_codegen.py config.desc ${GENERATED_DIR}
DEPENDS src/PrintConfigs/generated/*.proto tools/config_codegen.py
COMMAND ${ORCA_CODEGEN_PYTHON} tools/run_codegen.py --no-validate
DEPENDS src/PrintConfigs/*.proto src/PrintConfigs/layout.yaml tools/config_codegen.py
)
```
Generated files are checked into the repo (not gitignored) so builds work without `protoc`. CI validates that committed generated files match what the generator produces.
Generated files are **not** checked into the repo `src/slic3r/GUI/generated/` and `config.desc` are gitignored, and every build produces them. Each build script and CI job runs `tools/run_codegen.py` before configuring; a fresh clone that goes straight to `cmake` gets them generated at configure time.
`tools/run_codegen.py` resolves its own toolchain (`tools/codegen_toolchain.py`), so no caller has to install anything:
- **protoc** — `$PROTOC`, then `PATH`, then a cached copy, then a pinned checksum-verified release downloaded into `.codegen-tools/` (also gitignored)
- **protobuf + pyyaml** — the calling interpreter if it has them, otherwise a virtualenv under `.codegen-tools/` that the script re-execs into
Set `PROTOC=<path>` to build offline or against a distro protoc.
### 4.5 Provider Customization
@@ -431,7 +437,7 @@ Custom options get field numbers > 1000 to avoid conflicts.
1. Add a field to the appropriate `.proto` file (`print.proto`, `filament.proto`, or `printer.proto`) with all relevant annotations
2. Run `python tools/run_codegen.py`
3. Commit the `.proto` change and the updated generated files together
3. Commit the `.proto` change the generated files are gitignored build output, never committed
### Adding a virtual preset key
@@ -461,26 +467,25 @@ python tools/annotate_protos.py [--dry-run]
src/PrintConfigs/
├── config_metadata.proto # Custom field/message option extensions
├── layout.yaml # UI tab/page/group structure (Tab.cpp layout)
── generated/
├── print.proto # ~477 print/process settings
├── filament.proto # ~103 filament settings
└── printer.proto # ~42 printer/machine settings
── print.proto # ~477 print/process settings
├── filament.proto # ~103 filament settings
└── printer.proto # ~42 printer/machine settings
tools/
├── parse_printconfig.py # Bootstrap: PrintConfig.cpp → .proto
├── run_codegen.py # Full pipeline script — the entry point everything calls
├── codegen_toolchain.py # Resolves protoc / protobuf / pyyaml
├── config_codegen.py # Proto descriptor → C++ codegen
├── validate_codegen.py # Generated vs original validation
── run_codegen.py # Full pipeline script
├── annotate_protos.py # Inject (invalidates)/(list_membership) from C++
├── move_proto_fields.py # Utility: move fields between proto files
└── config_metadata_pb2.py # Generated Python bindings for extensions
├── config_metadata.py # Reads the orca.* option extensions from the descriptor set
── validate_codegen.py # Generated vs original validation
codegen/
── generated/
├── PrintConfigDef_generated.cpp # init_fff_params() body — #included by PrintConfig.cpp
├── Preset_options_generated.cpp # s_Preset_*_options — #included by Preset.cpp
├── Invalidation_generated.cpp # s_print_steps_map + s_object_steps_map — #included by Print.cpp
└── OptionKeys_generated.cpp # s_extruder_option_keys, s_filament_option_keys
src/slic3r/GUI/generated/ # gitignored — regenerated by every build
── PrintConfigDef_generated.cpp # init_fff_params() body — #included by PrintConfig.cpp
├── Preset_options_generated.cpp # s_Preset_*_options — #included by Preset.cpp
├── Invalidation_generated.cpp # s_print_steps_map + s_object_steps_map — #included by Print.cpp
├── OptionKeys_generated.cpp # s_extruder_option_keys, s_filament_option_keys
└── TabLayout_generated.cpp # Tab page/group layout from layout.yaml
.codegen-tools/ # gitignored — downloaded protoc + bootstrap virtualenv
cmake/modules/
└── ConfigCodegen.cmake # CMake integration (build-time regeneration)

View File

@@ -831,6 +831,16 @@ else ()
endif ()
target_include_directories(libslic3r_gui PRIVATE Utils ${CMAKE_CURRENT_BINARY_DIR})
# Tab.cpp #includes GUI/generated/TabLayout_generated.cpp, so it must not be compiled
# while the codegen is rewriting it (same contract as libslic3r's generated includes).
if(TARGET codegen_config)
add_dependencies(libslic3r_gui codegen_config)
set_source_files_properties(
"${CMAKE_CURRENT_SOURCE_DIR}/GUI/Tab.cpp"
PROPERTIES OBJECT_DEPENDS "${CONFIG_GENERATED_SOURCES}"
)
endif()
if (WIN32)
target_include_directories(libslic3r_gui SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/WebView2/include)
target_link_libraries(libslic3r_gui Advapi32)

260
tools/codegen_toolchain.py Normal file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""
Toolchain resolution for the config codegen.
The codegen needs exactly three things: a protoc binary, the protobuf Python
runtime and pyyaml. Every entry point used to install `grpcio-tools` to get
protoc, which drags in the grpcio C extension: it has no Windows/ARM64 wheel and
falls back to building from source there, which is what broke the ARM64 build
("Failed building wheel for grpcio" -> "protoc not found"). Nothing in the
codegen uses gRPC.
Resolution order, so callers only ever run `python tools/run_codegen.py`:
protoc $PROTOC -> PATH -> cached download -> grpc_tools (if installed) ->
pinned, checksum-verified protoc release downloaded into
.codegen-tools/ (gitignored)
runtime the current interpreter, else a cached virtualenv under
.codegen-tools/venv that the entry point re-execs into
Set PROTOC=/path/to/protoc (or put protoc on PATH) to build offline.
"""
import hashlib
import importlib.util
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import urllib.request
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CACHE_DIR = ROOT / ".codegen-tools"
# Pinned so every machine generates with the same compiler, and checksummed
# because we execute what we download. To bump: change the version and refresh
# every hash from https://github.com/protocolbuffers/protobuf/releases/tag/v<ver>
PROTOC_VERSION = "28.3"
PROTOC_ARCHIVE_SHA256 = {
"win64": "ce64f49bdeddef49ce4bd313a8f59bcf92fcf67b5831efbf66170386d2e66948",
"linux-x86_64": "0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f",
"linux-aarch_64": "1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a",
"osx-universal_binary": "52df502b263da20f3311b23b5c6553d10cc25c6ebb85df381d80a2806b6a698b",
}
# pip name -> import name
PYTHON_PACKAGES = {"protobuf": "google.protobuf", "pyyaml": "yaml"}
# Guards against an endless re-exec loop if the virtualenv still can't import.
_BOOTSTRAP_ENV = "ORCA_CODEGEN_BOOTSTRAPPED"
_PROTOC_EXE = "protoc.exe" if os.name == "nt" else "protoc"
def _protoc_dir():
return CACHE_DIR / f"protoc-{PROTOC_VERSION}"
def _archive_key():
"""Release asset for this host, or None if protobuf ships no build for it."""
if sys.platform == "win32":
# There is no win/arm64 release; the x64 build runs under Windows'
# emulation, which is how the ARM64 CI job gets a protoc.
return "win64"
if sys.platform == "darwin":
return "osx-universal_binary"
if sys.platform.startswith("linux"):
machine = platform.machine().lower()
if machine in ("x86_64", "amd64"):
return "linux-x86_64"
if machine in ("aarch64", "arm64"):
return "linux-aarch_64"
return None
def _download_protoc():
"""Fetch and unpack the pinned protoc. Returns the binary path, or None."""
key = _archive_key()
if key is None:
print(f" ERROR: no pinned protoc release for {sys.platform}/{platform.machine()}.")
print(" Install protoc from your package manager and re-run, or set PROTOC=<path>.")
return None
url = (f"https://github.com/protocolbuffers/protobuf/releases/download/"
f"v{PROTOC_VERSION}/protoc-{PROTOC_VERSION}-{key}.zip")
print(f" Downloading protoc {PROTOC_VERSION} ({key})...")
CACHE_DIR.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=CACHE_DIR) as tmp:
archive = Path(tmp) / "protoc.zip"
try:
with urllib.request.urlopen(url, timeout=120) as response:
archive.write_bytes(response.read())
except OSError as exc:
print(f" ERROR: download failed: {exc}")
print(" Install protoc manually and re-run, or set PROTOC=<path>.")
return None
actual = hashlib.sha256(archive.read_bytes()).hexdigest()
expected = PROTOC_ARCHIVE_SHA256[key]
if actual != expected:
print(f" ERROR: protoc archive checksum mismatch for {key} {PROTOC_VERSION}:")
print(f" expected {expected}")
print(f" actual {actual}")
return None
# Unpack the whole archive, not just bin/: protoc resolves the well-known
# imports (google/protobuf/descriptor.proto) from ../include next to it.
staging = Path(tmp) / "unpacked"
with zipfile.ZipFile(archive) as zf:
zf.extractall(staging)
target = _protoc_dir()
if target.exists():
shutil.rmtree(target)
# Move into place only once complete, so an interrupted run never leaves
# a half-extracted toolchain that later runs would happily use.
shutil.move(str(staging), str(target))
binary = target / "bin" / _PROTOC_EXE
if not binary.exists():
print(f" ERROR: protoc archive did not contain bin/{_PROTOC_EXE}")
return None
binary.chmod(binary.stat().st_mode | 0o755)
return binary
def find_protoc(allow_download=True):
"""Return the protoc command as a list, or None if it can't be resolved."""
override = os.environ.get("PROTOC")
if override:
return [override]
on_path = shutil.which("protoc")
if on_path:
return [on_path]
cached = _protoc_dir() / "bin" / _PROTOC_EXE
if cached.exists():
return [str(cached)]
# Honour a pre-existing grpcio-tools install rather than downloading.
if importlib.util.find_spec("grpc_tools") is not None:
return [sys.executable, "-m", "grpc_tools.protoc"]
if allow_download:
binary = _download_protoc()
if binary is not None:
return [str(binary)]
return None
def missing_packages():
"""pip names of the required Python packages this interpreter can't import."""
missing = []
for pip_name, module in PYTHON_PACKAGES.items():
try:
found = importlib.util.find_spec(module) is not None
except (ImportError, ValueError):
found = False
if not found:
missing.append(pip_name)
return missing
def _venv_python():
venv_dir = CACHE_DIR / "venv"
if os.name == "nt":
return venv_dir / "Scripts" / "python.exe"
return venv_dir / "bin" / "python"
def bootstrap_python():
"""The cached virtualenv interpreter, if it exists and has the packages."""
python = _venv_python()
if not python.exists():
return None
probe = subprocess.run([str(python), "-c", "import google.protobuf, yaml"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return python if probe.returncode == 0 else None
def _ensure_venv(missing):
"""Create/reuse .codegen-tools/venv with the required packages installed."""
python = _venv_python()
if not python.exists():
print(f" Creating codegen virtualenv in {python.parent.parent}...")
if subprocess.run([sys.executable, "-m", "venv", str(python.parent.parent)]).returncode != 0 \
or not python.exists():
return None
print(f" Installing {', '.join(missing)} into the codegen virtualenv...")
result = subprocess.run([str(python), "-m", "pip", "install", "--quiet",
"--disable-pip-version-check", *missing])
return python if result.returncode == 0 else None
def ensure_python_runtime():
"""
Guarantee protobuf + pyyaml are importable.
If they aren't, re-run the calling script in a cached virtualenv that has
them and exit with its status. Keeping the packages out of the caller's
interpreter is what lets the build scripts work on distros that refuse
`pip install` into a system Python (PEP 668).
"""
missing = missing_packages()
if not missing:
return
if os.environ.get(_BOOTSTRAP_ENV):
# We are already inside the bootstrapped environment: installing again
# would just fail the same way.
print(f" ERROR: {', '.join(missing)} still missing after bootstrap.")
sys.exit(1)
# Reuse a complete virtualenv as-is: builds call this every time, and pip
# would otherwise hit the network on each one.
python = bootstrap_python() or _ensure_venv(missing)
if python is None:
print(f" ERROR: could not provide {', '.join(missing)}.")
print(f" Install them for this interpreter: {sys.executable} -m pip install "
f"{' '.join(missing)}")
sys.exit(1)
env = dict(os.environ, **{_BOOTSTRAP_ENV: "1"})
sys.exit(subprocess.run([str(python), *sys.argv], env=env).returncode)
def toolchain_ready():
"""
True if the codegen can run without downloading or installing anything --
either from this interpreter or by re-execing into an existing bootstrap
virtualenv. This is what decides whether a build regenerates on proto edits.
"""
if missing_packages() and bootstrap_python() is None:
return False
return find_protoc(allow_download=False) is not None
def main():
# --check is what ConfigCodegen.cmake probes with: it answers whether this
# interpreter can regenerate during a build, and must not download, install
# or create anything while doing so.
if "--check" in sys.argv[1:]:
return 0 if toolchain_ready() else 1
missing = missing_packages()
print(f"python: {sys.executable}")
print(f"packages: {'all present' if not missing else 'missing ' + ', '.join(missing)}")
protoc = find_protoc()
print(f"protoc: {' '.join(protoc) if protoc else 'NOT FOUND'}")
return 0 if protoc else 1
if __name__ == "__main__":
sys.exit(main())

View File

@@ -10,11 +10,10 @@ Usage:
protoc --proto_path=src/PrintConfigs --descriptor_set_out=config.desc \
--include_imports src/PrintConfigs/*.proto
# Step 2: Generate Python bindings (one-time, or when config_metadata.proto changes)
protoc --proto_path=src/PrintConfigs --python_out=tools/ config_metadata.proto
# Step 2: Run codegen
python tools/config_codegen.py config.desc src/slic3r/GUI/generated/
# Step 3: Run codegen
python tools/config_codegen.py config.desc codegen/generated/
(tools/run_codegen.py does both, and resolves protoc for you)
Outputs:
- PrintConfigDef_generated.cpp (init_fff_params body)
@@ -29,20 +28,22 @@ import re
import argparse
from pathlib import Path
# Add tools/ to path so we can import generated config_metadata_pb2
# Add tools/ to path so we can import the sibling helper modules
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
try:
from google.protobuf import descriptor_pb2
# Import the generated bindings - this registers extensions globally
import config_metadata_pb2 as meta_pb2
from config_metadata import load_descriptor_set
except ImportError as e:
print(f"ERROR: {e}")
print("Ensure google-protobuf is installed: pip install protobuf")
print("And that config_metadata_pb2.py exists in tools/")
print("Generate it with: protoc --proto_path=src/PrintConfigs --python_out=tools/ config_metadata.proto")
print("Ensure protobuf is installed: pip install protobuf")
print("Or just run the pipeline, which bootstraps it: python tools/run_codegen.py")
sys.exit(1)
# The orca.* option extensions and enum constants, bound by main() from the
# descriptor set (see config_metadata.py for why they aren't imported).
meta_pb2 = None
# Proto FieldDescriptorProto.Type enum values
TYPE_DOUBLE = 1
@@ -179,9 +180,8 @@ def parse_field_options(field_desc_proto):
Re-parse FieldOptions from a FieldDescriptorProto with extensions registered.
This is needed because the FileDescriptorSet parser doesn't know about our
custom extensions, so they end up as unknown fields. Re-parsing with the
extensions registered (via config_metadata_pb2 import) resolves them.
extensions registered (see config_metadata.load_descriptor_set) resolves them.
"""
from google.protobuf import descriptor_pb2
opts = field_desc_proto.options
if not opts.ByteSize():
return descriptor_pb2.FieldOptions()
@@ -981,11 +981,8 @@ def main():
print(f"ERROR: Descriptor file not found: {desc_path}")
sys.exit(1)
with open(desc_path, 'rb') as f:
raw = f.read()
file_descriptor_set = descriptor_pb2.FileDescriptorSet()
file_descriptor_set.ParseFromString(raw)
global meta_pb2
file_descriptor_set, meta_pb2 = load_descriptor_set(desc_path)
print(f"Loaded {len(file_descriptor_set.file)} proto files")
for fd in file_descriptor_set.file:

67
tools/config_metadata.py Normal file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""
Access to the orca.* option extensions declared in config_metadata.proto.
The extensions are read straight out of the compiled descriptor set: protoc runs
with --include_imports, so config_metadata.proto travels inside the .desc file
that the codegen already consumes. Registering it in the default descriptor pool
before the descriptor set is parsed is what makes the custom options resolve
instead of landing in unknown fields.
Doing it this way keeps generated code out of git. The alternative -- a checked-in
config_metadata_pb2.py -- also pinned the protobuf runtime to whichever protoc
produced it, because gencode embeds a hard ValidateProtobufRuntimeVersion() check.
"""
from google.protobuf import descriptor_pb2, descriptor_pool
METADATA_PROTO = "config_metadata.proto"
class Metadata:
"""
Stand-in for the generated config_metadata_pb2 module.
Exposes each orca extension as an attribute holding its FieldDescriptor
(usable as `options.Extensions[meta.label]`) and each enum value as an int
constant (`meta.MODE_SIMPLE`, `meta.STEP_SLICE`, ...), matching how the
generated module was used.
"""
def __init__(self, file_descriptor):
for name, extension in file_descriptor.extensions_by_name.items():
setattr(self, name, extension)
for enum in file_descriptor.enum_types_by_name.values():
for value in enum.values:
setattr(self, value.name, value.number)
def load_descriptor_set(path):
"""
Read a protoc descriptor set and return (FileDescriptorSet, Metadata).
The file is parsed twice on purpose: the first pass only locates the embedded
config_metadata.proto so its extensions can be registered, the second one
parses with those extensions known.
"""
with open(path, 'rb') as f:
raw = f.read()
probe = descriptor_pb2.FileDescriptorSet()
probe.ParseFromString(raw)
metadata_file = next((f for f in probe.file if f.name == METADATA_PROTO), None)
if metadata_file is None:
raise RuntimeError(
f"{path} does not contain {METADATA_PROTO} -- protoc must be run with "
"--include_imports")
pool = descriptor_pool.Default()
try:
file_descriptor = pool.FindFileByName(METADATA_PROTO)
except KeyError:
pool.Add(metadata_file)
file_descriptor = pool.FindFileByName(METADATA_PROTO)
descriptor_set = descriptor_pb2.FileDescriptorSet()
descriptor_set.ParseFromString(raw)
return descriptor_set, Metadata(file_descriptor)

View File

@@ -1,51 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: config_metadata.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
6,
33,
5,
'',
'config_metadata.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x63onfig_metadata.proto\x12\x04orca\x1a google/protobuf/descriptor.proto\"0\n\x0e\x46loatOrPercent\x12\r\n\x05value\x18\x01 \x01(\x01\x12\x0f\n\x07percent\x18\x02 \x01(\x08\"\x1f\n\x07Point2D\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01*S\n\nConfigMode\x12\x0f\n\x0bMODE_SIMPLE\x10\x00\x12\x11\n\rMODE_ADVANCED\x10\x01\x12\x10\n\x0cMODE_DEVELOP\x10\x02\x12\x0f\n\x0bMODE_EXPERT\x10\x03*G\n\nPresetType\x12\x10\n\x0cPRESET_PRINT\x10\x00\x12\x13\n\x0fPRESET_FILAMENT\x10\x01\x12\x12\n\x0ePRESET_PRINTER\x10\x02*\xaa\x01\n\x10InvalidationStep\x12\x15\n\x11STEP_GCODE_EXPORT\x10\x00\x12\x13\n\x0fSTEP_SKIRT_BRIM\x10\x01\x12\x13\n\x0fSTEP_WIPE_TOWER\x10\x02\x12\x0e\n\nSTEP_SLICE\x10\x03\x12\x13\n\x0fSTEP_PERIMETERS\x10\x04\x12\x0f\n\x0bSTEP_INFILL\x10\x05\x12\x10\n\x0cSTEP_SUPPORT\x10\x06\x12\r\n\tSTEP_NONE\x10\x07*\x81\x01\n\x14OptionListMembership\x12\r\n\tLIST_NONE\x10\x00\x12\x1d\n\x19LIST_EXTRUDER_OPTION_KEYS\x10\x01\x12\x1d\n\x19LIST_FILAMENT_OPTION_KEYS\x10\x02\x12\x1c\n\x18LIST_VARIANT_OPTION_KEYS\x10\x03*\\\n\nCoTypeHint\x12\x16\n\x12\x43O_TYPE_HINT_UNSET\x10\x00\x12\r\n\tcoPercent\x10\x01\x12\x0e\n\ncoPercents\x10\x02\x12\n\n\x06\x63oEnum\x10\x03\x12\x0b\n\x07\x63oEnums\x10\x04*\x96\x01\n\x07GuiType\x12\x12\n\x0eGUI_TYPE_UNSET\x10\x00\x12\x0f\n\x0bi_enum_open\x10\x01\x12\x0f\n\x0b\x66_enum_open\x10\x02\x12\t\n\x05\x63olor\x10\x03\x12\x0f\n\x0bselect_open\x10\x04\x12\n\n\x06slider\x10\x05\x12\n\n\x06legend\x10\x06\x12\x0e\n\none_string\x10\x07\x12\x11\n\rplugin_picker\x10\x08:.\n\x05label\x12\x1d.google.protobuf.FieldOptions\x18\xd1\x86\x03 \x01(\t:3\n\nfull_label\x12\x1d.google.protobuf.FieldOptions\x18\xd2\x86\x03 \x01(\t:0\n\x07tooltip\x12\x1d.google.protobuf.FieldOptions\x18\xd3\x86\x03 \x01(\t:1\n\x08\x63\x61tegory\x12\x1d.google.protobuf.FieldOptions\x18\xd4\x86\x03 \x01(\t:1\n\x08sidetext\x12\x1d.google.protobuf.FieldOptions\x18\xd5\x86\x03 \x01(\t:2\n\tmin_value\x12\x1d.google.protobuf.FieldOptions\x18\xd6\x86\x03 \x01(\x01:2\n\tmax_value\x12\x1d.google.protobuf.FieldOptions\x18\xd7\x86\x03 \x01(\x01:4\n\x0bmax_literal\x12\x1d.google.protobuf.FieldOptions\x18\xd8\x86\x03 \x01(\x01:?\n\x04mode\x12\x1d.google.protobuf.FieldOptions\x18\xd9\x86\x03 \x01(\x0e\x32\x10.orca.ConfigMode:3\n\nratio_over\x12\x1d.google.protobuf.FieldOptions\x18\xda\x86\x03 \x01(\t:2\n\tmultiline\x12\x1d.google.protobuf.FieldOptions\x18\xdd\x86\x03 \x01(\x08:3\n\nfull_width\x12\x1d.google.protobuf.FieldOptions\x18\xde\x86\x03 \x01(\x08:/\n\x06height\x12\x1d.google.protobuf.FieldOptions\x18\xdf\x86\x03 \x01(\x05:A\n\x06preset\x12\x1d.google.protobuf.FieldOptions\x18\xdb\x86\x03 \x01(\x0e\x32\x10.orca.PresetType:L\n\x0binvalidates\x12\x1d.google.protobuf.FieldOptions\x18\xdc\x86\x03 \x03(\x0e\x32\x16.orca.InvalidationStep:T\n\x0flist_membership\x12\x1d.google.protobuf.FieldOptions\x18\xe2\x86\x03 \x03(\x0e\x32\x1a.orca.OptionListMembership:4\n\x0blegacy_name\x12\x1d.google.protobuf.FieldOptions\x18\xe0\x86\x03 \x01(\t:4\n\x0bis_nullable\x12\x1d.google.protobuf.FieldOptions\x18\xe1\x86\x03 \x01(\x08:@\n\x08gui_type\x12\x1d.google.protobuf.FieldOptions\x18\xe3\x86\x03 \x01(\x0e\x32\r.orca.GuiType:2\n\tgui_flags\x12\x1d.google.protobuf.FieldOptions\x18\xe4\x86\x03 \x01(\t::\n\x11\x65num_keys_map_ref\x12\x1d.google.protobuf.FieldOptions\x18\xe5\x86\x03 \x01(\t:/\n\x06no_cli\x12\x1d.google.protobuf.FieldOptions\x18\xe6\x86\x03 \x01(\x08:1\n\x08readonly\x12\x1d.google.protobuf.FieldOptions\x18\xe7\x86\x03 \x01(\x08:G\n\x0c\x63o_type_hint\x12\x1d.google.protobuf.FieldOptions\x18\xe8\x86\x03 \x01(\x0e\x32\x10.orca.CoTypeHint:6\n\rdefault_value\x12\x1d.google.protobuf.FieldOptions\x18\xe9\x86\x03 \x01(\t:4\n\x0bhas_default\x12\x1d.google.protobuf.FieldOptions\x18\xec\x86\x03 \x01(\x08:;\n\x12\x65num_value_entries\x12\x1d.google.protobuf.FieldOptions\x18\xea\x86\x03 \x03(\t:;\n\x12\x65num_label_entries\x12\x1d.google.protobuf.FieldOptions\x18\xeb\x86\x03 \x03(\t:1\n\x08tab_type\x12\x1d.google.protobuf.FieldOptions\x18\xed\x86\x03 \x01(\t:1\n\x08tab_page\x12\x1d.google.protobuf.FieldOptions\x18\xee\x86\x03 \x01(\t:5\n\x0ctab_optgroup\x12\x1d.google.protobuf.FieldOptions\x18\xef\x86\x03 \x01(\t:>\n\x13virtual_preset_keys\x12\x1f.google.protobuf.MessageOptions\x18\xe1\xd4\x03 \x03(\tb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'config_metadata_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
DESCRIPTOR._loaded_options = None
_globals['_CONFIGMODE']._serialized_start=148
_globals['_CONFIGMODE']._serialized_end=231
_globals['_PRESETTYPE']._serialized_start=233
_globals['_PRESETTYPE']._serialized_end=304
_globals['_INVALIDATIONSTEP']._serialized_start=307
_globals['_INVALIDATIONSTEP']._serialized_end=477
_globals['_OPTIONLISTMEMBERSHIP']._serialized_start=480
_globals['_OPTIONLISTMEMBERSHIP']._serialized_end=609
_globals['_COTYPEHINT']._serialized_start=611
_globals['_COTYPEHINT']._serialized_end=703
_globals['_GUITYPE']._serialized_start=706
_globals['_GUITYPE']._serialized_end=856
_globals['_FLOATORPERCENT']._serialized_start=65
_globals['_FLOATORPERCENT']._serialized_end=113
_globals['_POINT2D']._serialized_start=115
_globals['_POINT2D']._serialized_end=146
# @@protoc_insertion_point(module_scope)

View File

@@ -6,17 +6,23 @@ Convenience script: runs the codegen pipeline.
2. Generate C++ from descriptors (config_codegen.py)
3. Validate output against original
The toolchain (protoc, protobuf, pyyaml) is resolved by codegen_toolchain.py, so
this script is the single entry point every build script and CI job calls -- no
`pip install` lines needed around it.
Usage:
python tools/run_codegen.py # full pipeline
python tools/run_codegen.py --validate-only # just validate
"""
import argparse
import shutil
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import codegen_toolchain # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PROTO_DIR = ROOT / "src" / "PrintConfigs"
CODEGEN_OUT = ROOT / "src" / "slic3r" / "GUI" / "generated"
@@ -24,22 +30,6 @@ DESC_FILE = ROOT / "config.desc"
LAYOUT_YAML = PROTO_DIR / "layout.yaml"
def _ensure_pyyaml():
"""Install pyyaml if not present — needed for tab layout generation."""
try:
import yaml # noqa: F401
return True
except ImportError:
print(" Installing pyyaml (required for tab layout generation)...")
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "pyyaml", "-q"],
capture_output=True)
if result.returncode != 0:
print(" ERROR: failed to install pyyaml")
return False
return True
def run(cmd, **kwargs):
print(f" $ {' '.join(str(c) for c in cmd)}")
result = subprocess.run(cmd, **kwargs)
@@ -49,19 +39,6 @@ def run(cmd, **kwargs):
return True
def _protoc_cmd():
"""Return the protoc command list. Prefers standalone protoc, falls back to grpc_tools."""
if shutil.which("protoc"):
return ["protoc"]
try:
import grpc_tools.protoc # noqa: F401
return [sys.executable, "-m", "grpc_tools.protoc"]
except ImportError:
pass
print(" ERROR: protoc not found. Install protoc or run: pip install grpcio-tools")
return None
def step_compile():
print("\n=== Step 1: Compile .proto -> descriptor set ===")
proto_files = [f for f in PROTO_DIR.glob("*.proto") if not f.name.endswith("_gen.proto") and f.name != "config_metadata.proto"]
@@ -69,7 +46,7 @@ def step_compile():
print(" ERROR: No .proto files found")
return False
protoc = _protoc_cmd()
protoc = codegen_toolchain.find_protoc()
if protoc is None:
return False
@@ -82,7 +59,6 @@ def step_compile():
def step_generate():
print("\n=== Step 2: Generate C++ from descriptors + layout.yaml ===")
_ensure_pyyaml() # tab layout generation requires pyyaml
return run([sys.executable, str(ROOT / "tools" / "config_codegen.py"),
str(DESC_FILE), str(CODEGEN_OUT)])
@@ -106,8 +82,11 @@ def main():
help="Skip validation step (used by cmake build)")
args = parser.parse_args()
# Re-execs into a virtualenv with protobuf/pyyaml if this interpreter lacks them.
codegen_toolchain.ensure_python_runtime()
if args.validate_only:
# Compile + lint the protos, then check the committed generated files are current.
# Compile + lint the protos, then check the generated files against PrintConfig.cpp.
sys.exit(0 if (step_compile() and step_lint() and step_validate()) else 1)
for name, fn in [("Compile", step_compile), ("Generate", step_generate)]: