fix(codegen): drop grpcio-tools, stop committing generated code

The codegen only ever needed a protoc binary, but every entry point installed
grpcio-tools to get one. That drags in the grpcio C extension, which has no
Windows/ARM64 wheel and falls back to building from source there, so the ARM64
job died with "Failed building wheel for grpcio" -> "protoc not found".

tools/codegen_toolchain.py resolves the toolchain instead: protoc from $PROTOC,
PATH, a cache, grpc_tools when already installed, or a pinned checksum-verified
protoc release unpacked into .codegen-tools/; protobuf and pyyaml from the
calling interpreter or a cached virtualenv it re-execs into (distro Pythons
refuse `pip install` under PEP 668). All four build scripts and all three CI
jobs are now just `python tools/run_codegen.py`, with no pip lines around it.

tools/config_metadata_pb2.py was the one generated file checked into git. The
orca.* option extensions are now read out of the descriptor set, which already
carries config_metadata.proto via --include_imports, so nothing is generated
into the tree -- and the protobuf>=6.33.5,<7 CI pin goes away with it, since it
only existed to satisfy gencode's hard ValidateProtobufRuntimeVersion check.
Generated C++ verified byte-identical under upb and the pure-Python protobuf
runtime (what win/arm64 installs), and under both grpc_tools' and standalone
protoc.

Also fixed:
- build_release_macos.sh still passed -DPython3_EXECUTABLE=<codegen venv>,
  pointing the bundled *embed* interpreter at the codegen environment -- the
  same confusion fae4b124 fixed on the CMake side.
- Tab.cpp #includes TabLayout_generated.cpp but had no dependency on
  codegen_config, so an incremental build after a .proto edit could compile it
  while the file was being rewritten. libslic3r already had this guard.
- ConfigCodegen.cmake now probes with `codegen_toolchain.py --check` (which
  never downloads or installs), prefers a host interpreter over the embed one,
  and lets a fresh clone generate at configure time instead of erroring out.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ExPikaPaka
2026-07-28 12:53:05 +02:00
parent fae4b1241b
commit 2f67e46047
16 changed files with 446 additions and 188 deletions

260
tools/codegen_toolchain.py Normal file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""
Toolchain resolution for the config codegen.
The codegen needs exactly three things: a protoc binary, the protobuf Python
runtime and pyyaml. Every entry point used to install `grpcio-tools` to get
protoc, which drags in the grpcio C extension: it has no Windows/ARM64 wheel and
falls back to building from source there, which is what broke the ARM64 build
("Failed building wheel for grpcio" -> "protoc not found"). Nothing in the
codegen uses gRPC.
Resolution order, so callers only ever run `python tools/run_codegen.py`:
protoc $PROTOC -> PATH -> cached download -> grpc_tools (if installed) ->
pinned, checksum-verified protoc release downloaded into
.codegen-tools/ (gitignored)
runtime the current interpreter, else a cached virtualenv under
.codegen-tools/venv that the entry point re-execs into
Set PROTOC=/path/to/protoc (or put protoc on PATH) to build offline.
"""
import hashlib
import importlib.util
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import urllib.request
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CACHE_DIR = ROOT / ".codegen-tools"
# Pinned so every machine generates with the same compiler, and checksummed
# because we execute what we download. To bump: change the version and refresh
# every hash from https://github.com/protocolbuffers/protobuf/releases/tag/v<ver>
PROTOC_VERSION = "28.3"
PROTOC_ARCHIVE_SHA256 = {
"win64": "ce64f49bdeddef49ce4bd313a8f59bcf92fcf67b5831efbf66170386d2e66948",
"linux-x86_64": "0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f",
"linux-aarch_64": "1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a",
"osx-universal_binary": "52df502b263da20f3311b23b5c6553d10cc25c6ebb85df381d80a2806b6a698b",
}
# pip name -> import name
PYTHON_PACKAGES = {"protobuf": "google.protobuf", "pyyaml": "yaml"}
# Guards against an endless re-exec loop if the virtualenv still can't import.
_BOOTSTRAP_ENV = "ORCA_CODEGEN_BOOTSTRAPPED"
_PROTOC_EXE = "protoc.exe" if os.name == "nt" else "protoc"
def _protoc_dir():
return CACHE_DIR / f"protoc-{PROTOC_VERSION}"
def _archive_key():
"""Release asset for this host, or None if protobuf ships no build for it."""
if sys.platform == "win32":
# There is no win/arm64 release; the x64 build runs under Windows'
# emulation, which is how the ARM64 CI job gets a protoc.
return "win64"
if sys.platform == "darwin":
return "osx-universal_binary"
if sys.platform.startswith("linux"):
machine = platform.machine().lower()
if machine in ("x86_64", "amd64"):
return "linux-x86_64"
if machine in ("aarch64", "arm64"):
return "linux-aarch_64"
return None
def _download_protoc():
"""Fetch and unpack the pinned protoc. Returns the binary path, or None."""
key = _archive_key()
if key is None:
print(f" ERROR: no pinned protoc release for {sys.platform}/{platform.machine()}.")
print(" Install protoc from your package manager and re-run, or set PROTOC=<path>.")
return None
url = (f"https://github.com/protocolbuffers/protobuf/releases/download/"
f"v{PROTOC_VERSION}/protoc-{PROTOC_VERSION}-{key}.zip")
print(f" Downloading protoc {PROTOC_VERSION} ({key})...")
CACHE_DIR.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=CACHE_DIR) as tmp:
archive = Path(tmp) / "protoc.zip"
try:
with urllib.request.urlopen(url, timeout=120) as response:
archive.write_bytes(response.read())
except OSError as exc:
print(f" ERROR: download failed: {exc}")
print(" Install protoc manually and re-run, or set PROTOC=<path>.")
return None
actual = hashlib.sha256(archive.read_bytes()).hexdigest()
expected = PROTOC_ARCHIVE_SHA256[key]
if actual != expected:
print(f" ERROR: protoc archive checksum mismatch for {key} {PROTOC_VERSION}:")
print(f" expected {expected}")
print(f" actual {actual}")
return None
# Unpack the whole archive, not just bin/: protoc resolves the well-known
# imports (google/protobuf/descriptor.proto) from ../include next to it.
staging = Path(tmp) / "unpacked"
with zipfile.ZipFile(archive) as zf:
zf.extractall(staging)
target = _protoc_dir()
if target.exists():
shutil.rmtree(target)
# Move into place only once complete, so an interrupted run never leaves
# a half-extracted toolchain that later runs would happily use.
shutil.move(str(staging), str(target))
binary = target / "bin" / _PROTOC_EXE
if not binary.exists():
print(f" ERROR: protoc archive did not contain bin/{_PROTOC_EXE}")
return None
binary.chmod(binary.stat().st_mode | 0o755)
return binary
def find_protoc(allow_download=True):
"""Return the protoc command as a list, or None if it can't be resolved."""
override = os.environ.get("PROTOC")
if override:
return [override]
on_path = shutil.which("protoc")
if on_path:
return [on_path]
cached = _protoc_dir() / "bin" / _PROTOC_EXE
if cached.exists():
return [str(cached)]
# Honour a pre-existing grpcio-tools install rather than downloading.
if importlib.util.find_spec("grpc_tools") is not None:
return [sys.executable, "-m", "grpc_tools.protoc"]
if allow_download:
binary = _download_protoc()
if binary is not None:
return [str(binary)]
return None
def missing_packages():
"""pip names of the required Python packages this interpreter can't import."""
missing = []
for pip_name, module in PYTHON_PACKAGES.items():
try:
found = importlib.util.find_spec(module) is not None
except (ImportError, ValueError):
found = False
if not found:
missing.append(pip_name)
return missing
def _venv_python():
venv_dir = CACHE_DIR / "venv"
if os.name == "nt":
return venv_dir / "Scripts" / "python.exe"
return venv_dir / "bin" / "python"
def bootstrap_python():
"""The cached virtualenv interpreter, if it exists and has the packages."""
python = _venv_python()
if not python.exists():
return None
probe = subprocess.run([str(python), "-c", "import google.protobuf, yaml"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return python if probe.returncode == 0 else None
def _ensure_venv(missing):
"""Create/reuse .codegen-tools/venv with the required packages installed."""
python = _venv_python()
if not python.exists():
print(f" Creating codegen virtualenv in {python.parent.parent}...")
if subprocess.run([sys.executable, "-m", "venv", str(python.parent.parent)]).returncode != 0 \
or not python.exists():
return None
print(f" Installing {', '.join(missing)} into the codegen virtualenv...")
result = subprocess.run([str(python), "-m", "pip", "install", "--quiet",
"--disable-pip-version-check", *missing])
return python if result.returncode == 0 else None
def ensure_python_runtime():
"""
Guarantee protobuf + pyyaml are importable.
If they aren't, re-run the calling script in a cached virtualenv that has
them and exit with its status. Keeping the packages out of the caller's
interpreter is what lets the build scripts work on distros that refuse
`pip install` into a system Python (PEP 668).
"""
missing = missing_packages()
if not missing:
return
if os.environ.get(_BOOTSTRAP_ENV):
# We are already inside the bootstrapped environment: installing again
# would just fail the same way.
print(f" ERROR: {', '.join(missing)} still missing after bootstrap.")
sys.exit(1)
# Reuse a complete virtualenv as-is: builds call this every time, and pip
# would otherwise hit the network on each one.
python = bootstrap_python() or _ensure_venv(missing)
if python is None:
print(f" ERROR: could not provide {', '.join(missing)}.")
print(f" Install them for this interpreter: {sys.executable} -m pip install "
f"{' '.join(missing)}")
sys.exit(1)
env = dict(os.environ, **{_BOOTSTRAP_ENV: "1"})
sys.exit(subprocess.run([str(python), *sys.argv], env=env).returncode)
def toolchain_ready():
"""
True if the codegen can run without downloading or installing anything --
either from this interpreter or by re-execing into an existing bootstrap
virtualenv. This is what decides whether a build regenerates on proto edits.
"""
if missing_packages() and bootstrap_python() is None:
return False
return find_protoc(allow_download=False) is not None
def main():
# --check is what ConfigCodegen.cmake probes with: it answers whether this
# interpreter can regenerate during a build, and must not download, install
# or create anything while doing so.
if "--check" in sys.argv[1:]:
return 0 if toolchain_ready() else 1
missing = missing_packages()
print(f"python: {sys.executable}")
print(f"packages: {'all present' if not missing else 'missing ' + ', '.join(missing)}")
protoc = find_protoc()
print(f"protoc: {' '.join(protoc) if protoc else 'NOT FOUND'}")
return 0 if protoc else 1
if __name__ == "__main__":
sys.exit(main())