Repair shipped default materials and obsolete settings, and validate them (#15741)

* add orca profile skill

* add default material check

Improve validation for default materials and filament profiles

* Fix default materials and obsolete keys

* clarifying orca-profiles skill
This commit is contained in:
SoftFever
2026-09-18 00:39:46 +08:00
committed by GitHub
parent 60b4a61854
commit f520e9221f
2135 changed files with 85633 additions and 87858 deletions
+61 -50
View File
@@ -24,9 +24,11 @@ options shared by several commands:
since the snapshot describes resources/profiles alone
After adding, renaming or deleting profile files, run:
normalize -> trim -> update-index -> generate-id -> update-snapshot -> check
Each step feeds the next: normalize writes the "type" update-index files a
profile by, and trim judges against the index update-index is about to rebuild.
normalize -> update-index -> generate-id -> update-snapshot -> check
normalize supplies missing types; update-index registers presets before id
generation. update-snapshot is needed when filament ids or claims change.
Use trim only for deliberate cleanup, previewed with --dry-run: it judges against
the current index and can delete newly added, unindexed presets.
Run from anywhere; "python scripts/orca_profile_tool.py --help" repeats this list
and "... <command> --help" documents one command in full.
@@ -128,6 +130,9 @@ BAMBU_MAP_PATH = os.path.normpath(
os.path.join(SCRIPTS_DIR, "..", "resources", "printers", "bambu_filament_ids.json"))
OFL = "OrcaFilamentLibrary"
# The validator's data dir, created under resources/profiles by a local run;
# not a vendor bundle, so an unscoped pass leaves it alone.
USER_DIR = "user"
# Bambu (BBL) is the only vendor exempt from the setting_id rule: it keeps its
# authoritative "G*" cloud ids. No vendor is exempt from the filament_id rule.
@@ -146,7 +151,9 @@ PROFILE_TYPES = ("machine_model", "process", "filament", "machine")
# Data files that sit under a vendor bundle but are not presets: no name, no type.
NON_PROFILE_FILES = {"filaments_color_codes.json", "cli_config.json"}
# Settings dropped from PrintConfig.cpp. Reported by "check --obsolete-keys".
# Mirror PrintConfigDef::handle_legacy's ignore set in PrintConfig.cpp; a test
# checks parity. Used by normalize and check. Active options and
# legacy aliases that the loader migrates do not belong here.
OBSOLETE_KEYS = {
"acceleration", "scale", "rotate", "duplicate", "duplicate_grid",
"bed_size", "print_center", "g0", "wipe_tower_per_color_wipe",
@@ -158,10 +165,11 @@ OBSOLETE_KEYS = {
"bed_temperature_initial_layer", "can_switch_nozzle_type", "can_add_auxiliary_fan",
"extra_flush_volume", "spaghetti_detector", "adaptive_layer_height",
"z_hop_type", "z_lift_type", "bed_temperature_difference", "long_retraction_when_cut",
"retraction_distance_when_cut", "extruder_type", "internal_bridge_support_thickness",
"extruder_clearance_max_radius", "top_area_threshold", "reduce_wall_solid_infill",
"retraction_distance_when_cut", "internal_bridge_support_thickness",
"top_area_threshold", "reduce_wall_solid_infill",
"filament_load_time", "filament_unload_time", "smooth_coefficient",
"overhang_totally_speed", "silent_mode", "overhang_speed_classic"
"overhang_totally_speed", "silent_mode", "overhang_speed_classic",
"anisotropic_surfaces",
}
# Keys renamed at some point, whose old and new spellings must never co-exist:
@@ -1050,13 +1058,12 @@ def load_available_filament_profiles(profiles_dir, vendor):
def check_machine_default_materials(profiles_dir, vendor):
"""Every default material a machine names must exist, in the bundle or in OFL.
Returns (errors, warnings); the warning is the bundle having no machine/ at all.
Returns (errors, warnings); a bundle with no machine/ has nothing to check.
"""
error_count = 0
machine_dir = Path(profiles_dir) / vendor / "machine"
if not machine_dir.exists():
print_warning(f"No machine profiles found for vendor: {vendor}")
return 0, 1
return 0, 0
available = (load_available_filament_profiles(profiles_dir, vendor)
| load_available_filament_profiles(profiles_dir, OFL))
@@ -1246,7 +1253,7 @@ def check_filament_id_length(profiles_dir, vendor):
def check_obsolete_keys(profiles_dir, vendor):
"""Warn about settings PrintConfig.cpp no longer defines. Returns the count."""
"""Warn about settings PrintConfig.cpp explicitly discards. Returns the count."""
warn_count = 0
profiles_path = Path(profiles_dir)
vendor_path = profiles_path / vendor / "filament"
@@ -1397,16 +1404,17 @@ def check_normalized(profiles_dir, vendor):
# check
# ---------------------------------------------------------------------------
def check_profiles(profiles_dir=PROFILES_DIR, vendors=None, snapshot_path=SNAPSHOT_PATH,
materials=False, obsolete_keys=False):
def check_profiles(profiles_dir=PROFILES_DIR, vendors=None, snapshot_path=SNAPSHOT_PATH):
"""Validate the whole profile tree. Returns the error count.
The per-vendor checks honour `vendors`; the setting_id and filament_id checks are
cross-vendor properties a narrowed run cannot answer, so they always cover the
whole tree. With no `vendors`, OrcaFilamentLibrary is left out of the per-vendor
pass: it is the shared base bundle, its filaments are generic by design, and they
are checked through the vendors that inherit them. Naming it explicitly checks it.
The normalization pass covers it either way - see the comment on that loop.
whole tree. With no `vendors`, every bundle is checked except the `user` directory
a local validator run leaves behind, being its data dir rather than a bundle;
naming it explicitly checks it. OrcaFilamentLibrary is checked like any other
bundle, its only exemption being that a library filament may leave
compatible_printers empty - what check_filament_compatible_printers applies. The
normalization pass takes its own vendor list - see the comment on that loop.
"""
print_info("Checking profiles ...")
errors_found = 0
@@ -1416,19 +1424,17 @@ def check_profiles(profiles_dir=PROFILES_DIR, vendors=None, snapshot_path=SNAPSH
if vendors:
checked = list(vendors)
else:
checked = [v for v in list_profile_dirs(profiles_dir) if v != OFL]
checked = [v for v in list_profile_dirs(profiles_dir) if v != USER_DIR]
for vendor in checked:
errors_found += check_preset_name_uniqueness(profiles_dir, vendor)
errors_found += check_filament_compatible_printers(profiles_dir, vendor)
if materials:
new_errors, new_warnings = check_machine_default_materials(profiles_dir, vendor)
errors_found += new_errors
warnings_found += new_warnings
new_errors, new_warnings = check_machine_default_materials(profiles_dir, vendor)
errors_found += new_errors
warnings_found += new_warnings
if obsolete_keys:
warnings_found += check_obsolete_keys(profiles_dir, vendor)
warnings_found += check_obsolete_keys(profiles_dir, vendor)
new_errors, new_warnings = check_name_consistency(profiles_dir, vendor)
errors_found += new_errors
@@ -1445,12 +1451,11 @@ def check_profiles(profiles_dir=PROFILES_DIR, vendors=None, snapshot_path=SNAPSH
errors_found += new_errors
remedies.update(gaps)
# normalize and update-index know nothing of the OrcaFilamentLibrary exemption
# above - that bundle sits out the per-vendor pass because its filaments are
# generic by design, which says nothing about the shape of its files - so this pass
# takes its own vendor list. Unscoped that is the bundles with an index, exactly
# what those two commands take; a --vendor is passed through as given, so a bundle
# whose index has not landed yet still has its files held to what normalize writes.
# normalize and update-index judge file and index shape, not the preset-content
# rules above, so this pass takes its own vendor list. Unscoped that is the
# bundles with an index, exactly what those two commands take; a --vendor is
# passed through as given, so a bundle whose index has not landed yet still has
# its files held to what normalize writes.
for vendor in (vendors or list_vendor_names(profiles_dir)):
new_errors, gaps = check_normalized(profiles_dir, vendor)
errors_found += new_errors
@@ -2061,6 +2066,10 @@ def _normalize_profile(data, sub):
del data[field]
changes.append(f"remove {field}")
for field in sorted(OBSOLETE_KEYS.intersection(data)):
del data[field]
changes.append(f"remove {field}")
# BBS renamed extruder_clearance_radius to extruder_clearance_max_radius, but some
# profiles carry both with different values, and the slicer cannot tell which one
# to obey - a toolhead collision waiting to happen. Keep the larger one only.
@@ -2306,8 +2315,9 @@ def build_index_sections(profiles_dir, vendor, profile_types=None):
one message each, for the caller to report. `sections` is None when two files claim
one preset name: the bundle can only hold one profile under a name, so rebuilding
would pick a winner by directory order and quietly drop the other, and the index has
to be left alone instead. Deleting the stale copy is trim's job, which is why it
runs before this.
to be left alone instead. Identify the intended preset and delete or rename the
duplicate before retrying. Use trim only for deliberate unindexed-file cleanup,
previewed with --dry-run.
"""
vendor_dir = os.path.join(profiles_dir, vendor)
sections = {}
@@ -2360,8 +2370,8 @@ def build_index_sections(profiles_dir, vendor, profile_types=None):
problems.append(f'{vendor}.json: {len(subs)} profiles are named "{name}" '
f'({", ".join(sorted(subs))}); only one can be indexed under '
f"that name, so delete or rename the others - "
f'"python scripts/orca_profile_tool.py trim" removes an '
f"unindexed copy")
f"preview unindexed-file cleanup with "
f'"python scripts/orca_profile_tool.py trim --dry-run"')
return (None if clashes else sections), problems
@@ -2433,9 +2443,11 @@ examples:
re-record the sanctioned filament_id state after a generate-id run
after adding, renaming or deleting profile files, run in this order:
normalize -> trim -> update-index -> generate-id -> update-snapshot -> check
each step feeds the next: normalize writes the "type" update-index files a
profile by, and trim judges against the index update-index is about to rebuild.
normalize -> update-index -> generate-id -> update-snapshot -> check
normalize supplies missing types; update-index registers presets before id
generation. update-snapshot is needed when filament ids or claims change.
Use trim only for deliberate cleanup, previewed with --dry-run: it judges against
the current index and can delete newly added, unindexed presets.
"""
@@ -2483,23 +2495,18 @@ def build_parser():
name, parents=parents, help=help_text, description=description,
allow_abbrev=False, formatter_class=argparse.RawDescriptionHelpFormatter)
check_cmd = add(
add(
"check", [vendor_opt, snapshot_opt, profiles_opt],
"validate the whole profile tree -- what CI runs",
"Validate the whole profile tree: preset name uniqueness, index coverage\n"
"both ways, compatible_printers, conflicting and vector-typed keys,\n"
"filament_id length, that normalize and update-index would leave every\n"
"bundle alone, and the tree-wide setting_id and filament_id state.\n"
"Exits nonzero on errors.\n"
"both ways, compatible_printers, default-material references, obsolete,\n"
"conflicting and vector-typed keys, filament_id length, that normalize and\n"
"update-index would leave every bundle alone, and the tree-wide setting_id\n"
"and filament_id state. Exits nonzero on errors.\n"
"\n"
"--vendor narrows the per-vendor checks only: setting_id uniqueness and the\n"
"filament_id state are cross-vendor properties a narrowed run cannot answer,\n"
"so they always cover the whole tree.")
check_cmd.add_argument("--materials", action="store_true",
help="also check that every default material a machine names "
"exists")
check_cmd.add_argument("--obsolete-keys", action="store_true", dest="obsolete_keys",
help="also warn about settings the slicer no longer defines")
generate_cmd = add(
"generate-id", [vendor_opt, dry_run_opt, profiles_opt],
@@ -2533,6 +2540,9 @@ def build_parser():
"loader only ever reads the sub_paths listed there, so an unindexed preset\n"
"never loads.\n"
"\n"
"Use only for deliberate cleanup, previewed with --dry-run. Newly added,\n"
"unindexed presets can be deleted too; omit trim from the authoring workflow.\n"
"\n"
"Assets and data files are kept, a file that cannot be parsed is kept and\n"
"reported, and so is one a surviving profile inherits from that no indexed\n"
"profile provides -- but a stale copy of an indexed profile goes, since\n"
@@ -2546,7 +2556,9 @@ def build_parser():
"A profile is indexed under the section its own \"type\" names, so run\n"
"normalize first: it is what writes a missing type. Two files claiming one\n"
"preset name leave that index alone, because a rebuild could only keep one\n"
"of them; run trim first, which is what clears a stale copy.")
"of them. Identify the intended preset and delete or rename the duplicate.\n"
"Use trim only for deliberate unindexed-file cleanup, previewed with\n"
"--dry-run; it can also delete newly authored presets.")
add("update-snapshot", [dry_run_opt, snapshot_opt, profiles_opt],
"re-record scripts/filament_id_snapshot.json",
@@ -2591,8 +2603,7 @@ def main(argv=None):
snapshot_path = SNAPSHOT_PATH
if args.command == "check":
errors = check_profiles(profiles_dir, vendors, snapshot_path,
materials=args.materials, obsolete_keys=args.obsolete_keys)
errors = check_profiles(profiles_dir, vendors, snapshot_path)
return 1 if errors else 0
if args.command == "generate-id":
+3 -1
View File
@@ -1696,7 +1696,9 @@ class TestCli(unittest.TestCase):
["--update-snapshot"], # the pre-subcommand flag
["nonsense"], # not a command
["generate-id", "--filament-id", "--setting-id"],
["generate-id", "--materials"], # check's option
["generate-id", "--snapshot", "x"], # check's option
["check", "--materials"], # removed flag
["check", "--obsolete-keys"], # removed flag
["check", "--filament-id"], # generate-id's option
["normalize", "--snapshot", "x"], # not a snapshot command
["normalize", "--profile-type", "nozzle"]): # not a profile type
+114 -10
View File
@@ -12,6 +12,7 @@ import contextlib
import io
import json
import os
import re
import shutil
import sys
import tempfile
@@ -125,6 +126,20 @@ class TreeCase(unittest.TestCase):
# normalize
# ---------------------------------------------------------------------------
class TestObsoleteKeys(unittest.TestCase):
def test_obsolete_keys_match_the_loader_ignore_set(self):
path = os.path.join(REPO_ROOT, "src", "libslic3r", "PrintConfig.cpp")
with open(path, encoding="utf-8") as f:
source = f.read()
match = re.search(
r"void PrintConfigDef::handle_legacy\(.*?"
r"static\s+std::set<std::string>\s+ignore\s*=\s*\{(.*?)\};",
source, re.DOTALL)
self.assertIsNotNone(match, "Could not locate the loader's obsolete-key set")
keys = re.sub(r"//[^\n]*|/\*.*?\*/", "", match.group(1), flags=re.DOTALL)
self.assertEqual(apt.OBSOLETE_KEYS, set(re.findall(r'"([^"\n]+)"', keys)))
class TestNormalize(TreeCase):
def test_a_missing_type_is_filled_in_from_the_directory(self):
self.t.write("V", "filament/A.json", {"name": "A"})
@@ -148,16 +163,36 @@ class TestNormalize(TreeCase):
self.t.write("V", "filament/A.json", {
"type": "filament", "name": "A", "version": "1.2.3",
"is_custom_defined": "1", "filament_type": "PLA",
"filament_vendor": "AV", "travel_speed": 200})
"filament_vendor": "AV", "travel_speed": 200,
"filament_load_time": ["15"], "filament_unload_time": "0"})
rc, out = self.run_command("normalize")
self.assertEqual(rc, 0, out)
data = self.t.read("V", "filament/A.json")
self.assertNotIn("version", data)
self.assertNotIn("is_custom_defined", data)
self.assertNotIn("travel_speed", data) # a process setting, not a filament one
self.assertNotIn("filament_load_time", data)
self.assertNotIn("filament_unload_time", data)
self.assertEqual(data["filament_type"], ["PLA"])
self.assertEqual(data["filament_vendor"], ["AV"])
def test_obsolete_keys_are_removed_from_every_profile_type(self):
for sub in ("filament", "process", "machine"):
with self.subTest(profile_type=sub):
expected = {"type": sub, "name": "A"}
self.t.write("V", f"{sub}/A.json", {
**expected, "silent_mode": "", "adaptive_layer_height": "0",
"anisotropic_surfaces": "1", "filament_load_time": ["0"],
"filament_unload_time": "0"})
rc, out = self.run_command("normalize")
self.assertEqual(rc, 0, out)
self.assertEqual(self.t.read("V", f"{sub}/A.json"), expected)
before = self.t.bytes_map()
rc, out = self.run_command("normalize")
self.assertEqual(rc, 0, out)
self.assertIn("0 profile(s) normalized", out)
self.assertEqual(self.t.bytes_map(), before)
def test_the_larger_extruder_clearance_wins(self):
# Keeping the smaller one would licence a toolhead collision.
self.t.write("V", "machine/M.json", {
@@ -186,6 +221,15 @@ class TestNormalize(TreeCase):
def test_a_conforming_tree_is_left_byte_identical(self):
self.t.write("V", "filament/A.json", {"type": "filament", "name": "A"})
# These used to be misclassified as obsolete: one is active, the other
# is a legacy alias that still supplies the toolhead clearance on load.
self.t.write("V", "machine/M.json", {
"type": "machine", "name": "M", "extruder_type": ["Direct Drive"],
"extruder_clearance_max_radius": "68", "machine_load_filament_time": "15",
"machine_unload_filament_time": "10"})
self.t.write("V", "process/P.json", {
"type": "process", "name": "P", "travel_speed": "200",
"top_surface_fill_order": "outward"})
before = self.t.bytes_map()
rc, out = self.run_command("normalize")
self.assertEqual(rc, 0, out)
@@ -200,7 +244,7 @@ class TestNormalize(TreeCase):
b'{\n\t"type": "filament",\n\t"name": "A"\n}\n')
def test_dry_run_writes_nothing(self):
self.t.write("V", "filament/A.json", {"name": "A"})
self.t.write("V", "filament/A.json", {"name": "A", "bed_temperature": ["60"]})
before = self.t.bytes_map()
rc, out = self.run_command("normalize", "--dry-run")
self.assertEqual(rc, 0, out)
@@ -453,6 +497,8 @@ class TestCheck(TreeCase):
errors += apt.check_filament_id_length(self.t.profiles, "V")
conflict, _warn = apt.check_conflict_keys(self.t.profiles, "V")
errors += conflict
materials, _warn = apt.check_machine_default_materials(self.t.profiles, "V")
errors += materials
return errors, buf.getvalue()
def test_a_clean_bundle_reports_nothing(self):
@@ -467,6 +513,27 @@ class TestCheck(TreeCase):
self.assertGreater(errors, 0)
self.assertIn("'compatible_printers' missing", out)
def test_a_library_filament_may_leave_compatible_printers_empty(self):
# The shared library is exempt from that rule and nothing else.
self.t.write(apt.OFL, "filament/A.json",
{"type": "filament", "name": "A", "instantiation": "true"})
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
errors = apt.check_filament_compatible_printers(self.t.profiles, apt.OFL)
self.assertEqual(errors, 0, buf.getvalue())
def test_the_library_is_checked_like_any_other_bundle(self):
# A file the library's own index does not reference must fail plain
# `check`, now that the per-vendor pass no longer skips it.
self.t.write(apt.OFL, "filament/Stray.json",
{"type": "filament", "name": "Stray"})
snapshot = os.path.join(self.t.dir, "snapshot.json")
self.run_command("update-snapshot", "--snapshot", snapshot)
rc, out = self.run_command("check", "--snapshot", snapshot)
self.assertEqual(rc, 1, out)
self.assertIn(f"{apt.OFL}/filament/Stray.json: no {apt.OFL}.json list "
f"references it", out)
def test_a_duplicate_key_is_an_error(self):
self.bundle().write_raw("V", "filament/B.json",
b'{"type":"filament","name":"B","name":"B2"}')
@@ -516,15 +583,28 @@ class TestCheck(TreeCase):
self.assertGreater(errors, 0)
self.assertIn("Filament id too long", out)
def test_obsolete_keys_are_opt_in_warnings(self):
def test_obsolete_key_warnings_exclude_active_and_renamed_options(self):
self.bundle().write("V", "filament/B.json", {
"type": "filament", "name": "B", "silent_mode": True})
"type": "filament", "name": "B", "silent_mode": "0",
"anisotropic_surfaces": "0", "extruder_type": ["Direct Drive"],
"extruder_clearance_max_radius": "68"})
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
warnings = apt.check_obsolete_keys(self.t.profiles, "V")
self.assertEqual(warnings, 1)
self.assertEqual(warnings, 2)
self.assertIn("Obsolete key", buf.getvalue())
def test_obsolete_key_warnings_run_without_a_flag(self):
self.t.write("V", "filament/A.json", {
"type": "filament", "name": "A", "silent_mode": "0"})
self.run_command("update-index")
snapshot = os.path.join(self.t.dir, "snapshot.json")
self.run_command("update-snapshot", "--snapshot", snapshot)
rc, out = self.run_command("check", "--snapshot", snapshot)
self.assertEqual(rc, 1, out) # normalization also rejects the obsolete key
self.assertIn("Obsolete key: 'silent_mode' found in V/filament/A.json", out)
self.assertIn("Files with warnings : 1", out)
def test_a_default_material_must_exist_somewhere(self):
self.bundle().write("V", "machine/M.json", {
"type": "machine", "name": "M 0.4 nozzle",
@@ -535,6 +615,32 @@ class TestCheck(TreeCase):
self.assertEqual(errors, 1)
self.assertIn("'Nope'", buf.getvalue())
def test_a_default_material_fails_check_without_a_flag(self):
# The reference check is part of the default run, not an opt-in: a
# dangling name has to fail plain `check`.
self.bundle()
self.t.write("V", "machine/M.json", {
"type": "machine", "name": "M 0.4 nozzle",
"default_filament_profile": ["A", "Nope"]})
self.t.index("V", "machine", "M 0.4 nozzle", "machine/M.json")
snapshot = os.path.join(self.t.dir, "snapshot.json")
self.run_command("update-snapshot", "--snapshot", snapshot)
rc, out = self.run_command("check", "--snapshot", snapshot)
self.assertEqual(rc, 1, out)
self.assertIn("Missing filament profile: 'Nope'", out)
def test_the_stray_user_directory_is_not_a_vendor(self):
# A local validator run leaves resources/profiles/user/ behind; an
# unscoped check must not count it as a bundle and warn about it.
self.bundle()
for sub in apt.PROFILE_SUBDIRS:
os.makedirs(os.path.join(self.t.profiles, apt.USER_DIR, "default", sub))
snapshot = os.path.join(self.t.dir, "snapshot.json")
self.run_command("update-snapshot", "--snapshot", snapshot)
_rc, out = self.run_command("check", "--snapshot", snapshot)
self.assertIn("Checked vendors : 1", out)
self.assertNotIn("user", out)
def names(self, vendor="V"):
"""The preset name check for one bundle, which is what --vendor narrows."""
buf = io.StringIO()
@@ -752,9 +858,8 @@ class TestNormalized(TreeCase):
self.assertEqual(gaps["stale_index"], 0, out)
def test_the_shared_base_bundle_is_covered_too(self):
# The per-vendor pass leaves OrcaFilamentLibrary out because its filaments are
# generic by design. That says nothing about the shape of its files, and
# normalize and update-index rewrite that bundle like any other.
# normalize and update-index own the shape of every bundle, the shared
# library included.
self.t.write(apt.OFL, "filament/A.json",
{"type": "filament", "name": "A", "version": "01.00.00.00"})
rc, out = self.run_command("check", "--snapshot", self.snapshot())
@@ -791,8 +896,7 @@ class TestDispatch(TreeCase):
self.assertIn(expected, out)
def test_an_option_belongs_to_one_command_only(self):
for argv in (["normalize", "--materials"],
["trim", "--force"],
for argv in (["trim", "--force"],
["update-index", "--filament-id"],
["check", "--profile-type", "filament"],
["update-snapshot", "--vendor", "V"]):