Post a readable diff of changed custom G-code on profile PRs (#16605)

A profile stores each custom G-code setting as one JSON string, so GitHub
shows a G-code edit as one long changed line and reviewers cannot see
what changed.

Add a gcode-diff command to orca_profile_tool.py. For each changed
profile file it decodes, on both sides, every *_gcode and multiline
string option PrintConfig.cpp defines, the options Tab.cpp edits as
code, and any string value that spans several lines. It prints one
collapsed unified diff per setting as markdown. Files with the same diff
for a setting share one section. Trailing spaces are removed, file names
cannot mention GitHub users, and the output is cut to whole sections to
fit a GitHub comment.

Check profiles runs it between the PR's merge commit and its first
parent and adds the result to its artifact, with a line naming the PR
commit. Post profile check comment posts it under its own marker. It
keeps the comment while the diff is unchanged, edits it in place when
only the commit changed, replaces it when the diff changes, and deletes
it when the PR no longer changes any G-code.

The posting job still runs no PR code. It now posts only when the run
checked the head of the PR the artifact names, which covers the
validation comment too.

scripts/orca_profile_tool.py

@@ -12,6 +12,7 @@ commands:
   trim             delete profile files no <vendor>.json list references
   update-index     regenerate the *_list sections of <vendor>.json
   legacy-compat    write the printers user presets can be migrated back to
+  gcode-diff       print a readable markdown diff of changed custom G-code
 
 options shared by several commands:
   --vendor VENDOR      act on one vendor bundle only; repeatable, empty means all
@@ -94,11 +95,14 @@ parent in the OFL map. filament_vendor / filament_type resolve the same way.
 """
 
 import argparse
+import difflib
 import functools
+import html
 import json
 import os
 import posixpath
 import re
+import subprocess
 import sys
 import uuid
 from collections import Counter, defaultdict
@@ -192,6 +196,7 @@ VECTOR_KEYS = {
 
 PRINT_CONFIG_CPP = os.path.normpath(
     os.path.join(SCRIPTS_DIR, "..", "src", "libslic3r", "PrintConfig.cpp"))
+TAB_CPP = os.path.normpath(os.path.join(SCRIPTS_DIR, "..", "src", "slic3r", "GUI", "Tab.cpp"))
 
 
 @functools.lru_cache(maxsize=None)
@@ -313,11 +318,11 @@ REMEDY_HINTS = {
 }
 
 
-def print_error(msg):
-    print(f"\033[91m[ERROR]\033[0m {msg}")  # Red
+def print_error(msg, file=None):
+    print(f"\033[91m[ERROR]\033[0m {msg}", file=file)  # Red
 
-def print_warning(msg):
-    print(f"\033[93m[WARNING]\033[0m {msg}")  # Yellow
+def print_warning(msg, file=None):
+    print(f"\033[93m[WARNING]\033[0m {msg}", file=file)  # Yellow
 
 def print_info(msg):
     print(f"\033[94m[INFO]\033[0m {msg}")  # Blue
@@ -3209,6 +3214,223 @@ def run_legacy_compat(base_dir, profiles_dir, output, check):
     return 0
 
 
+# ---------------------------------------------------------------------------
+# gcode-diff
+# ---------------------------------------------------------------------------
+
+# UTF-8 bytes, kept well under GitHub's 65,536-character comment limit.
+GCODE_DIFF_BUDGET = 60000
+# The file names a section's summary shows before "and N more".
+GCODE_DIFF_SUMMARY_FILES = 3
+
+GCODE_DIFF_HEADER = (
+    "## Custom G-code changes\n"
+    "\n"
+    "GitHub shows custom G-code in a profile as one long line, so below is a line by line "
+    "diff of each changed G-code or other text setting.\n")
+
+
+@functools.lru_cache(maxsize=None)
+def _text_option_keys():
+    """The string options PrintConfig.cpp defines as G-code or other multi-line text
+    (every *_gcode option, and every one it marks multiline, such as notes), plus the
+    ones Tab.cpp edits as code, such as filename_format."""
+    with open(PRINT_CONFIG_CPP, encoding="utf-8") as f:
+        source = f.read()
+    keys = set()
+    for body in re.split(r"\bdef\s*=\s*this->add(?:_nullable)?\(", source)[1:]:
+        match = re.match(r'"(\w+)",\s*coStrings?\b', body)
+        if match and (match.group(1).endswith("_gcode")
+                      or re.search(r"def->multiline\s*=\s*true", body)):
+            keys.add(match.group(1))
+    if "machine_start_gcode" not in keys:
+        raise RuntimeError(f"{PRINT_CONFIG_CPP} no longer defines machine_start_gcode")
+    # Tab.cpp only adds keys, so a missing file is not an error.
+    try:
+        with open(TAB_CPP, encoding="utf-8") as f:
+            gui = f.read()
+    except OSError:
+        gui = ""
+    keys.update(re.findall(r'get_option\("(\w+)"\);(?:(?!get_option\().)*?\bis_code\s*=\s*true',
+                           gui, re.DOTALL))
+    return frozenset(keys)
+
+
+def _git(cwd, *args, stdin=None):
+    return subprocess.run(["git", "-C", cwd, *args], input=stdin, check=True,
+                          capture_output=True).stdout
+
+
+def _blobs_at(profiles_dir, specs):
+    """{(ref, path): bytes, or None if not a file there} for (ref, path relative to
+    profiles_dir) pairs, all read by one git cat-file process."""
+    prefix = _git(profiles_dir, "rev-parse", "--show-prefix").decode("utf-8").strip()
+    out = _git(profiles_dir, "cat-file", "--batch",
+               stdin="".join(f"{ref}:{prefix}{rel}\n" for ref, rel in specs).encode("utf-8"))
+    blobs, pos = {}, 0
+    for spec in specs:
+        end = out.index(b"\n", pos)
+        header = out[pos:end]
+        pos = end + 1
+        # "<name> missing" and "<name> ambiguous" have no content; "<oid> <type> <size>" does.
+        if header.endswith((b" missing", b" ambiguous")):
+            blobs[spec] = None
+            continue
+        _oid, kind, size = header.split()
+        blobs[spec] = out[pos:pos + int(size)] if kind == b"blob" else None
+        pos += int(size) + 1
+    return blobs
+
+
+def _parse_profile(raw, rel, where):
+    """The JSON in raw, or None with a warning when it is absent or not JSON."""
+    if raw is None:
+        print_warning(f"{rel}: skipped, not found at {where}", file=sys.stderr)
+        return None
+    try:
+        return json.loads(raw.decode("utf-8-sig"))
+    except ValueError as e:
+        print_warning(f"{rel}: skipped, cannot read it at {where}: {e}", file=sys.stderr)
+        return None
+
+
+def _changed_profiles(profiles_dir, base, head):
+    """(old path, new path) of each JSON file under profiles_dir that differs between
+    base and head (the working tree, untracked files included, when None); the absent
+    side of an added or deleted file is None, and so is a side that is not JSON. Renames
+    are paired."""
+    out = _git(profiles_dir, "diff", "--name-status", "-z", "-M", "--relative", base,
+               *([head] if head else []), "--", ".")
+    fields = out.decode("utf-8").split("\0")
+    pairs, i = [], 0
+    while i + 1 < len(fields):
+        status = fields[i]
+        old = new = fields[i + 1]
+        if status[0] in "RC":
+            new = fields[i + 2]
+            i += 3
+        else:
+            i += 2
+        old = None if status[0] == "A" or not old.endswith(".json") else old
+        new = None if status[0] == "D" or not new.endswith(".json") else new
+        if old or new:
+            pairs.append((old, new))
+    if head is None:
+        out = _git(profiles_dir, "ls-files", "-z", "--others", "--exclude-standard", "--", ".")
+        pairs += [(None, rel) for rel in out.decode("utf-8").split("\0")
+                  if rel.endswith(".json") and rel.split("/")[0] != USER_DIR]
+    return pairs
+
+
+def _text_lines(value):
+    return [line.rstrip() for line in value.split("\n")] if value else []
+
+
+def _text_changes(old, new):
+    """(setting, unified diff) for every G-code or other text value that differs
+    between two profiles; a vector option with several values gives one per value."""
+    changes = []
+    for key in list(new) + [k for k in old if k not in new]:
+        a, b = _as_list(old.get(key, [])), _as_list(new.get(key, []))
+        if a == b or not all(isinstance(v, str) for v in a + b):
+            continue
+        if key not in _text_option_keys() and not any("\n" in v for v in a + b):
+            continue
+        width = max(len(a), len(b))
+        for i in range(width):
+            diff = list(difflib.unified_diff(
+                _text_lines(a[i] if i < len(a) else ""), _text_lines(b[i] if i < len(b) else ""),
+                lineterm=""))[2:]  # drops the ---/+++ file header
+            if diff:
+                changes.append((key if width == 1 else f"{key}[{i}]", "\n".join(diff)))
+    return changes
+
+
+def _gcode_diff_section(files, setting, diff):
+    # A fence longer than any backtick run in the G-code cannot be closed by it.
+    fence = "`" * max([3] + [len(run) + 1 for run in re.findall(r"`+", diff)])
+    # In <code>, the "@BBL" of a filament file name does not mention a GitHub user.
+    def code(text):
+        return f"<code>{html.escape(text, quote=False)}</code>"
+    names, listing = ", ".join(map(code, files)), ""
+    if len(files) > GCODE_DIFF_SUMMARY_FILES + 1:  # never "and 1 more"
+        listing = f"<p>{names}</p>\n\n"  # an HTML block, so markdown in names stays literal
+        names = (", ".join(map(code, files[:GCODE_DIFF_SUMMARY_FILES]))
+                 + f" and {len(files) - GCODE_DIFF_SUMMARY_FILES} more")
+    return (f"\n<details>\n<summary><b>{names} - {code(setting)} (click to expand)</b>"
+            f"</summary>\n\n{listing}{fence}diff\n{diff}\n{fence}\n\n</details>\n")
+
+
+def gcode_diff(profiles_dir, base, head=None, vendors=None):
+    """Markdown with one collapsed diff per changed G-code setting between base and
+    head, or "" when none changed. Files whose diff of a setting is identical share
+    one section. Cut to whole sections under GCODE_DIFF_BUDGET."""
+    pairs = [(old, new) for old, new in _changed_profiles(profiles_dir, base, head)
+             if not vendors or any(p and p.split("/")[0] in vendors for p in (old, new))]
+    specs = ([(base, old) for old, _ in pairs if old]
+             + [(head, new) for _, new in pairs if new and head])
+    blobs = _blobs_at(profiles_dir, specs) if specs else {}
+
+    def load(ref, rel):
+        if rel is None:
+            return {}
+        if ref:
+            return _parse_profile(blobs[(ref, rel)], rel, ref)
+        try:
+            return _parse_profile(Path(profiles_dir, rel).read_bytes(), rel, "the working tree")
+        except OSError as e:
+            print_warning(f"{rel}: skipped, cannot read it: {e}", file=sys.stderr)
+            return None
+
+    groups = {}  # (setting, diff) -> files, in the order git lists them
+    for old_rel, new_rel in pairs:
+        old, new = load(base, old_rel), load(head, new_rel)
+        if not isinstance(old, dict) or not isinstance(new, dict):
+            continue
+        for setting, diff in _text_changes(old, new):
+            groups.setdefault((setting, diff), []).append(new_rel or old_rel)
+    if not groups:
+        return ""
+
+    sections = [_gcode_diff_section(files, setting, diff)
+                for (setting, diff), files in groups.items()]
+    budget = GCODE_DIFF_BUDGET - len(GCODE_DIFF_HEADER.encode("utf-8"))
+    shown = 0
+    for section in sections:
+        budget -= len(section.encode("utf-8"))
+        if budget < 0:
+            break
+        shown += 1
+    note, hidden = "", len(sections) - shown
+    if hidden:
+        note = (f"\n{hidden} of {len(sections)} sections {'is' if hidden == 1 else 'are'} not "
+                "shown because the comment would be too long for GitHub. See the Files "
+                "changed tab for the rest.\n")
+    return GCODE_DIFF_HEADER + "".join(sections[:shown]) + note
+
+
+def _default_gcode_diff_base(profiles_dir, head):
+    for main in ("upstream/main", "origin/main"):
+        try:
+            return _git(profiles_dir, "merge-base", head or "HEAD", main).decode().strip()
+        except subprocess.CalledProcessError:
+            continue
+    raise RuntimeError("no --base given, and neither upstream/main nor origin/main exists")
+
+
+def run_gcode_diff(profiles_dir, vendors, base, head):
+    try:
+        markdown = gcode_diff(profiles_dir, base or _default_gcode_diff_base(profiles_dir, head),
+                              head, vendors)
+    # UnicodeDecodeError means git listed a file name that is not UTF-8.
+    except (OSError, RuntimeError, UnicodeDecodeError, subprocess.CalledProcessError) as e:
+        stderr = getattr(e, "stderr", None)
+        print_error(stderr.decode(errors="replace").strip() if stderr else str(e), file=sys.stderr)
+        return 1
+    sys.stdout.write(markdown)
+    return 0
+
+
 # ---------------------------------------------------------------------------
 # CLI
 # ---------------------------------------------------------------------------
@@ -3233,6 +3455,9 @@ examples:
   orca_profile_tool.py legacy-compat --base /tmp/v2.4.2/resources/profiles
       rewrite resources/info/legacy_compatible_printers.json against 2.4.2's
       profiles (git archive v2.4.2 resources/profiles | tar -x -C /tmp/v2.4.2)
+  orca_profile_tool.py gcode-diff > gcode.md
+      the changed custom G-code of the working tree since it left upstream/main,
+      as markdown with one collapsed diff per setting
 
 after adding, renaming or deleting profile files, run in this order:
   normalize -> update-index -> generate-id -> check
@@ -3387,6 +3612,25 @@ def build_parser():
     legacy_cmd.add_argument("--check", action="store_true",
                             help="fail if the map differs from what --base implies; write nothing")
 
+    gcode_cmd = add(
+        "gcode-diff", [vendor_opt, profiles_opt],
+        "print a readable markdown diff of changed custom G-code",
+        "Print, as markdown, a unified diff of every G-code or other multi-line\n"
+        "setting that differs between --base and --head in each changed profile\n"
+        "file, one collapsed section per setting. Files whose diff of a setting is\n"
+        "identical share a section. Trailing spaces are removed from every line, and\n"
+        "the output is cut to whole sections to fit a GitHub comment. Prints nothing\n"
+        "when no such setting changed. CI posts it on profile PRs.\n"
+        "\n"
+        "The settings are the *_gcode and multiline string options PrintConfig.cpp\n"
+        "defines, the options Tab.cpp edits as code (such as filename_format), and any\n"
+        "string value that spans several lines.")
+    gcode_cmd.add_argument("--base", metavar="REF",
+                           help="compare against this git ref (default: the merge base "
+                                "with upstream/main, else origin/main)")
+    gcode_cmd.add_argument("--head", metavar="REF",
+                           help="compare this git ref (default: the working tree)")
+
     return parser
 
 
@@ -3423,6 +3667,9 @@ def main(argv=None):
     if args.command == "legacy-compat":
         return run_legacy_compat(args.base, profiles_dir, args.output, args.check)
 
+    if args.command == "gcode-diff":
+        return run_gcode_diff(profiles_dir, vendors, args.base, args.head)
+
     if args.command == "fix-variant":
         _changed, errors = fix_variant_arrays(profiles_dir, vendors, dry_run=args.dry_run,
                                                strict=args.strict)