mirror of
https://github.com/OrcaSlicer/OrcaSlicer.git
synced 2026-10-06 23:31:19 +00:00
Hanif Koh
7412dab76a
A using-directive or using-declaration in the global namespace of a header reaches every file that includes it, and a using-declaration also makes the include checker treat that header as the one to include for the name. google-global-names-in-headers reports both, on changed lines like the existing check, so headers that still have one are not held to it until the line is touched. The check does not see a using inside a namespace. clang_tidy_diff.py's closing message assumed every finding was a missing include; it now says other findings need a manual fix.
327 lines
14 KiB
Python
327 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Run clang-tidy on the C++ files changed since a base revision.
|
|
|
|
Check findings are reported only on added or modified lines, so existing code
|
|
is not held to checks it predates. A changed source file that does not compile
|
|
(for example one that only built because the precompiled header supplied an
|
|
include) fails wherever the error is. Headers are compiled on their own, and
|
|
fail only on errors in their changed lines. Deleting an #include also fails
|
|
on every use, changed or not, that now lacks the header it provided.
|
|
|
|
With -- --fix, clang-tidy adds the missing includes, on those lines only, and
|
|
each file it changed is checked again so that only what remains is reported.
|
|
|
|
The checks come from .clang-tidy at the repository root. The compile database
|
|
must come from a configure with SLIC3R_PCH=OFF, or the precompiled header hides
|
|
missing includes.
|
|
|
|
python3 scripts/clang_tidy_diff.py -p build-tidy --base origin/main
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass, field
|
|
from pathlib import PurePosixPath
|
|
|
|
SOURCE_EXTENSIONS = {".cpp", ".cc", ".cxx"}
|
|
HEADER_EXTENSIONS = {".hpp", ".h", ".hxx"}
|
|
CHECKED_DIRS = ("src/", "tests/")
|
|
# Vendored code inside the checked directories.
|
|
EXCLUDED_DIRS = ("src/glad/", "tests/catch2/")
|
|
|
|
# Per file. A deleted include can leave hundreds of follow-on errors.
|
|
MAX_REPORTED = 30
|
|
|
|
# Subprocess output is UTF-8 whatever the locale, which is cp1252 on Windows.
|
|
UTF8 = {"encoding": "utf-8", "errors": "replace"}
|
|
|
|
HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
DIAGNOSTIC_RE = re.compile(r"^(.+?):(\d+):(\d+): (error|warning): (.*)$")
|
|
FIX_MESSAGE_RE = re.compile(r"^\s+Message:\s+(['\"])(.*)\1$")
|
|
FIX_INCLUDE_RE = re.compile(r"^\s+ReplacementText:\s+['\"]#include ([^\\'\"]*)")
|
|
REMOVED_INCLUDE_RE = re.compile(r'^-\s*#\s*include\s*[<"]([^>"]+)[>"]')
|
|
|
|
|
|
@dataclass
|
|
class FileChange:
|
|
lines: list = field(default_factory=list) # [first, last] ranges of added lines
|
|
removed_includes: set = field(default_factory=set) # header names whose #include was deleted
|
|
|
|
|
|
@dataclass
|
|
class Diagnostic:
|
|
file: str
|
|
line: int
|
|
col: int
|
|
level: str
|
|
message: str
|
|
include: str = "" # header named by the fix, when the fix adds an #include
|
|
|
|
@property
|
|
def is_compile_error(self):
|
|
return self.message.endswith("[clang-diagnostic-error]")
|
|
|
|
def __str__(self):
|
|
return self.message + (f" (add #include {self.include})" if self.include else "")
|
|
|
|
|
|
def parse_diff(diff):
|
|
"""Map each file in a `git diff -U0` to the lines it adds and the includes it deletes."""
|
|
changes = {}
|
|
change = None
|
|
for line in diff.splitlines():
|
|
if line.startswith("+++ "):
|
|
# git appends a tab to the header of a path that contains a space.
|
|
target = line[4:].removesuffix("\t")
|
|
change = changes.setdefault(target[2:], FileChange()) if target.startswith("b/") else None
|
|
continue
|
|
if change is None:
|
|
continue
|
|
match = HUNK_RE.match(line)
|
|
if match:
|
|
first = int(match.group(1))
|
|
count = int(match.group(2) or 1)
|
|
if count > 0:
|
|
change.lines.append([first, first + count - 1])
|
|
continue
|
|
match = REMOVED_INCLUDE_RE.match(line)
|
|
if match:
|
|
change.removed_includes.add(match.group(1))
|
|
return changes
|
|
|
|
|
|
def is_checked(path):
|
|
if not path.startswith(CHECKED_DIRS) or path.startswith(EXCLUDED_DIRS):
|
|
return False
|
|
return PurePosixPath(path).suffix in SOURCE_EXTENSIONS | HEADER_EXTENSIONS
|
|
|
|
|
|
def changed_files(merge_base):
|
|
# Against the working tree, so a local run covers uncommitted edits too.
|
|
# core.quotePath=false keeps a non-ASCII path unquoted, and the explicit prefixes
|
|
# override diff.noprefix and diff.mnemonicPrefix, so parse_diff sees its b/ prefix.
|
|
diff = subprocess.run(["git", "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff",
|
|
"--src-prefix=a/", "--dst-prefix=b/", "--diff-filter=AMR", merge_base],
|
|
check=True, capture_output=True, **UTF8).stdout
|
|
return {path: change for path, change in parse_diff(diff).items() if is_checked(path)}
|
|
|
|
|
|
def compiled_sources(build_dir):
|
|
with open(os.path.join(build_dir, "compile_commands.json"), encoding="utf-8") as f:
|
|
return {os.path.realpath(os.path.join(entry["directory"], entry["file"])) for entry in json.load(f)}
|
|
|
|
|
|
def run_clang_tidy(clang_tidy, build_dir, path, lines, extra_args):
|
|
"""Run clang-tidy on one file, limited to `lines` unless it is None."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fixes = os.path.join(tmp, "fixes.yaml")
|
|
# The compile database comes from the system clang, which may know warning
|
|
# flags a newer clang-tidy has dropped.
|
|
cmd = [clang_tidy, "-p", build_dir, "--quiet", "--export-fixes=" + fixes,
|
|
"--extra-arg=-Wno-unknown-warning-option", "--extra-arg=-ferror-limit=0", *extra_args, path]
|
|
if lines is not None:
|
|
# clang-tidy matches the name against the end of the file's native path.
|
|
cmd.insert(1, "--line-filter=" + json.dumps([{"name": os.path.normpath(path), "lines": lines}]))
|
|
result = subprocess.run(cmd, capture_output=True, **UTF8)
|
|
suggestions = parse_suggested_includes(fixes)
|
|
output = result.stdout + result.stderr
|
|
return result.returncode, output, parse_diagnostics(output, suggestions)
|
|
|
|
|
|
def parse_suggested_includes(fixes_path):
|
|
"""Map each message in an --export-fixes file to the header its fix includes."""
|
|
if not os.path.exists(fixes_path):
|
|
return {}
|
|
suggestions = {}
|
|
message = None
|
|
with open(fixes_path, encoding="utf-8") as f:
|
|
for line in f:
|
|
match = FIX_MESSAGE_RE.match(line)
|
|
if match:
|
|
message = match.group(2).replace("''", "'")
|
|
continue
|
|
match = FIX_INCLUDE_RE.match(line)
|
|
if match and message:
|
|
suggestions[message] = match.group(1)
|
|
message = None
|
|
return suggestions
|
|
|
|
|
|
def parse_diagnostics(output, suggestions):
|
|
diagnostics = []
|
|
for line in output.splitlines():
|
|
match = DIAGNOSTIC_RE.match(line)
|
|
if match:
|
|
file, row, col, level, message = match.groups()
|
|
include = suggestions.get(re.sub(r" \[[^]]*\]$", "", message), "")
|
|
diagnostics.append(Diagnostic(file, int(row), int(col), level, message, include))
|
|
return diagnostics
|
|
|
|
|
|
def in_ranges(line, ranges):
|
|
return any(first <= line <= last for first, last in ranges)
|
|
|
|
|
|
def names_removed_include(include, removed):
|
|
"""Whether `include` (<a/b.hpp>) is one of the deleted includes, however it was spelled."""
|
|
name = include.strip('<>"')
|
|
return bool(name) and any(name == r or name.endswith("/" + r) or r.endswith("/" + name) for r in removed)
|
|
|
|
|
|
def own_errors(diagnostics, path):
|
|
real = os.path.realpath(path)
|
|
return [d for d in diagnostics if d.is_compile_error and os.path.realpath(d.file) == real]
|
|
|
|
|
|
def error_sites(errors, text):
|
|
"""Where each error points, as (source line, column): stable across edits to other lines,
|
|
unlike line numbers, and across a removed include, unlike clang's wording."""
|
|
lines = text.splitlines()
|
|
return Counter((lines[d.line - 1].strip() if d.line <= len(lines) else "", d.col) for d in errors)
|
|
|
|
|
|
def errors_alone_at(revision, clang_tidy, build_dir, path):
|
|
"""The error sites a header had when compiled on its own at `revision`."""
|
|
shown = subprocess.run(["git", "show", f"{revision}:{path}"], capture_output=True, **UTF8)
|
|
if shown.returncode != 0:
|
|
return Counter()
|
|
# Beside the original, so its quoted includes resolve the same way.
|
|
p = PurePosixPath(path)
|
|
copy = str(p.with_name(f".{p.stem}.clang-tidy-base{p.suffix}"))
|
|
try:
|
|
with open(copy, "w", encoding="utf-8") as f:
|
|
f.write(shown.stdout)
|
|
_, _, diagnostics = run_clang_tidy(clang_tidy, build_dir, copy, None, [])
|
|
finally:
|
|
os.remove(copy)
|
|
return error_sites(own_errors(diagnostics, copy), shown.stdout)
|
|
|
|
|
|
def check_file(clang_tidy, build_dir, merge_base, path, change, extra_args):
|
|
"""Run clang-tidy on one file and return (failed, output, failing diagnostics, fixed)."""
|
|
fixing = any(arg.startswith("--fix") for arg in extra_args)
|
|
if fixing:
|
|
with open(path, "rb") as f:
|
|
before = f.read()
|
|
# A deleted include can orphan uses on unchanged lines, so such a file is
|
|
# checked whole and the findings narrowed here. --fix keeps a line filter so
|
|
# it never rewrites unrelated code, which for such a file means a second,
|
|
# fixing run limited to the lines the first one found wanting.
|
|
whole = bool(change.removed_includes)
|
|
returncode, output, diagnostics = run_clang_tidy(clang_tidy, build_dir, path,
|
|
None if whole else change.lines,
|
|
[] if whole else extra_args)
|
|
real = os.path.realpath(path)
|
|
|
|
def introduced(d):
|
|
return os.path.realpath(d.file) == real and (
|
|
in_ranges(d.line, change.lines) or names_removed_include(d.include, change.removed_includes))
|
|
|
|
if PurePosixPath(path).suffix in HEADER_EXTENSIONS:
|
|
# Many existing headers only compile after what their includers include
|
|
# first, so a header is held to errors it introduces: on its changed
|
|
# lines, or anywhere a deleted include leaves it with new errors.
|
|
failing = [d for d in diagnostics if introduced(d)]
|
|
errors = own_errors(diagnostics, path)
|
|
if errors and change.removed_includes:
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
before_sites = errors_alone_at(merge_base, clang_tidy, build_dir, path)
|
|
failing += [d for d in errors if d not in failing
|
|
and (error_sites([d], text) - before_sites)]
|
|
failed = bool(failing)
|
|
elif whole:
|
|
failing = [d for d in diagnostics if d.is_compile_error or introduced(d)]
|
|
failed = bool(failing)
|
|
else:
|
|
failing, failed = diagnostics, returncode != 0
|
|
if not fixing:
|
|
return failed, output, failing, False
|
|
if whole and failing:
|
|
lines = change.lines + [[d.line, d.line] for d in failing if os.path.realpath(d.file) == real]
|
|
run_clang_tidy(clang_tidy, build_dir, path, lines, extra_args)
|
|
with open(path, "rb") as f:
|
|
if f.read() == before:
|
|
return failed, output, failing, False
|
|
# Checked again, so what is reported is what the fixes left.
|
|
return check_file(clang_tidy, build_dir, merge_base, path, change, [])[:3] + (True,)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("-p", "--build-dir", required=True, help="directory holding compile_commands.json")
|
|
parser.add_argument("--base", default="origin/main", help="revision to diff against (default: origin/main)")
|
|
parser.add_argument("--clang-tidy", default="clang-tidy", help="clang-tidy executable")
|
|
parser.add_argument("-j", "--jobs", type=int, default=os.cpu_count())
|
|
parser.add_argument("extra_args", nargs="*", help="passed to clang-tidy after --, e.g. -- --fix")
|
|
args = parser.parse_args()
|
|
# A piped stdout on Windows is cp1252, which cannot encode every character clang-tidy prints.
|
|
sys.stdout.reconfigure(errors="replace")
|
|
|
|
merge_base = subprocess.run(["git", "merge-base", args.base, "HEAD"], check=True,
|
|
capture_output=True, text=True).stdout.strip()
|
|
files = changed_files(merge_base)
|
|
sources = compiled_sources(args.build_dir)
|
|
todo = []
|
|
for path, change in sorted(files.items()):
|
|
is_source = PurePosixPath(path).suffix in SOURCE_EXTENSIONS
|
|
if is_source and os.path.realpath(path) not in sources:
|
|
print(f"Skipping {path}: not compiled in this configuration")
|
|
elif change.lines or change.removed_includes:
|
|
todo.append((path, change))
|
|
if not todo:
|
|
print("No changed C++ lines to check.")
|
|
return 0
|
|
|
|
print(f"Checking {len(todo)} file(s) with {args.clang_tidy}")
|
|
annotate = os.environ.get("GITHUB_ACTIONS") == "true"
|
|
root = os.getcwd() + os.sep
|
|
failed = []
|
|
fixed = []
|
|
with ThreadPoolExecutor(max_workers=args.jobs) as pool:
|
|
jobs = {path: pool.submit(check_file, args.clang_tidy, args.build_dir, merge_base, path, change, args.extra_args)
|
|
for path, change in todo}
|
|
for path, job in jobs.items():
|
|
file_failed, output, diagnostics, file_fixed = job.result()
|
|
if file_fixed:
|
|
fixed.append(path)
|
|
if not file_failed:
|
|
continue
|
|
failed.append(path)
|
|
print(f"\n==== {path}")
|
|
if not diagnostics:
|
|
print(output, end="")
|
|
for d in diagnostics[:MAX_REPORTED]:
|
|
file = d.file.removeprefix(root)
|
|
print(f"{file}:{d.line}:{d.col}: {d.level}: {d}")
|
|
if annotate:
|
|
print(f"::{d.level} file={file},line={d.line},col={d.col}::{d}")
|
|
if len(diagnostics) > MAX_REPORTED:
|
|
print(f"... and {len(diagnostics) - MAX_REPORTED} more")
|
|
|
|
if fixed:
|
|
print(f"\nAdded includes to {len(fixed)} file(s):")
|
|
for path in fixed:
|
|
print(f" {path}")
|
|
if failed and fixed:
|
|
print(f"\nclang-tidy still fails on {len(failed)} file(s); the findings above are what --fix could not fix.")
|
|
return 1
|
|
if failed:
|
|
print(f"\nclang-tidy failed on {len(failed)} file(s). Add the includes it names, or apply its "
|
|
"suggestions locally with scripts/run_clang_tidy.sh --fix (scripts\\run_clang_tidy.ps1 -Fix on Windows). "
|
|
"Other findings need a manual fix.")
|
|
return 1
|
|
print("clang-tidy passed.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|