diff --git a/scripts/reuse_refs_check.py b/scripts/reuse_refs_check.py index 8ef20d8..a32caca 100644 --- a/scripts/reuse_refs_check.py +++ b/scripts/reuse_refs_check.py @@ -1,21 +1,142 @@ # -*- coding: utf-8 -*- """REUSE.md refs check — staleness defense for the per-repo reuse maps. -Usage: python scripts/reuse_refs_check.py [ ...] +Usage: python3 scripts/reuse_refs_check.py [ ...] For each repo root given, parses /REUSE.md, extracts every cited file path -(*.go, *.py, *.html, *.css, *.yml, *.yaml, *.sh) and verifies the file exists in -the tree. Only slash-containing (repo-relative) tokens are checked — bare filenames -are conventions, not citations. Prints offending lines; exits non-zero if any cited -path is missing. Symbols are NOT checked here — those are spot-verified by the -reviewer at file:line. +(*.go, *.py, *.html, *.css, *.yml, *.yaml, *.sh) and verifies the file EXISTS somewhere it +could honestly be. Only slash-containing tokens are checked — bare filenames are conventions, +not citations. Symbols are NOT checked here; those are spot-verified by the reviewer at +file:line. Exits non-zero if any cited path resolves nowhere. + +RESOLUTION ORDER (2026-08-02 — operator ruling; first hit wins, and every non-exact hit is +PRINTED so a weakening of the check is visible rather than silent): + + 1. exact / exists — no note + 2. suffix exactly one indexed file under ends with / + 3. ambiguous more than one does — still OK: the citation is real, the shorthand is + imprecise. All matches are printed and marked AMBIGUOUS. + 4. cross-repo resolved in an immediate SIBLING repo (a sibling dir containing .git), either + as-is or with the sibling's own name stripped from the front of the token + 5. FAIL nowhere — prints the file, line, token, and EVERY resolution attempted + +WHY THIS SHAPE. Before this, the checker demanded repo-relative paths and was RED on all four +repos: 13 findings, and a hand audit of all 13 on 2026-08-02 found **zero** genuine drift. +Twelve were package shorthand whose file sits one or two directories deeper +(`appbackup/userdata.go` → `controller/internal/appbackup/userdata.go`); one, +`wgsync/reconciler.go`, is cited by the controller's REUSE.md and lives in the HUB. REUSE.md +cites by package shorthand and across repos on purpose — that convention is the useful one, and +the tool was what was wrong. Rejected alternatives, recorded so they are not revisited: +rewriting all four REUSE.md files to full paths (makes the docs worse to serve the tool), and +deleting the checker (REUSE drift across four repos is a live risk). + +THE POSITIVE OBSERVABLE. Every root prints a per-rule tally. "0 failures" alone cannot tell a +working checker from a blind one — a run that suddenly resolves everything by SUFFIX is telling +you something, and the counts are where you see it. The kill condition is pinned by +scripts/test_reuse_refs_check.py: a citation that exists nowhere still FAILS. """ import io, os, re, sys # path-looking tokens ending in a checked extension; globs (*) are conventions, not refs PATH_RE = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_./\-]*/[A-Za-z0-9_./\-]*\.(?:go|py|html|css|yml|yaml|sh)\b') +# An EVIDENCE COPY of a file is not the file — never let an audit or a test-findings tree satisfy +# a citation. `.git`/`vendor`/`node_modules` are excluded as noise. +EXCLUDE_NAMES = {".git", "node_modules", "vendor", "audits"} +EXCLUDE_RELPATHS = {"documentation/tests"} + fails = 0 +_index_cache = {} + + +class RepoIndex(object): + """One walk per repo root, reused across every token and every sibling lookup.""" + + def __init__(self, root): + self.root = root + self.name = os.path.basename(root) + self.files = set() # posix-style relpaths + self.by_base = {} # basename -> [relpath, ...] + for dirpath, dirs, filenames in os.walk(root): + rel = os.path.relpath(dirpath, root).replace(os.sep, "/") + if rel == ".": + rel = "" + dirs[:] = [d for d in dirs + if d not in EXCLUDE_NAMES + and ((rel + "/" + d).lstrip("/") not in EXCLUDE_RELPATHS)] + for fn in filenames: + p = (rel + "/" + fn).lstrip("/") + self.files.add(p) + self.by_base.setdefault(fn, []).append(p) + + def exact(self, token): + return token in self.files + + def suffix_matches(self, token): + base = token.rsplit("/", 1)[-1] + return sorted(p for p in self.by_base.get(base, []) + if p != token and p.endswith("/" + token)) + + +def index_for(root): + root = os.path.abspath(root) + if root not in _index_cache: + _index_cache[root] = RepoIndex(root) + return _index_cache[root] + + +def find_siblings(root): + """Immediate sibling dirs of that are themselves git working trees. One level only. + + Returns (list_of_paths, error_message_or_None). A sibling repo that is simply absent is NEVER + a failure — a clone in isolation must still be able to check itself. + """ + parent = os.path.dirname(os.path.abspath(root)) + try: + entries = sorted(os.listdir(parent)) + except OSError as e: + return [], "parent %s not readable (%s) — siblings were NOT searched" % (parent, e) + sibs = [] + for e in entries: + p = os.path.join(parent, e) + if os.path.abspath(p) == os.path.abspath(root): + continue + if os.path.isdir(p) and os.path.exists(os.path.join(p, ".git")): + sibs.append(p) + return sibs, None + + +def resolve(token, idx, siblings): + """(rule, note, tried) — rule is one of exact/suffix/ambiguous/cross-repo/None.""" + tried = ["repo-relative %s/%s" % (idx.name, token)] + if idx.exact(token): + return "exact", "", tried + + tried.append("suffix search over %d indexed files in %s" % (len(idx.files), idx.name)) + m = idx.suffix_matches(token) + if len(m) == 1: + return "suffix", "resolved by suffix → %s" % m[0], tried + if len(m) > 1: + return "ambiguous", "AMBIGUOUS — %d matches: %s" % (len(m), ", ".join(m)), tried + + for sib in siblings: + sidx = index_for(sib) + # a token may carry the sibling's own repo name on the front (app-catalog's REUSE.md + # cites `felhom.eu/scripts/site_gates.py` that way) — try both forms. + cands = [token] + if token.startswith(sidx.name + "/"): + cands.append(token[len(sidx.name) + 1:]) + for cand in cands: + tried.append("sibling %s: %s" % (sidx.name, cand)) + if sidx.exact(cand): + return "cross-repo", "cross-repo → %s/%s" % (sidx.name, cand), tried + sm = sidx.suffix_matches(cand) + if len(sm) == 1: + return "cross-repo", "cross-repo (suffix) → %s/%s" % (sidx.name, sm[0]), tried + if len(sm) > 1: + return "cross-repo", "cross-repo AMBIGUOUS in %s — %d matches: %s" % ( + sidx.name, len(sm), ", ".join(sm)), tried + return None, "", tried def check_repo(root): @@ -27,7 +148,14 @@ def check_repo(root): print("FAIL [%s]: no REUSE.md at %s" % (name, reuse)) fails += 1 return - seen, missing = set(), 0 + idx = index_for(root) + siblings, sib_err = find_siblings(root) + if sib_err: + # say so and continue — do NOT silently pretend siblings were searched + print("NOTE [%s]: %s" % (name, sib_err)) + + seen = set() + tally = {"exact": 0, "suffix": 0, "ambiguous": 0, "cross-repo": 0, "failed": 0} with io.open(reuse, encoding="utf-8") as f: for lineno, line in enumerate(f, 1): for m in PATH_RE.finditer(line): @@ -37,18 +165,36 @@ def check_repo(root): if p in seen: continue seen.add(p) - if not os.path.isfile(os.path.join(root, p)): - print("FAIL [%s] line %d: cited path missing: %s" % (name, lineno, p)) - missing += 1 - if missing: - fails += missing - else: - print("OK [%s]: %d cited paths, all exist" % (name, len(seen))) + rule, note, tried = resolve(p, idx, siblings) + if rule is None: + tally["failed"] += 1 + print("FAIL [%s] line %d: cited path resolves NOWHERE: %s" % (name, lineno, p)) + for t in tried: + print(" tried: %s" % t) + if not siblings and not sib_err: + print(" tried: no sibling git repos found beside %s" % name) + else: + tally[rule] += 1 + if note: + print("note [%s] line %d: %s (%s)" % (name, lineno, p, note)) + + print("%s [%s]: %d cited paths — exact %d, suffix %d, ambiguous %d, cross-repo %d, FAILED %d " + "(siblings searched: %s)" % ( + "FAIL" if tally["failed"] else "OK ", name, len(seen), + tally["exact"], tally["suffix"], tally["ambiguous"], tally["cross-repo"], + tally["failed"], + ", ".join(os.path.basename(s) for s in siblings) or "none")) + fails += tally["failed"] -if len(sys.argv) < 2: - print(__doc__) - sys.exit(2) -for r in sys.argv[1:]: - check_repo(r) -sys.exit(1 if fails else 0) +def main(argv): + if len(argv) < 1: + print(__doc__) + return 2 + for r in argv: + check_repo(r) + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_reuse_refs_check.py b/scripts/test_reuse_refs_check.py new file mode 100644 index 0000000..167f4a7 --- /dev/null +++ b/scripts/test_reuse_refs_check.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +"""Fixture tests for scripts/reuse_refs_check.py — one per row of its resolution table, plus the +kill condition. + +Run: python3 scripts/test_reuse_refs_check.py + +THE ONE THAT MATTERS is test_absent_path_fails (Scenario E). The 2026-08-02 change taught the +checker to resolve suffixes and sibling repos, which turned 13 findings green in one step. A +checker made green by being made BLIND is a failure this project has shipped before, so the +ability to still fail is pinned here, and the assertion is on the EXIT CODE — the effect — not on +the summary text, which the checker can print without having decided anything. +""" +import importlib.util +import io +import os +import shutil +import sys +import tempfile +import unittest + +SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "reuse_refs_check.py") + + +def load_checker(): + """Fresh module per test — the checker keeps a global failure count and an index cache.""" + spec = importlib.util.spec_from_file_location("reuse_refs_check_under_test", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def write(path, text=""): + d = os.path.dirname(path) + if d and not os.path.isdir(d): + os.makedirs(d) + with io.open(path, "w", encoding="utf-8") as f: + f.write(text) + + +class ReuseRefsCheckTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="reuse-refs-") + self.root = os.path.join(self.tmp, "myrepo") + write(os.path.join(self.root, ".git"), "gitdir: elsewhere\n") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + # ── harness ────────────────────────────────────────────────────────────────── + def run_check(self, *roots): + """Returns (exit_code, stdout). Exit code is the assertion that counts.""" + mod = load_checker() + buf = io.StringIO() + real = sys.stdout + sys.stdout = buf + try: + rc = mod.main(list(roots) or [self.root]) + finally: + sys.stdout = real + return rc, buf.getvalue() + + def reuse(self, body): + write(os.path.join(self.root, "REUSE.md"), body) + + def sibling(self, name): + p = os.path.join(self.tmp, name) + write(os.path.join(p, ".git"), "gitdir: elsewhere\n") + return p + + # ── row 1: exact ───────────────────────────────────────────────────────────── + def test_exact_match_is_silent_and_passes(self): + write(os.path.join(self.root, "a", "b.go"), "package a\n") + self.reuse("see `a/b.go` for the thing\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("exact 1", out) + self.assertNotIn("note [", out) # an exact hit prints no note + + # ── row 2: suffix ──────────────────────────────────────────────────────────── + def test_package_shorthand_resolves_by_suffix(self): + write(os.path.join(self.root, "controller", "internal", "pkg", "x.go"), "package pkg\n") + self.reuse("see `pkg/x.go`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("resolved by suffix", out) + self.assertIn("controller/internal/pkg/x.go", out) + self.assertIn("suffix 1", out) + + # ── row 3: ambiguous — real citation, imprecise shorthand; NOT a failure ───── + def test_two_suffix_matches_are_ambiguous_not_fatal(self): + write(os.path.join(self.root, "one", "pkg", "x.go"), "package pkg\n") + write(os.path.join(self.root, "two", "pkg", "x.go"), "package pkg\n") + self.reuse("see `pkg/x.go`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("AMBIGUOUS", out) + self.assertIn("one/pkg/x.go", out) + self.assertIn("two/pkg/x.go", out) + self.assertIn("ambiguous 1", out) + + # ── row 4: cross-repo, by suffix in a sibling ──────────────────────────────── + def test_sibling_repo_resolution(self): + sib = self.sibling("otherrepo") + write(os.path.join(sib, "hub", "internal", "wgsync", "reconciler.go"), "package wgsync\n") + self.reuse("see `wgsync/reconciler.go`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("cross-repo", out) + self.assertIn("otherrepo", out) + self.assertIn("cross-repo 1", out) + + # ── row 4b: cross-repo where the token CARRIES the sibling's repo name ─────── + def test_sibling_repo_name_prefixed_token(self): + sib = self.sibling("felhom.eu") + write(os.path.join(sib, "scripts", "site_gates.py"), "# gate\n") + self.reuse("run `felhom.eu/scripts/site_gates.py`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("cross-repo", out) + + def test_non_git_sibling_is_not_searched(self): + plain = os.path.join(self.tmp, "notarepo") # no .git — not a repo, must not resolve + write(os.path.join(plain, "pkg", "x.go"), "package pkg\n") + self.reuse("see `pkg/x.go`\n") + rc, out = self.run_check() + self.assertNotEqual(rc, 0, out) + + # ── row 5: THE KILL CONDITION (Scenario E) ────────────────────────────────── + def test_absent_path_fails(self): + write(os.path.join(self.root, "internal", "present.go"), "package internal\n") + self.reuse("line one\nsee `internal/definitely_absent_xyz.go`\n") + rc, out = self.run_check() + self.assertNotEqual(rc, 0, "a citation that exists NOWHERE must fail:\n" + out) + self.assertIn("definitely_absent_xyz.go", out) + self.assertIn("line 2", out) # names the line + self.assertIn("FAILED 1", out) + + def test_failure_lists_every_resolution_attempted(self): + """CLAUDE.md standing rule: a 'not found' claim must name what was tried.""" + self.sibling("otherrepo") + self.reuse("see `internal/definitely_absent_xyz.go`\n") + rc, out = self.run_check() + self.assertNotEqual(rc, 0, out) + self.assertIn("tried: repo-relative myrepo/internal/definitely_absent_xyz.go", out) + self.assertIn("tried: suffix search over", out) + self.assertIn("tried: sibling otherrepo", out) + + # ── evidence trees are not the file ───────────────────────────────────────── + def test_evidence_copy_does_not_satisfy_a_citation(self): + write(os.path.join(self.root, "documentation", "audits", "pkg", "x.go"), "package pkg\n") + write(os.path.join(self.root, "documentation", "tests", "pkg", "y.go"), "package pkg\n") + self.reuse("see `pkg/x.go` and `pkg/y.go`\n") + rc, out = self.run_check() + self.assertNotEqual(rc, 0, "an audits/ or documentation/tests/ copy must NOT resolve:\n" + out) + self.assertIn("FAILED 2", out) + + # ── a clone in isolation must still check itself ──────────────────────────── + def test_no_siblings_is_not_a_failure(self): + write(os.path.join(self.root, "a", "b.go"), "package a\n") + self.reuse("see `a/b.go`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("siblings searched: none", out) + + def test_missing_reuse_md_fails(self): + rc, out = self.run_check() + self.assertNotEqual(rc, 0, out) + self.assertIn("no REUSE.md", out) + + # ── globs stay conventions, not refs ──────────────────────────────────────── + def test_glob_is_not_a_citation(self): + self.reuse("the gates are `scripts/*.py`\n") + rc, out = self.run_check() + self.assertEqual(rc, 0, out) + self.assertIn("0 cited paths", out) + + # ── no args → usage, exit 2 ───────────────────────────────────────────────── + def test_no_args_is_usage_exit_2(self): + mod = load_checker() + buf, real = io.StringIO(), sys.stdout + sys.stdout = buf + try: + rc = mod.main([]) + finally: + sys.stdout = real + self.assertEqual(rc, 2) + + +if __name__ == "__main__": + unittest.main(verbosity=2)