# -*- coding: utf-8 -*- """REUSE.md refs check — staleness defense for the per-repo reuse maps. Usage: python3 scripts/reuse_refs_check.py [ ...] For each repo root given, parses /REUSE.md, extracts every cited file path (*.go, *.py, *.html, *.css, *.yml, *.yaml, *.sh) and verifies the file EXISTS somewhere it could honestly be. Only slash-containing tokens are checked — bare filenames are conventions, not citations. Symbols are NOT checked here; those are spot-verified by the reviewer at file:line. Exits non-zero if any cited path resolves nowhere. RESOLUTION ORDER (2026-08-02 — operator ruling; first hit wins, and every non-exact hit is PRINTED so a weakening of the check is visible rather than silent): 1. exact / exists — no note 2. suffix exactly one indexed file under ends with / 3. ambiguous more than one does — still OK: the citation is real, the shorthand is imprecise. All matches are printed and marked AMBIGUOUS. 4. cross-repo resolved in an immediate SIBLING repo (a sibling dir containing .git), either as-is or with the sibling's own name stripped from the front of the token 5. FAIL nowhere — prints the file, line, token, and EVERY resolution attempted WHY THIS SHAPE. Before this, the checker demanded repo-relative paths and was RED on all four repos: 13 findings, and a hand audit of all 13 on 2026-08-02 found **zero** genuine drift. Twelve were package shorthand whose file sits one or two directories deeper (`appbackup/userdata.go` → `controller/internal/appbackup/userdata.go`); one, `wgsync/reconciler.go`, is cited by the controller's REUSE.md and lives in the HUB. REUSE.md cites by package shorthand and across repos on purpose — that convention is the useful one, and the tool was what was wrong. Rejected alternatives, recorded so they are not revisited: rewriting all four REUSE.md files to full paths (makes the docs worse to serve the tool), and deleting the checker (REUSE drift across four repos is a live risk). THE POSITIVE OBSERVABLE. Every root prints a per-rule tally. "0 failures" alone cannot tell a working checker from a blind one — a run that suddenly resolves everything by SUFFIX is telling you something, and the counts are where you see it. The kill condition is pinned by scripts/test_reuse_refs_check.py: a citation that exists nowhere still FAILS. """ import io, os, re, sys # path-looking tokens ending in a checked extension; globs (*) are conventions, not refs PATH_RE = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_./\-]*/[A-Za-z0-9_./\-]*\.(?:go|py|html|css|yml|yaml|sh)\b') # An EVIDENCE COPY of a file is not the file — never let an audit or a test-findings tree satisfy # a citation. `.git`/`vendor`/`node_modules` are excluded as noise. EXCLUDE_NAMES = {".git", "node_modules", "vendor", "audits"} EXCLUDE_RELPATHS = {"documentation/tests"} fails = 0 _index_cache = {} class RepoIndex(object): """One walk per repo root, reused across every token and every sibling lookup.""" def __init__(self, root): self.root = root self.name = os.path.basename(root) self.files = set() # posix-style relpaths self.by_base = {} # basename -> [relpath, ...] for dirpath, dirs, filenames in os.walk(root): rel = os.path.relpath(dirpath, root).replace(os.sep, "/") if rel == ".": rel = "" dirs[:] = [d for d in dirs if d not in EXCLUDE_NAMES and ((rel + "/" + d).lstrip("/") not in EXCLUDE_RELPATHS)] for fn in filenames: p = (rel + "/" + fn).lstrip("/") self.files.add(p) self.by_base.setdefault(fn, []).append(p) def exact(self, token): return token in self.files def suffix_matches(self, token): base = token.rsplit("/", 1)[-1] return sorted(p for p in self.by_base.get(base, []) if p != token and p.endswith("/" + token)) def index_for(root): root = os.path.abspath(root) if root not in _index_cache: _index_cache[root] = RepoIndex(root) return _index_cache[root] def find_siblings(root): """Immediate sibling dirs of that are themselves git working trees. One level only. Returns (list_of_paths, error_message_or_None). A sibling repo that is simply absent is NEVER a failure — a clone in isolation must still be able to check itself. """ parent = os.path.dirname(os.path.abspath(root)) try: entries = sorted(os.listdir(parent)) except OSError as e: return [], "parent %s not readable (%s) — siblings were NOT searched" % (parent, e) sibs = [] for e in entries: p = os.path.join(parent, e) if os.path.abspath(p) == os.path.abspath(root): continue if os.path.isdir(p) and os.path.exists(os.path.join(p, ".git")): sibs.append(p) return sibs, None def resolve(token, idx, siblings): """(rule, note, tried) — rule is one of exact/suffix/ambiguous/cross-repo/None.""" tried = ["repo-relative %s/%s" % (idx.name, token)] if idx.exact(token): return "exact", "", tried tried.append("suffix search over %d indexed files in %s" % (len(idx.files), idx.name)) m = idx.suffix_matches(token) if len(m) == 1: return "suffix", "resolved by suffix → %s" % m[0], tried if len(m) > 1: return "ambiguous", "AMBIGUOUS — %d matches: %s" % (len(m), ", ".join(m)), tried for sib in siblings: sidx = index_for(sib) # a token may carry the sibling's own repo name on the front (app-catalog's REUSE.md # cites `felhom.eu/scripts/site_gates.py` that way) — try both forms. cands = [token] if token.startswith(sidx.name + "/"): cands.append(token[len(sidx.name) + 1:]) for cand in cands: tried.append("sibling %s: %s" % (sidx.name, cand)) if sidx.exact(cand): return "cross-repo", "cross-repo → %s/%s" % (sidx.name, cand), tried sm = sidx.suffix_matches(cand) if len(sm) == 1: return "cross-repo", "cross-repo (suffix) → %s/%s" % (sidx.name, sm[0]), tried if len(sm) > 1: return "cross-repo", "cross-repo AMBIGUOUS in %s — %d matches: %s" % ( sidx.name, len(sm), ", ".join(sm)), tried return None, "", tried def check_repo(root): global fails root = os.path.abspath(root) reuse = os.path.join(root, "REUSE.md") name = os.path.basename(root) if not os.path.isfile(reuse): print("FAIL [%s]: no REUSE.md at %s" % (name, reuse)) fails += 1 return idx = index_for(root) siblings, sib_err = find_siblings(root) if sib_err: # say so and continue — do NOT silently pretend siblings were searched print("NOTE [%s]: %s" % (name, sib_err)) seen = set() tally = {"exact": 0, "suffix": 0, "ambiguous": 0, "cross-repo": 0, "failed": 0} with io.open(reuse, encoding="utf-8") as f: for lineno, line in enumerate(f, 1): for m in PATH_RE.finditer(line): p = m.group(0) if "*" in line[max(0, m.start() - 2):m.end() + 2]: continue # glob like scripts/*.py — a convention, not a file ref if p in seen: continue seen.add(p) rule, note, tried = resolve(p, idx, siblings) if rule is None: tally["failed"] += 1 print("FAIL [%s] line %d: cited path resolves NOWHERE: %s" % (name, lineno, p)) for t in tried: print(" tried: %s" % t) if not siblings and not sib_err: print(" tried: no sibling git repos found beside %s" % name) else: tally[rule] += 1 if note: print("note [%s] line %d: %s (%s)" % (name, lineno, p, note)) print("%s [%s]: %d cited paths — exact %d, suffix %d, ambiguous %d, cross-repo %d, FAILED %d " "(siblings searched: %s)" % ( "FAIL" if tally["failed"] else "OK ", name, len(seen), tally["exact"], tally["suffix"], tally["ambiguous"], tally["cross-repo"], tally["failed"], ", ".join(os.path.basename(s) for s in siblings) or "none")) fails += tally["failed"] def main(argv): if len(argv) < 1: print(__doc__) return 2 for r in argv: check_repo(r) return 1 if fails else 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))