felhom.eu/scripts/reuse_refs_check.py

# -*- coding: utf-8 -*-
"""REUSE.md refs check — staleness defense for the per-repo reuse maps.

Usage:  python3 scripts/reuse_refs_check.py <repo-root> [<repo-root> ...]

For each repo root given, parses <root>/REUSE.md, extracts every cited file path
(*.go, *.py, *.html, *.css, *.yml, *.yaml, *.sh) and verifies the file EXISTS somewhere it
could honestly be. Only slash-containing tokens are checked — bare filenames are conventions,
not citations. Symbols are NOT checked here; those are spot-verified by the reviewer at
file:line. Exits non-zero if any cited path resolves nowhere.

RESOLUTION ORDER (2026-08-02 — operator ruling; first hit wins, and every non-exact hit is
PRINTED so a weakening of the check is visible rather than silent):

  1. exact      <root>/<token> exists                                  — no note
  2. suffix     exactly one indexed file under <root> ends with /<token>
  3. ambiguous  more than one does — still OK: the citation is real, the shorthand is
                imprecise. All matches are printed and marked AMBIGUOUS.
  4. cross-repo resolved in an immediate SIBLING repo (a sibling dir containing .git), either
                as-is or with the sibling's own name stripped from the front of the token
  5. FAIL       nowhere — prints the file, line, token, and EVERY resolution attempted

WHY THIS SHAPE. Before this, the checker demanded repo-relative paths and was RED on all four
repos: 13 findings, and a hand audit of all 13 on 2026-08-02 found **zero** genuine drift.
Twelve were package shorthand whose file sits one or two directories deeper
(`appbackup/userdata.go` → `controller/internal/appbackup/userdata.go`); one,
`wgsync/reconciler.go`, is cited by the controller's REUSE.md and lives in the HUB. REUSE.md
cites by package shorthand and across repos on purpose — that convention is the useful one, and
the tool was what was wrong. Rejected alternatives, recorded so they are not revisited:
rewriting all four REUSE.md files to full paths (makes the docs worse to serve the tool), and
deleting the checker (REUSE drift across four repos is a live risk).

THE POSITIVE OBSERVABLE. Every root prints a per-rule tally. "0 failures" alone cannot tell a
working checker from a blind one — a run that suddenly resolves everything by SUFFIX is telling
you something, and the counts are where you see it. The kill condition is pinned by
scripts/test_reuse_refs_check.py: a citation that exists nowhere still FAILS.
"""
import io, os, re, sys

# path-looking tokens ending in a checked extension; globs (*) are conventions, not refs.
# R-422: `.md` joined the list — a rotted document citation was invisible (measured 2026-09-01:
# `documentation/architecture/99-does-not-exist.md` passed). The false-positive walk across all four
# repos found ONE: an audit document, which the evidence-copy exclusion below hid. See DOC_EXT.
PATH_RE = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_./\-]*/[A-Za-z0-9_./\-]*\.(?:go|py|html|css|yml|yaml|sh|md)\b')

# An EVIDENCE COPY of a file is not the file — never let an audit or a test-findings tree satisfy
# a citation. `.git`/`vendor`/`node_modules` are excluded as noise.
EXCLUDE_NAMES = {".git", "node_modules", "vendor", "audits"}
EXCLUDE_RELPATHS = {"documentation/tests"}
# R-422: the exclusion exists so an evidence COPY of a code file cannot satisfy a citation. A document
# under audits/ or documentation/tests is not a copy of anything — it is the cited thing — so `.md`
# files in the excluded trees ARE indexed (and only they). `.git`/`vendor`/`node_modules` stay out.
DOC_EXT = ".md"
NEVER_WALK = {".git", "node_modules", "vendor"}

fails = 0
_index_cache = {}


class RepoIndex(object):
    """One walk per repo root, reused across every token and every sibling lookup."""

    def __init__(self, root):
        self.root = root
        self.name = os.path.basename(root)
        self.files = set()          # posix-style relpaths
        self.by_base = {}           # basename -> [relpath, ...]
        doc_only = set()            # excluded trees, walked for .md documents only
        for dirpath, dirs, filenames in os.walk(root):
            rel = os.path.relpath(dirpath, root).replace(os.sep, "/")
            if rel == ".":
                rel = ""
            doc_only_here = any(rel == d or rel.startswith(d + "/") for d in doc_only)
            keep = []
            for d in dirs:
                if d in NEVER_WALK:
                    continue
                drel = (rel + "/" + d).lstrip("/")
                if d in EXCLUDE_NAMES or drel in EXCLUDE_RELPATHS:
                    doc_only.add(drel)  # walked for documents only (R-422)
                keep.append(d)
            dirs[:] = keep
            for fn in filenames:
                if doc_only_here and not fn.endswith(DOC_EXT):
                    continue
                p = (rel + "/" + fn).lstrip("/")
                self.files.add(p)
                self.by_base.setdefault(fn, []).append(p)

    def exact(self, token):
        return token in self.files

    def suffix_matches(self, token):
        base = token.rsplit("/", 1)[-1]
        return sorted(p for p in self.by_base.get(base, [])
                      if p != token and p.endswith("/" + token))


def index_for(root):
    root = os.path.abspath(root)
    if root not in _index_cache:
        _index_cache[root] = RepoIndex(root)
    return _index_cache[root]


def find_siblings(root):
    """Immediate sibling dirs of <root> that are themselves git working trees. One level only.

    Returns (list_of_paths, error_message_or_None). A sibling repo that is simply absent is NEVER
    a failure — a clone in isolation must still be able to check itself.
    """
    parent = os.path.dirname(os.path.abspath(root))
    try:
        entries = sorted(os.listdir(parent))
    except OSError as e:
        return [], "parent %s not readable (%s) — siblings were NOT searched" % (parent, e)
    sibs = []
    for e in entries:
        p = os.path.join(parent, e)
        if os.path.abspath(p) == os.path.abspath(root):
            continue
        if os.path.isdir(p) and os.path.exists(os.path.join(p, ".git")):
            sibs.append(p)
    return sibs, None


def resolve(token, idx, siblings):
    """(rule, note, tried) — rule is one of exact/suffix/ambiguous/cross-repo/None."""
    tried = ["repo-relative %s/%s" % (idx.name, token)]
    if idx.exact(token):
        return "exact", "", tried

    tried.append("suffix search over %d indexed files in %s" % (len(idx.files), idx.name))
    m = idx.suffix_matches(token)
    if len(m) == 1:
        return "suffix", "resolved by suffix → %s" % m[0], tried
    if len(m) > 1:
        return "ambiguous", "AMBIGUOUS — %d matches: %s" % (len(m), ", ".join(m)), tried

    for sib in siblings:
        sidx = index_for(sib)
        # a token may carry the sibling's own repo name on the front (app-catalog's REUSE.md
        # cites `felhom.eu/scripts/site_gates.py` that way) — try both forms.
        cands = [token]
        if token.startswith(sidx.name + "/"):
            cands.append(token[len(sidx.name) + 1:])
        for cand in cands:
            tried.append("sibling %s: %s" % (sidx.name, cand))
            if sidx.exact(cand):
                return "cross-repo", "cross-repo → %s/%s" % (sidx.name, cand), tried
            sm = sidx.suffix_matches(cand)
            if len(sm) == 1:
                return "cross-repo", "cross-repo (suffix) → %s/%s" % (sidx.name, sm[0]), tried
            if len(sm) > 1:
                return "cross-repo", "cross-repo AMBIGUOUS in %s — %d matches: %s" % (
                    sidx.name, len(sm), ", ".join(sm)), tried
    return None, "", tried


def check_repo(root):
    global fails
    root = os.path.abspath(root)
    reuse = os.path.join(root, "REUSE.md")
    name = os.path.basename(root)
    if not os.path.isfile(reuse):
        print("FAIL [%s]: no REUSE.md at %s" % (name, reuse))
        fails += 1
        return
    idx = index_for(root)
    siblings, sib_err = find_siblings(root)
    if sib_err:
        # say so and continue — do NOT silently pretend siblings were searched
        print("NOTE [%s]: %s" % (name, sib_err))

    seen = set()
    tally = {"exact": 0, "suffix": 0, "ambiguous": 0, "cross-repo": 0, "failed": 0}
    with io.open(reuse, encoding="utf-8") as f:
        for lineno, line in enumerate(f, 1):
            for m in PATH_RE.finditer(line):
                p = m.group(0)
                if "*" in line[max(0, m.start() - 2):m.end() + 2]:
                    continue  # glob like scripts/*.py — a convention, not a file ref
                if p in seen:
                    continue
                seen.add(p)
                rule, note, tried = resolve(p, idx, siblings)
                if rule is None:
                    tally["failed"] += 1
                    print("FAIL [%s] line %d: cited path resolves NOWHERE: %s" % (name, lineno, p))
                    for t in tried:
                        print("        tried: %s" % t)
                    if not siblings and not sib_err:
                        print("        tried: no sibling git repos found beside %s" % name)
                else:
                    tally[rule] += 1
                    if note:
                        print("note [%s] line %d: %s  (%s)" % (name, lineno, p, note))

    print("%s [%s]: %d cited paths — exact %d, suffix %d, ambiguous %d, cross-repo %d, FAILED %d "
          "(siblings searched: %s)" % (
              "FAIL" if tally["failed"] else "OK  ", name, len(seen),
              tally["exact"], tally["suffix"], tally["ambiguous"], tally["cross-repo"],
              tally["failed"],
              ", ".join(os.path.basename(s) for s in siblings) or "none"))
    fails += tally["failed"]


def main(argv):
    if len(argv) < 1:
        print(__doc__)
        return 2
    for r in argv:
        check_repo(r)
    return 1 if fails else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))