app-catalog-felhom.eu/scripts/ladder_backfill.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""ladder_backfill.py — ONE-OFF (night 2026-09-23): the first ladder entry for every image move that
went live BEFORE the test-record gate existed, written from the verdict record that move cited.

    python3 scripts/ladder_backfill.py --workspace /mnt/5_hdd/felhom.eu/git [--write] <commit> ...

For each commit: the app is the one template whose images it moved; `from`/`to` are the per-service
images at <commit>~1 and <commit>; the evidence is the `Evidence:` path in the commit message. The
entry is `proven` ONLY when that record exists, says `proven`, and names the same `to` refs;
otherwise it is written `unrecorded` and NAMED — never invented. Every entry carries
`backfilled: <today>` and the digest the registry serves for its refs TODAY (decision 17 has no
earlier measurement to cite; the entry says so in `note`). An `extra` record may be given per app
(`--extra app=path`) for a later measurement of the SAME step — romm's memory watch (M1) is one. A
commit that cited no record may be given one (`--evidence app=path`); it is used only if its verdict
and refs match, and the entry says the backfill named it.

It refuses to write an app whose compose no longer stands at the move's `to` (a later move would
need its own entry first) — the static gate would convict that ladder anyway.
"""
import datetime
import json
import os
import re
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import image_digest  # noqa: E402
import ladder  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def git(*a):
    return subprocess.run(["git", "-C", ROOT] + list(a), capture_output=True, text=True)


def main(argv):
    ws = argv[argv.index("--workspace") + 1]
    write = "--write" in argv
    extra, cite = {}, {}
    for i, a in enumerate(argv):
        if a == "--extra":
            k, _, v = argv[i + 1].partition("=")
            extra[k] = v
        if a == "--evidence":
            k, _, v = argv[i + 1].partition("=")
            cite[k] = v
    commits = [a for i, a in enumerate(argv) if re.match(r"^[0-9a-f]{7,40}$", a)]
    today = datetime.date.today().isoformat()
    rows, rc = [], 0
    for c in commits:
        files = [f for f in git("show", "--name-only", "--format=", c).stdout.split()
                 if re.match(r"^templates/[^/]+/docker-compose\.yml$", f)]
        if len(files) != 1:
            print("SKIP %s: moves %d templates, expected exactly one" % (c, len(files)))
            rc = 1
            continue
        app = files[0].split("/")[1]
        frm = ladder.images_in(git("show", "%s~1:%s" % (c, files[0])).stdout)
        to = ladder.images_in(git("show", "%s:%s" % (c, files[0])).stdout)
        cur = ladder.images_in(open(os.path.join(ROOT, files[0]), encoding="utf-8").read())
        if cur != to:
            print("REFUSE %s (%s): the compose has moved since (%s) — not backfilled" % (app, c, cur))
            rc = 1
            continue
        body = git("show", "-s", "--format=%B", c).stdout
        m = re.search(r"Evidence:\s*(\S+verdict\.json)", body)
        ev = m.group(1) if m else None
        cited_here = False
        if ev is None and app in cite:
            ev, cited_here = cite[app], True
        verdict, why, tested_at = "unrecorded", None, None
        if ev and os.path.exists(os.path.join(ws, ev)):
            rec = json.load(open(os.path.join(ws, ev)))
            rto = {k: v for k, v in (rec.get("to") or {}).items() if k in to}
            if rec.get("verdict") == "proven" and all(to.get(k) == v for k, v in rto.items()) and rto:
                verdict, tested_at = "proven", rec.get("measured_at")
            else:
                why = "the cited record says verdict=%r to=%r" % (rec.get("verdict"), rec.get("to"))
        else:
            why = "no verdict record found (commit cites %r)" % ev
        digests = {}
        for svc, ref in sorted(to.items()):
            d, err = image_digest.resolve(ref)
            if not d:
                print("INCONCLUSIVE %s: %s %s: %s" % (app, svc, ref, err))
                return 2
            digests[svc] = d
        if tested_at and not ladder.TS_RE.match(tested_at):
            tested_at = tested_at.split(".")[0].replace("+00:00", "") + "Z"
        e = {"from": frm, "to": to, "digest": digests, "verdict": verdict, "tested_at": tested_at,
             "harness_version": 1 if verdict == "proven" else None, "evidence": ev,
             "memory_peak_pct": None,
             "marks": {"files_may_change": False, "needs_person": None, "memory_tight": False},
             "backfilled": today,
             "note": "backfilled from catalog commit %s; box walk only (harness v1, no memory watch); "
                     "digest = what the registry served on %s, not a measurement of the tested image"
                     % (c, today)}
        if why:
            e["note"] += "; UNRECORDED because " + why
        if cited_here:
            e["note"] += ("; the commit cited no record — this one was named by the backfill because its "
                          "from/to refs are the commit's and the commit describes the same walk")
        if app in extra:
            x = json.load(open(os.path.join(ws, extra[app])))
            peaks = [p.get("peak_pct") for p in ((x.get("memory") or {}).get("containers") or {}).values()
                     if isinstance(p.get("peak_pct"), (int, float))]
            if x.get("verdict") == "proven" and peaks and x.get("to", {}).get(next(iter(x["to"]))) == \
                    to.get(next(iter(x["to"]))):
                pk = round(max(peaks) * 100, 1)
                e["memory_peak_pct"] = pk
                e["marks"]["memory_tight"] = pk > ladder.MEMORY_TIGHT_PCT
                e["note"] += "; memory watch from %s (bench, harness v%s): peak %s%%" % (
                    extra[app], x.get("harness_version"), pk)
        probs = ladder.check_entry(e)
        if probs:
            print("REFUSE %s: entry not well-formed: %s" % (app, probs))
            rc = 1
            continue
        rows.append((app, c, verdict, ev, why))
        if write:
            p = os.path.join(ROOT, "templates", app, ".felhom.yml")
            text = open(p, encoding="utf-8").read()
            if "update_ladder:" in text:
                print("REFUSE %s: already carries a ladder" % app)
                rc = 1
                continue
            open(p, "w", encoding="utf-8").write(ladder.append_entry(text, e))
    for app, c, v, ev, why in rows:
        print("%-16s %s %-10s %s%s" % (app, c, v, ev or "-", ("  (" + why + ")") if why else ""))
    print("%d backfilled (%d proven, %d unrecorded)%s" % (
        len(rows), sum(1 for r in rows if r[2] == "proven"), sum(1 for r in rows if r[2] != "proven"),
        "" if write else " — DRY RUN, nothing written"))
    return rc


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))