test record: an image move must carry its proof (09 decision 13, part 4)
gates / gates (push) Successful in 1s

update_ladder: in .felhom.yml, one JSON entry per line (spiked live on
controller v0.266.0 and v0.267.0 first). Two gates: check-test-record.py
(static, CI too) and check-test-record-move.py (history + registry for
moved refs only). 16 decoys, 3 red-proofs. The ONLY writer is
upgrade-test.py --write-ladder (bench AND box proven, digests resolved).
Harness v3: box fixtures on the bench, files_may_change.
Backfill: the 21 moves of 2026-09-22, 21 proven from their records.
No image: line moved.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-23 20:52:32 +02:00
parent cfcfe52784
commit 6db08a5eb3
38 changed files with 2944 additions and 37 deletions
+122
View File
@@ -51,6 +51,8 @@ COVERS = {
"probe-matches-compose": "the probe TARGET resolves by exact name, explicit `container`, or a UNIQUE prefix - an ambiguity is refused, not guessed (R-630); the DEGRADED no-PyYAML mode CI actually runs; the port/path moved in a COMMENT, in traefik's loadbalancer label, in "
"`ports:`/`expose:`, or on a NON-probed service - none of which is where "
"the app listens; vs a real probe port/path that the app does not answer (R-618)",
"test-record": "a ladder whose newest step is not the compose's images (a move without a record), a gap, a line that is not one JSON entry, a failed verdict - vs a clean ladder (09 decision 13)",
"test-record-move": "an image move with NO entry, with the entry only in a COMMENT or in README, with a failed/backfilled entry, with a digest the registry no longer serves, memory_tight without a raised limit - vs a proven entry that matches; a ref moving in a compose COMMENT is not a move (09 decision 13)",
"copy-i18n": "Hungarian edited in a COMMENT/README/display_name (label, not copy) vs a real frozen string changed; an English block that is not English, is not matched to a Hungarian twin, or rewrites a credential (R-560). Also the DEGRADED mode CI actually runs — PyYAML shadowed out, freeze only (R-595)",
}
@@ -280,6 +282,124 @@ def swap_image(service, frm, to):
return _fn
# ── test record (09 §3 decision 13) ────────────────────────────────────────────────────────────
TR_D1 = "sha256:" + "a" * 64
TR_D2 = "sha256:" + "b" * 64
def tr_entry(frm, to, digest, verdict="proven", peak=41.0, tight=False, **extra):
import json as _j
e = {"from": frm, "to": to, "digest": digest, "verdict": verdict,
"tested_at": "2026-09-23T22:00:00Z", "harness_version": 2,
"evidence": "felhom.eu/documentation/audits/night-2026-09-23/apps/x/", "memory_peak_pct": peak,
"marks": {"files_may_change": False, "needs_person": None, "memory_tight": tight}}
e.update(extra)
return " - " + _j.dumps(e)
def tr_append(line, header=True):
def _fn(t):
block = ("\nupdate_ladder:\n" if header else "") + line + "\n"
return t.rstrip("\n") + "\n" + block
return _fn
def case_tr_move(name, clone, edits, expect_rc, must_contain=(), table=None):
import json as _j
tf = os.path.join(clone, "..", os.path.basename(clone) + "-digests.json")
_j.dump(table or {}, open(tf, "w"))
global ran
ran += 1
base = sh(["git", "rev-parse", "HEAD"], cwd=clone).stdout.strip()
try:
for relpath, fn in edits:
edit(clone, relpath, fn)
commit(clone, name)
r = sh([sys.executable, os.path.join(ROOT, "scripts", "check-test-record-move.py"),
"--range", "HEAD~1..HEAD", "--digests-from", tf], cwd=clone)
out = r.stdout + r.stderr
ok = r.returncode == expect_rc and all(m in out for m in must_contain)
print(" %s %-52s rc=%d (expected %d)" % ("ok" if ok else "XX", name, r.returncode, expect_rc))
if not ok:
fails.append("%s: rc=%d expected %d; missing %s\n%s" % (
name, r.returncode, expect_rc, [m for m in must_contain if m not in out], out[-900:]))
finally:
sh(["git", "reset", "-q", "--hard", base], cwd=clone)
os.remove(tf)
def case_tr_static(name, clone, edits, expect_rc, must_contain=(), apps=("navidrome",)):
global ran
ran += 1
try:
for relpath, fn in edits:
edit(clone, relpath, fn)
r = sh([sys.executable, os.path.join(ROOT, "scripts", "check-test-record.py"),
"--root", clone] + list(apps), cwd=clone)
out = r.stdout + r.stderr
ok = r.returncode == expect_rc and all(m in out for m in must_contain)
print(" %s %-52s rc=%d (expected %d)" % ("ok" if ok else "XX", name, r.returncode, expect_rc))
if not ok:
fails.append("%s: rc=%d expected %d; missing %s\n%s" % (
name, r.returncode, expect_rc, [m for m in must_contain if m not in out], out[-900:]))
finally:
reset(clone)
def test_record_cases(clone):
NC, NF = "templates/navidrome/docker-compose.yml", "templates/navidrome/.felhom.yml"
old, new = "deluan/navidrome:0.64.0", "deluan/navidrome:0.64.1"
frm, to = {"navidrome": old}, {"navidrome": new}
move = (NC, swap_image("navidrome", old, new))
good = tr_entry(frm, to, {"navidrome": TR_D1})
table = {new: TR_D1}
print("-- test-record-move: an image move needs its own proven record")
case_tr_move("FACT: a bare image move, no entry", clone, [move], 1,
("adds NO update_ladder entry",), table)
case_tr_move("GENUINE: a proven entry whose digest the registry serves", clone,
[move, (NF, tr_append(good))], 0, ("0.64.1" if False else "test-record-move gate",), table)
case_tr_move("FACT: the entry's verdict is failed", clone,
[move, (NF, tr_append(tr_entry(frm, to, {"navidrome": TR_D1}, verdict="failed")))], 1,
("not allowed in a ladder",), table)
case_tr_move("FACT: the registry now serves another digest", clone,
[move, (NF, tr_append(good))], 1, ("the registry serves",), {new: TR_D2})
case_tr_move("INCONCLUSIVE: the registry cannot be asked", clone,
[move, (NF, tr_append(good))], 2, ("could not be asked",), {})
case_tr_move("DECOY: the entry only in a COMMENT under update_ladder", clone,
[move, (NF, lambda t: t.rstrip("\n") + "\nupdate_ladder:\n # " + good.strip() + "\n")], 1,
("holds no entry",), table)
case_tr_move("DECOY: the entry only in README.md", clone,
[move, ("README.md", lambda t: t + "\n" + good + "\n")], 1,
("adds NO update_ladder entry",), table)
case_tr_move("FACT: a new move carrying a BACKFILLED entry", clone,
[move, (NF, tr_append(tr_entry(frm, to, {"navidrome": TR_D1}, backfilled="2026-09-23")))], 1,
("marked backfilled",), table)
case_tr_move("FACT: memory_tight and the limit did not move", clone,
[move, (NF, tr_append(tr_entry(frm, to, {"navidrome": TR_D1}, peak=86.0, tight=True)))], 1,
("memory_tight",), table)
case_tr_move("GENUINE: memory_tight WITH the limit raised", clone,
[move, (NC, lambda t: t.replace("memory: 256M", "memory: 384M")),
(NF, tr_append(tr_entry(frm, to, {"navidrome": TR_D1}, peak=86.0, tight=True)))], 0,
(), table)
case_tr_move("DECOY: a ref moves in a compose COMMENT only", clone,
[(NC, lambda t: t + "\n# was: deluan/navidrome:0.63.2\n")], 0, (), table)
print("-- test-record (static): the newest step IS the compose")
case_tr_static("GENUINE: a ladder whose head is the compose", clone,
[(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))], 0)
case_tr_static("FACT: the compose moved past the ladder's head", clone,
[(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}))),
move], 1, ("not the ladder's newest step",))
case_tr_static("FACT: a line that is not one JSON entry", clone,
[(NF, lambda t: t + "\nupdate_ladder:\n - from: x\n")], 1, ("not a one-line JSON entry",))
case_tr_static("FACT: a gap between steps", clone,
[(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.62.0"}, {"navidrome": "deluan/navidrome:0.63.0"}, {"navidrome": TR_D1})
+ "\n" + tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))],
1, ("the ladder has a gap",))
case_tr_static("FACT: a failed verdict sits in the ladder", clone,
[(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}, verdict="failed")))],
1, ("not allowed in a ladder",))
def main():
gate = os.path.join(ROOT, "scripts", "check-engine-major.py")
if not os.path.isfile(gate):
@@ -659,6 +779,8 @@ i18n:
lambda t: t.replace(" container: paperless-webserver\n", ""))],
expect_rc=1, must_contain=("FAIL paperless-ngx",), apps=("paperless-ngx",))
test_record_cases(clone)
finally:
shutil.rmtree(clone, ignore_errors=True)