#!/usr/bin/env python3 # -*- coding: utf-8 -*- """retest_box.py (stdin: {"svc": [ref, tested_digest, new_digest], ...}) The BOX venue of a same-tag re-test (`09` §3 decision 52), on scratch guest 9202 pointed at the DRILL catalog. Called by retest-floating.py; usable alone. Through the product's own endpoints only (box_walk.py): 1. the drill checkout is pulled; its template for must be the live one (the runbook resets the drill first); 2. is (re)installed fresh — the box renders the ladder head's TESTED (old) digest — and the running digest of every re-tested service is read back: it must BE the old one, or the proof would start from the wrong image; 3. the fixture seeds and reads back (C1); 4. a DRILL-only commit appends the re-test entry (from == to, digest = new, digest_from = old); sync + rescan until the box's catalog_digests carry the new digest; the badge is read (both languages); 5. PRESS=update (default): the product's guarded Update; PRESS=chain: "run tonight's chain now" (POST /api/debug/backup/night-chain) and the leg's own log line is quoted; 6. the seed reads back, the running digest must be the NEW one, the badge is read again; 7. /box-verdict-.json: proven only if 5 ended done, 6 read back and 6's digest is the new one. The app is removed through the product at the end (KEEP=1 keeps it). """ import json import os import re import subprocess import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import box_walk as w # noqa: E402 import upgrade_fixtures_box as fixtures # noqa: E402 import upgrade_fixtures_box28 as _f28 # noqa: E402 FX = dict(_f28.FIXTURES28) FX.update(fixtures.FIXTURES) DRILL = os.environ.get("DRILL", "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill") # RETEST_SAME_AS: the checkout the drill must equal (default: this catalog). The end-to-end proof of 2026-09-30 simulated # the month in the drill itself and set it to the drill. CAT = os.environ.get("RETEST_SAME_AS") or os.path.dirname(HERE) def main(): app, evdir = sys.argv[1], sys.argv[2] moved = json.loads(sys.stdin.read() or "{}") os.makedirs(evdir, exist_ok=True) log = open(os.path.join(evdir, "box.txt"), "a", buffering=1) verdict = {"app": app, "verdict": "failed", "venue": "box 9202 (drill catalog), " + os.environ.get("PRESS", "update"), "retested": {s: {"ref": v[0], "from_digest": v[1], "to_digest": v[2]} for s, v in moved.items()}} def say(*a): w.say(*a) log.write(" ".join(str(x) for x in a) + "\n") def finish(why): verdict["why"] = why json.dump(verdict, open(os.path.join(evdir, "box-verdict-%s.json" % app), "w"), indent=2) say("RESULT %s — %s" % (verdict["verdict"], why)) return 0 if verdict["verdict"] == "proven" else 1 def running_digests(): st = w.stack(app) ii = (st.get("app_config") or {}).get("installed_images") or {} return {s: (ii.get(s) or {}).get("digest") for s in moved} fx = FX.get(app) if fx is None: return finish("no box fixture for %s" % app) sub = getattr(fx, "sub", app) w.login() conf = w.guest("grep -A3 '^git:' /var/lib/docker/volumes/felhom-controller-data/_data/controller.yaml") if "app-catalog-drill" not in conf: return finish("9202 is not on the drill catalog (runbook §3) — nothing done") subprocess.run(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], check=True) for f in ("docker-compose.yml", ".felhom.yml"): if open(os.path.join(DRILL, "templates", app, f)).read() != open(os.path.join(CAT, "templates", app, f)).read(): return finish("the drill's %s/%s differs from this checkout's — reset the drill first (runbook §3)" % (app, f)) if w.stack(app).get("deployed"): say("removing the existing %s first (scratch box)" % app) w.remove(app) w.sync_rescan() if not w.deploy(app, sub): return finish("the install did not complete") before = running_digests() say("running digests after install: %s" % before) wrong = {s: d for s, d in before.items() if d != moved[s][1]} if wrong: return finish("the box did not start at the tested digest: %s" % wrong) tok = fx.seed(w, sub, say) if tok is None or not fx.verify(w, sub, tok, say): return finish("C1: the fixture could not seed and read back before the re-test") verdict["seed_read_before"] = True # the drill-only re-test entry fy = os.path.join(DRILL, "templates", app, ".felhom.yml") comp = os.path.join(DRILL, "templates", app, "docker-compose.yml") sys.path.insert(0, HERE) import ladder entries, _, _ = ladder.parse(open(fy).read()) head = entries[-1] e = dict(head) e["from"] = dict(head["to"]) e["digest"] = dict(head["digest"]) e["digest_from"] = dict(head["digest"]) for s, v in moved.items(): e["digest"][s] = v[2] e["digest_from"][s] = v[1] e["tested_at"], e["evidence"], e["box_evidence"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "DRILL", "DRILL (box proof in progress)" for k in ("engine_conversion", "backfilled"): e.pop(k, None) open(fy, "w").write(ladder.append_entry(open(fy).read(), e)) subprocess.run(["git", "-C", DRILL, "commit", "-q", "-am", "DRILL %s: re-test of the same tag %s (box proof)" % (app, {s: v[2][:19] for s, v in moved.items()})], check=True) subprocess.run(["git", "-C", DRILL, "push", "-q", "origin", "main"], check=True, capture_output=True) say("drill:", subprocess.run(["git", "-C", DRILL, "log", "--oneline", "-1"], capture_output=True, text=True).stdout.strip()) for i in range(24): w.sync_rescan() cd = w.stack(app).get("catalog_digests") or {} if all(cd.get(s) == v[2] for s, v in moved.items()): say("the box's catalog_digests carry the new digest after %d sync round(s)" % (i + 1)) break time.sleep(5) else: return finish("the box never read the re-test's digest from the drill catalog") verdict["badge_before"] = w.badges(app) say("badge before: %s" % verdict["badge_before"]) since = w.guest("date -u +%Y-%m-%dT%H:%M:%SZ").strip() if os.environ.get("PRESS") == "chain": code, d = w.ctl("POST", "/api/debug/backup/night-chain") say("night chain -> %s %s" % (code, str(d)[:200])) leg = "" for _ in range(240): time.sleep(10) leg = w.guest("docker logs --since %s felhom-controller 2>&1 | grep -E 'update-leg' | grep -v DEBUG | cut -c1-300" % since) if re.search(r"update leg .*: done=", leg): break say("the leg's own lines:\n" + leg) verdict["leg_log"] = leg.strip().splitlines() pressed = "%s: step pressed" % app in leg ended = re.search(r"%s: step ended (\w+)" % re.escape(app), leg) final = ended.group(1) if ended else None if not pressed: return finish("the leg did not press %s" % app) else: res = w.press_update(app, poll=1, cap_s=1800) final = res.get("final_phase") time.sleep(10) after = running_digests() read = fx.verify(w, sub, tok, say) verdict.update({"final_phase": final, "digests_before": before, "digests_after": after, "seed_read_after": read, "badge_after": w.badges(app), "from": w.stack(app).get("app_config", {}).get("pinned_images"), "to": w.stack(app).get("app_config", {}).get("pinned_images"), "measured_at": since}) say("after: phase=%s digests=%s read_back=%s badge=%s" % (final, after, read, verdict["badge_after"])) if final == "done" and read and all(after.get(s) == v[2] for s, v in moved.items()): verdict["verdict"] = "proven" code = finish("the re-test step ran on the box" if verdict["verdict"] == "proven" else "the step did not end done / did not read back / runs another digest") if not os.environ.get("KEEP"): w.remove(app) return code if __name__ == "__main__": sys.exit(main())