app-catalog-felhom.eu/scripts/retest_box.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""retest_box.py <app> <evidence dir>  (stdin: {"svc": [ref, tested_digest, new_digest], ...})

The BOX venue of a same-tag re-test (`09` §3 decision 52), on scratch guest 9202 pointed at the DRILL catalog. Called
by retest-floating.py; usable alone. Through the product's own endpoints only (box_walk.py):
  1. the drill checkout is pulled; its template for <app> must be the live one (the runbook resets the drill first);
  2. <app> is (re)installed fresh — the box renders the ladder head's TESTED (old) digest — and the running digest of
     every re-tested service is read back: it must BE the old one, or the proof would start from the wrong image;
  3. the fixture seeds and reads back (C1);
  4. a DRILL-only commit appends the re-test entry (from == to, digest = new, digest_from = old); sync + rescan until
     the box's catalog_digests carry the new digest; the badge is read (both languages);
  5. PRESS=update (default): the product's guarded Update; PRESS=chain: "run tonight's chain now"
     (POST /api/debug/backup/night-chain) and the leg's own log line is quoted;
  6. the seed reads back, the running digest must be the NEW one, the badge is read again;
  7. <evidence dir>/box-verdict-<app>.json: proven only if 5 ended done, 6 read back and 6's digest is the new one.
The app is removed through the product at the end (KEEP=1 keeps it).
"""
import json
import os
import re
import subprocess
import sys
import time

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import box_walk as w  # noqa: E402
import upgrade_fixtures_box as fixtures  # noqa: E402
import upgrade_fixtures_box28 as _f28  # noqa: E402

FX = dict(_f28.FIXTURES28)
FX.update(fixtures.FIXTURES)
DRILL = os.environ.get("DRILL", "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill")
# RETEST_SAME_AS: the checkout the drill must equal (default: this catalog). The end-to-end proof of 2026-09-30 simulated
# the month in the drill itself and set it to the drill.
CAT = os.environ.get("RETEST_SAME_AS") or os.path.dirname(HERE)


def main():
    app, evdir = sys.argv[1], sys.argv[2]
    moved = json.loads(sys.stdin.read() or "{}")
    os.makedirs(evdir, exist_ok=True)
    log = open(os.path.join(evdir, "box.txt"), "a", buffering=1)
    verdict = {"app": app, "verdict": "failed", "venue": "box 9202 (drill catalog), " + os.environ.get("PRESS", "update"),
               "retested": {s: {"ref": v[0], "from_digest": v[1], "to_digest": v[2]} for s, v in moved.items()}}

    def say(*a):
        w.say(*a)
        log.write(" ".join(str(x) for x in a) + "\n")

    def finish(why):
        verdict["why"] = why
        json.dump(verdict, open(os.path.join(evdir, "box-verdict-%s.json" % app), "w"), indent=2)
        say("RESULT %s — %s" % (verdict["verdict"], why))
        return 0 if verdict["verdict"] == "proven" else 1

    def running_digests():
        st = w.stack(app)
        ii = (st.get("app_config") or {}).get("installed_images") or {}
        return {s: (ii.get(s) or {}).get("digest") for s in moved}

    fx = FX.get(app)
    if fx is None:
        return finish("no box fixture for %s" % app)
    sub = getattr(fx, "sub", app)
    w.login()
    conf = w.guest("grep -A3 '^git:' /var/lib/docker/volumes/felhom-controller-data/_data/controller.yaml")
    if "app-catalog-drill" not in conf:
        return finish("9202 is not on the drill catalog (runbook §3) — nothing done")
    subprocess.run(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], check=True)
    for f in ("docker-compose.yml", ".felhom.yml"):
        if open(os.path.join(DRILL, "templates", app, f)).read() != open(os.path.join(CAT, "templates", app, f)).read():
            return finish("the drill's %s/%s differs from this checkout's — reset the drill first (runbook §3)" % (app, f))
    if w.stack(app).get("deployed"):
        say("removing the existing %s first (scratch box)" % app)
        w.remove(app)
    w.sync_rescan()
    if not w.deploy(app, sub):
        return finish("the install did not complete")
    before = running_digests()
    say("running digests after install: %s" % before)
    wrong = {s: d for s, d in before.items() if d != moved[s][1]}
    if wrong:
        return finish("the box did not start at the tested digest: %s" % wrong)
    tok = fx.seed(w, sub, say)
    if tok is None or not fx.verify(w, sub, tok, say):
        return finish("C1: the fixture could not seed and read back before the re-test")
    verdict["seed_read_before"] = True
    # the drill-only re-test entry
    fy = os.path.join(DRILL, "templates", app, ".felhom.yml")
    comp = os.path.join(DRILL, "templates", app, "docker-compose.yml")
    sys.path.insert(0, HERE)
    import ladder
    entries, _, _ = ladder.parse(open(fy).read())
    head = entries[-1]
    e = dict(head)
    e["from"] = dict(head["to"])
    e["digest"] = dict(head["digest"])
    e["digest_from"] = dict(head["digest"])
    for s, v in moved.items():
        e["digest"][s] = v[2]
        e["digest_from"][s] = v[1]
    e["tested_at"], e["evidence"], e["box_evidence"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "DRILL", "DRILL (box proof in progress)"
    for k in ("engine_conversion", "backfilled"):
        e.pop(k, None)
    open(fy, "w").write(ladder.append_entry(open(fy).read(), e))
    subprocess.run(["git", "-C", DRILL, "commit", "-q", "-am", "DRILL %s: re-test of the same tag %s (box proof)" % (app, {s: v[2][:19] for s, v in moved.items()})], check=True)
    subprocess.run(["git", "-C", DRILL, "push", "-q", "origin", "main"], check=True, capture_output=True)
    say("drill:", subprocess.run(["git", "-C", DRILL, "log", "--oneline", "-1"], capture_output=True, text=True).stdout.strip())
    for i in range(24):
        w.sync_rescan()
        cd = w.stack(app).get("catalog_digests") or {}
        if all(cd.get(s) == v[2] for s, v in moved.items()):
            say("the box's catalog_digests carry the new digest after %d sync round(s)" % (i + 1))
            break
        time.sleep(5)
    else:
        return finish("the box never read the re-test's digest from the drill catalog")
    verdict["badge_before"] = w.badges(app)
    say("badge before: %s" % verdict["badge_before"])
    since = w.guest("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
    if os.environ.get("PRESS") == "chain":
        code, d = w.ctl("POST", "/api/debug/backup/night-chain")
        say("night chain -> %s %s" % (code, str(d)[:200]))
        leg = ""
        for _ in range(240):
            time.sleep(10)
            leg = w.guest("docker logs --since %s felhom-controller 2>&1 | grep -E 'update-leg' | grep -v DEBUG | cut -c1-300" % since)
            if re.search(r"update leg .*: done=", leg):
                break
        say("the leg's own lines:\n" + leg)
        verdict["leg_log"] = leg.strip().splitlines()
        pressed = "%s: step pressed" % app in leg
        ended = re.search(r"%s: step ended (\w+)" % re.escape(app), leg)
        final = ended.group(1) if ended else None
        if not pressed:
            return finish("the leg did not press %s" % app)
    else:
        res = w.press_update(app, poll=1, cap_s=1800)
        final = res.get("final_phase")
    time.sleep(10)
    after = running_digests()
    read = fx.verify(w, sub, tok, say)
    verdict.update({"final_phase": final, "digests_before": before, "digests_after": after, "seed_read_after": read,
                    "badge_after": w.badges(app), "from": w.stack(app).get("app_config", {}).get("pinned_images"),
                    "to": w.stack(app).get("app_config", {}).get("pinned_images"), "measured_at": since})
    say("after: phase=%s digests=%s read_back=%s badge=%s" % (final, after, read, verdict["badge_after"]))
    if final == "done" and read and all(after.get(s) == v[2] for s, v in moved.items()):
        verdict["verdict"] = "proven"
    code = finish("the re-test step ran on the box" if verdict["verdict"] == "proven" else "the step did not end done / did not read back / runs another digest")
    if not os.environ.get("KEEP"):
        w.remove(app)
    return code


if __name__ == "__main__":
    sys.exit(main())