harness: --restep rewrites one superseded step's definition from a re-proof; zipline's readback waits on its health route

- upgrade-test.py --restep <verdict> --definition <dir> --catalog <c> --evidence <rel>: the only way a
  superseded step's own files (steps/<key>.yml + .felhom.yml) change after the fact (Part D: immich's
  step 0b8272068aab36bf still pins 512M). Refuses a non-proven or OOM-killing re-proof, the head entry,
  and a definition whose images are not the step's; leaves the ladder entry untouched (digests, box
  evidence, the box's failed-step fingerprint). Two tests; the gate accepts the result.
- zipline's verify waited on `/` for 200/302/307; on 9202 it answers 301 and the readback returned
  False with no line — it now waits on /api/healthcheck like its seed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

scripts/test_ladder_writer.py

@@ -131,5 +131,58 @@ class WriterTest(unittest.TestCase):
         self.assertEqual(r.returncode, 0, r.stdout)
 
 
+    # --- --restep (2026-09-30, Part D): rewrite ONE superseded step's own definition from a re-proof ---------------
+    def _two_steps(self):
+        rc, out = self.run_writer(bench()); self.assertEqual(rc, 0, out)
+        rc, out = self.run_writer(bench(frm=NEXT, to=AFTER)); self.assertEqual(rc, 0, out)
+        return ladder.step_file({"navidrome": "deluan/navidrome:" + NEXT})
+
+    def run_restep(self, b, definition_images=None, extra=""):
+        d = os.path.join(self.tmp, "def"); os.makedirs(d, exist_ok=True)
+        imgs = definition_images or b["to"]
+        open(os.path.join(d, "docker-compose.yml"), "w").write(
+            "services:\n  navidrome:\n    image: %s\n    deploy:\n      resources:\n        limits:\n          memory: 999M\n%s"
+            % (imgs["navidrome"], extra))
+        meta = open(os.path.join(self.tmp, "templates", "navidrome", ladder.step_meta_file(b["to"]))).read() \
+            if os.path.exists(os.path.join(self.tmp, "templates", "navidrome", ladder.step_meta_file(b["to"]))) \
+            else open(self.fy).read()
+        open(os.path.join(d, ".felhom.yml"), "w").write(meta + "\n# RESTEP-META-MARKER\n")
+        bp = os.path.join(self.tmp, "rb.json"); json.dump(b, open(bp, "w"))
+        buf = io.StringIO()
+        with redirect_stdout(buf):
+            rc = ut.write_restep([bp, "--definition", d, "--catalog", self.tmp, "--evidence", "ev/restep/"])
+        return rc, buf.getvalue()
+
+    def test_restep_rewrites_the_superseded_steps_files_and_nothing_else(self):
+        key = self._two_steps()
+        fy_before = open(self.fy).read()
+        rc, out = self.run_restep(bench())
+        self.assertEqual(rc, 0, out)
+        sp = os.path.join(self.tmp, "templates", "navidrome", key)
+        self.assertIn("memory: 999M", open(sp).read())
+        self.assertIn("RE-PROVEN", open(sp).read())
+        self.assertIn("RESTEP-META-MARKER", open(sp[:-4] + ".felhom.yml").read())
+        self.assertEqual(open(self.fy).read(), fy_before, "the ladder entry must not be touched")
+        import subprocess
+        r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp,
+                            "navidrome"], capture_output=True, text=True)
+        self.assertEqual(r.returncode, 0, r.stdout)
+
+    def test_restep_refuses_the_head_a_failed_bench_and_a_wrong_definition(self):
+        key = self._two_steps()
+        sp = os.path.join(self.tmp, "templates", "navidrome", key)
+        before = open(sp).read()
+        rc, out = self.run_restep(bench(frm=NEXT, to=AFTER))
+        self.assertEqual(rc, 1); self.assertIn("HEAD", out)
+        rc, out = self.run_restep(bench(verdict="failed"))
+        self.assertEqual(rc, 1)
+        rc, out = self.run_restep(bench(), definition_images={"navidrome": "deluan/navidrome:" + AFTER})
+        self.assertEqual(rc, 1); self.assertIn("the definition names", out)
+        b = bench(); b["memory"]["containers"]["navidrome"]["oom_kill"] = 3
+        rc, out = self.run_restep(b)
+        self.assertEqual(rc, 1); self.assertIn("OOM", out)
+        self.assertEqual(open(sp).read(), before, "a refused restep must not write")
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)

scripts/upgrade-test.py

@@ -39,6 +39,8 @@ Usage:  python3 upgrade-test.py [--soak SECONDS] <edge-id> [<edge-id> …]   (se
         python3 upgrade-test.py [--soak SECONDS] --move <app> <svc>=<ref> [...]   (FROM = the template)
         python3 upgrade-test.py --write-ladder <verdict.json> --box <box verdict.json> \
                 --catalog <checkout> --evidence <rel> [--box-evidence <rel>]   (the ONLY ladder writer)
+        python3 upgrade-test.py --restep <verdict.json> --definition <dir> --catalog <checkout> --evidence <rel>
+                (rewrites ONE superseded step's own definition from a bench re-proof of it)
         python3 upgrade-test.py --list
         --soak: how long the memory watch runs after a successful readback (default 600; 0 = off)
 Layout: templates under /opt/upg/templates, evidence under /opt/upg/evidence
@@ -1093,10 +1095,64 @@ def write_ladder(argv) -> int:
     return 0
 
 
+def write_restep(argv) -> int:
+    """`--restep <bench verdict.json> --definition <dir> --catalog <checkout> --evidence <rel>` — rewrite ONE
+    superseded step's own definition (`steps/<key>.yml` + `steps/<key>.felhom.yml`) from a bench re-proof of THAT
+    step on a corrected definition (2026-09-30, R-732 → immich's step 0b8272068aab36bf at 768M). Never by hand.
+
+    It refuses unless the bench says `proven` with a memory watch and no kill; the verdict's from/to are exactly
+    one ladder entry that is NOT the head (the head's definition is the compose, moved by --write-ladder); and the
+    definition's own images are that entry's `to`. The ladder entry itself is NOT touched: its digests, its box
+    evidence and the box's failed-step fingerprint stay what they were. The new files say where their re-proof is."""
+    import datetime as _dt, hashlib
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    import ladder
+
+    def arg(name, default=None):
+        return argv[argv.index(name) + 1] if name in argv else default
+    bench = json.loads(Path(argv[0]).read_text())
+    dfn, cat, evid = Path(arg("--definition")), Path(arg("--catalog")), arg("--evidence")
+    app = bench["app"]
+    if bench.get("verdict") != "proven" or not bench.get("memory") or (bench.get("harness_version") or 0) < 2:
+        print(f"REFUSED {app}: the bench verdict is {bench.get('verdict')!r} / no memory watch — only a proven re-proof rewrites a step")
+        return 1
+    kills = [n for n, c in (bench["memory"].get("containers") or {}).items() if (c.get("oom_kill") or 0) > 0]
+    if kills:
+        print(f"REFUSED {app}: the memory watch saw OOM kills in {kills}")
+        return 1
+    tdir = cat / "templates" / app
+    entries, _, errs = ladder.parse((tdir / ".felhom.yml").read_text())
+    idx = [i for i, e in enumerate(entries or []) if e.get("from") == bench["from"] and e.get("to") == bench["to"]]
+    if not idx:
+        print(f"REFUSED {app}: no ladder entry is {bench['from']} -> {bench['to']}")
+        return 1
+    if idx[-1] == len(entries) - 1:
+        print(f"REFUSED {app}: that entry is the HEAD — its definition is the compose (use --write-ladder)")
+        return 1
+    e = entries[idx[-1]]
+    comp = (dfn / "docker-compose.yml").read_text()
+    if ladder.images_in(comp) != e["to"]:
+        print(f"REFUSED {app}: the definition names {ladder.images_in(comp)}, the step is {e['to']}")
+        return 1
+    sp, smp = tdir / ladder.step_file(e["to"]), tdir / ladder.step_meta_file(e["to"])
+    if not sp.exists():
+        print(f"REFUSED {app}: {ladder.step_file(e['to'])} does not exist — nothing to rewrite")
+        return 1
+    head = (f"# RE-PROVEN {_dt.date.today().isoformat()} on the bench on THIS definition (upgrade-test.py --restep): "
+            f"{evid} — definition sha256 {hashlib.sha256(comp.encode()).hexdigest()[:16]}\n")
+    sp.write_text(head + comp)
+    smp.write_text(head + ladder.strip_ladder_block((dfn / ".felhom.yml").read_text()))
+    peaks = {n: c.get("anon_peak_pct") for n, c in (bench["memory"].get("containers") or {}).items()}
+    print(f"RESTEP {app}: {ladder.step_file(e['to'])} and its .felhom.yml rewritten from {dfn} (anon peaks {peaks})")
+    return 0
+
+
 def main(argv):
     global SOAK_SECONDS
     if argv and argv[0] == "--write-ladder":
         return write_ladder(argv[1:])
+    if argv and argv[0] == "--restep":
+        return write_restep(argv[1:])
     if argv and argv[0] == "--soak":
         SOAK_SECONDS = int(argv[1])
         argv = argv[2:]

scripts/upgrade_fixtures_box.py

@@ -817,7 +817,10 @@ class Zipline:
         return None
 
     def verify(self, w, sub, t, say):
-        if not w.wait_app(sub, "/", want=("200", "302", "307"), tries=60):
+        # 2026-09-30: `/` answers 301 on 9202 once set up — the old wait (200/302/307) never saw the app and the
+        # readback returned False with no line in the log. The seed's own health route is the wait now.
+        if not w.wait_app(sub, "/api/healthcheck", want=("200",), tries=60):
+            say("    zipline: /api/healthcheck never answered 200")
             return False
         def login(p):
             return w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json",