diff --git a/scripts/test_ladder_writer.py b/scripts/test_ladder_writer.py index ac9f037..2e89fbf 100644 --- a/scripts/test_ladder_writer.py +++ b/scripts/test_ladder_writer.py @@ -131,5 +131,58 @@ class WriterTest(unittest.TestCase): self.assertEqual(r.returncode, 0, r.stdout) + # --- --restep (2026-09-30, Part D): rewrite ONE superseded step's own definition from a re-proof --------------- + def _two_steps(self): + rc, out = self.run_writer(bench()); self.assertEqual(rc, 0, out) + rc, out = self.run_writer(bench(frm=NEXT, to=AFTER)); self.assertEqual(rc, 0, out) + return ladder.step_file({"navidrome": "deluan/navidrome:" + NEXT}) + + def run_restep(self, b, definition_images=None, extra=""): + d = os.path.join(self.tmp, "def"); os.makedirs(d, exist_ok=True) + imgs = definition_images or b["to"] + open(os.path.join(d, "docker-compose.yml"), "w").write( + "services:\n navidrome:\n image: %s\n deploy:\n resources:\n limits:\n memory: 999M\n%s" + % (imgs["navidrome"], extra)) + meta = open(os.path.join(self.tmp, "templates", "navidrome", ladder.step_meta_file(b["to"]))).read() \ + if os.path.exists(os.path.join(self.tmp, "templates", "navidrome", ladder.step_meta_file(b["to"]))) \ + else open(self.fy).read() + open(os.path.join(d, ".felhom.yml"), "w").write(meta + "\n# RESTEP-META-MARKER\n") + bp = os.path.join(self.tmp, "rb.json"); json.dump(b, open(bp, "w")) + buf = io.StringIO() + with redirect_stdout(buf): + rc = ut.write_restep([bp, "--definition", d, "--catalog", self.tmp, "--evidence", "ev/restep/"]) + return rc, buf.getvalue() + + def test_restep_rewrites_the_superseded_steps_files_and_nothing_else(self): + key = self._two_steps() + fy_before = open(self.fy).read() + rc, out = self.run_restep(bench()) + self.assertEqual(rc, 0, out) + sp = os.path.join(self.tmp, "templates", "navidrome", key) + self.assertIn("memory: 999M", open(sp).read()) + self.assertIn("RE-PROVEN", open(sp).read()) + self.assertIn("RESTEP-META-MARKER", open(sp[:-4] + ".felhom.yml").read()) + self.assertEqual(open(self.fy).read(), fy_before, "the ladder entry must not be touched") + import subprocess + r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp, + "navidrome"], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout) + + def test_restep_refuses_the_head_a_failed_bench_and_a_wrong_definition(self): + key = self._two_steps() + sp = os.path.join(self.tmp, "templates", "navidrome", key) + before = open(sp).read() + rc, out = self.run_restep(bench(frm=NEXT, to=AFTER)) + self.assertEqual(rc, 1); self.assertIn("HEAD", out) + rc, out = self.run_restep(bench(verdict="failed")) + self.assertEqual(rc, 1) + rc, out = self.run_restep(bench(), definition_images={"navidrome": "deluan/navidrome:" + AFTER}) + self.assertEqual(rc, 1); self.assertIn("the definition names", out) + b = bench(); b["memory"]["containers"]["navidrome"]["oom_kill"] = 3 + rc, out = self.run_restep(b) + self.assertEqual(rc, 1); self.assertIn("OOM", out) + self.assertEqual(open(sp).read(), before, "a refused restep must not write") + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/upgrade-test.py b/scripts/upgrade-test.py index d06c30f..5e83eba 100755 --- a/scripts/upgrade-test.py +++ b/scripts/upgrade-test.py @@ -39,6 +39,8 @@ Usage: python3 upgrade-test.py [--soak SECONDS] [ …] (se python3 upgrade-test.py [--soak SECONDS] --move = [...] (FROM = the template) python3 upgrade-test.py --write-ladder --box \ --catalog --evidence [--box-evidence ] (the ONLY ladder writer) + python3 upgrade-test.py --restep --definition --catalog --evidence + (rewrites ONE superseded step's own definition from a bench re-proof of it) python3 upgrade-test.py --list --soak: how long the memory watch runs after a successful readback (default 600; 0 = off) Layout: templates under /opt/upg/templates, evidence under /opt/upg/evidence @@ -1093,10 +1095,64 @@ def write_ladder(argv) -> int: return 0 +def write_restep(argv) -> int: + """`--restep --definition --catalog --evidence ` — rewrite ONE + superseded step's own definition (`steps/.yml` + `steps/.felhom.yml`) from a bench re-proof of THAT + step on a corrected definition (2026-09-30, R-732 → immich's step 0b8272068aab36bf at 768M). Never by hand. + + It refuses unless the bench says `proven` with a memory watch and no kill; the verdict's from/to are exactly + one ladder entry that is NOT the head (the head's definition is the compose, moved by --write-ladder); and the + definition's own images are that entry's `to`. The ladder entry itself is NOT touched: its digests, its box + evidence and the box's failed-step fingerprint stay what they were. The new files say where their re-proof is.""" + import datetime as _dt, hashlib + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import ladder + + def arg(name, default=None): + return argv[argv.index(name) + 1] if name in argv else default + bench = json.loads(Path(argv[0]).read_text()) + dfn, cat, evid = Path(arg("--definition")), Path(arg("--catalog")), arg("--evidence") + app = bench["app"] + if bench.get("verdict") != "proven" or not bench.get("memory") or (bench.get("harness_version") or 0) < 2: + print(f"REFUSED {app}: the bench verdict is {bench.get('verdict')!r} / no memory watch — only a proven re-proof rewrites a step") + return 1 + kills = [n for n, c in (bench["memory"].get("containers") or {}).items() if (c.get("oom_kill") or 0) > 0] + if kills: + print(f"REFUSED {app}: the memory watch saw OOM kills in {kills}") + return 1 + tdir = cat / "templates" / app + entries, _, errs = ladder.parse((tdir / ".felhom.yml").read_text()) + idx = [i for i, e in enumerate(entries or []) if e.get("from") == bench["from"] and e.get("to") == bench["to"]] + if not idx: + print(f"REFUSED {app}: no ladder entry is {bench['from']} -> {bench['to']}") + return 1 + if idx[-1] == len(entries) - 1: + print(f"REFUSED {app}: that entry is the HEAD — its definition is the compose (use --write-ladder)") + return 1 + e = entries[idx[-1]] + comp = (dfn / "docker-compose.yml").read_text() + if ladder.images_in(comp) != e["to"]: + print(f"REFUSED {app}: the definition names {ladder.images_in(comp)}, the step is {e['to']}") + return 1 + sp, smp = tdir / ladder.step_file(e["to"]), tdir / ladder.step_meta_file(e["to"]) + if not sp.exists(): + print(f"REFUSED {app}: {ladder.step_file(e['to'])} does not exist — nothing to rewrite") + return 1 + head = (f"# RE-PROVEN {_dt.date.today().isoformat()} on the bench on THIS definition (upgrade-test.py --restep): " + f"{evid} — definition sha256 {hashlib.sha256(comp.encode()).hexdigest()[:16]}\n") + sp.write_text(head + comp) + smp.write_text(head + ladder.strip_ladder_block((dfn / ".felhom.yml").read_text())) + peaks = {n: c.get("anon_peak_pct") for n, c in (bench["memory"].get("containers") or {}).items()} + print(f"RESTEP {app}: {ladder.step_file(e['to'])} and its .felhom.yml rewritten from {dfn} (anon peaks {peaks})") + return 0 + + def main(argv): global SOAK_SECONDS if argv and argv[0] == "--write-ladder": return write_ladder(argv[1:]) + if argv and argv[0] == "--restep": + return write_restep(argv[1:]) if argv and argv[0] == "--soak": SOAK_SECONDS = int(argv[1]) argv = argv[2:] diff --git a/scripts/upgrade_fixtures_box.py b/scripts/upgrade_fixtures_box.py index b59803a..7f48142 100644 --- a/scripts/upgrade_fixtures_box.py +++ b/scripts/upgrade_fixtures_box.py @@ -817,7 +817,10 @@ class Zipline: return None def verify(self, w, sub, t, say): - if not w.wait_app(sub, "/", want=("200", "302", "307"), tries=60): + # 2026-09-30: `/` answers 301 on 9202 once set up — the old wait (200/302/307) never saw the app and the + # readback returned False with no line in the log. The seed's own health route is the wait now. + if not w.wait_app(sub, "/api/healthcheck", want=("200",), tries=60): + say(" zipline: /api/healthcheck never answered 200") return False def login(p): return w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json",