steps/ per ladder step: gate rule 4, writer, backfill (8); R-653, R-656; decoy suite reads live pins (R-663)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

scripts/check-test-record.py

@@ -19,6 +19,9 @@ WHAT IT CHECKS, per template that carries `update_ladder:` (format and field rul
   3. the NEWEST entry's `to` is EXACTLY the compose's current image per service. This is the fact
      that makes the rule hold without history: a compose moved without a new entry no longer
      matches its ladder's head, whoever pushed it and however.
+  4. every entry but the newest carries its OWN definition at `steps/<step_key(to)>.yml`, whose
+     images per service are EXACTLY that entry's `to` (`09` §6.4 part 5: the box pins that file, one
+     step per press). The name alone is not the fact: the file's own `image:` lines are read.
 
 A template WITHOUT a ladder passes here — it has never been moved since the gate existed, and its
 first move is refused by the twin unless that move brings the first entry.
@@ -50,6 +53,18 @@ def check_app(app_dir):
     for i in range(1, len(entries)):
         if entries[i].get("from") != entries[i - 1].get("to"):
             problems.append("entry %d's `from` is not entry %d's `to` — the ladder has a gap" % (i + 1, i))
+    for i, e in enumerate(entries[:-1]):
+        to = e.get("to")
+        if not isinstance(to, dict) or not to:
+            continue  # already convicted by check_entry
+        sp = os.path.join(app_dir, ladder.step_file(to))
+        if not os.path.isfile(sp):
+            problems.append("entry %d of %d has no definition at %s — the box climbs one step at a time "
+                            "and needs this step's own compose file" % (i + 1, len(entries), ladder.step_file(to)))
+            continue
+        got = ladder.images_in(open(sp, encoding="utf-8").read())
+        if got != to:
+            problems.append("%s names %s, but entry %d's `to` is %s" % (ladder.step_file(to), got, i + 1, to))
     if entries:
         current = ladder.images_in(open(comp, encoding="utf-8").read())
         head = entries[-1].get("to")

scripts/ladder.py

@@ -30,8 +30,16 @@ AN ENTRY (all keys required unless marked):
   backfilled        (optional) "YYYY-MM-DD" — written by the backfill from an EXISTING record,
                     never by a new test; a new move may not carry it
 
+STEP DEFINITIONS (`09` §6.4 part 5, controller v0.268.0): every entry but the NEWEST carries its own
+complete compose file at `templates/<app>/steps/<step_key(to)>.yml` — the box climbs one step at a time
+and pins exactly that file; the newest step's definition is the template's `docker-compose.yml`. The box's
+catalog clone is `--depth 1`, so git history is not a place a box can read a step from, and the commit
+that moved an image is not always the definition that works (romm 15f9ebf). `step_key` is computed the
+SAME way by the controller (`stacks.StepKey`, pinned by TestLadder_StepKeyMatchesTheCatalog).
+
 Every path that reads or writes the format is here, so the gate and the writer cannot disagree.
 """
+import hashlib
 import json
 import re
 
@@ -191,3 +199,14 @@ def append_entry(felhom_text, e):
             last = j
     lines.insert(last + 1, line)
     return "\n".join(lines) + "\n"
+
+
+def step_key(to):
+    """The 16-hex name of a step's definition: sha256 of `to` as canonical JSON (keys sorted, no spaces).
+    The controller computes the same string (stacks.StepKey)."""
+    return hashlib.sha256(json.dumps(to, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:16]
+
+
+def step_file(to):
+    """The step definition's path RELATIVE to the template directory."""
+    return "steps/%s.yml" % step_key(to)

scripts/steps_backfill.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""steps_backfill.py — ONE-OFF (2026-09-24, `09` §6.4 part 5): write `steps/<step_key(to)>.yml` for every
+ladder entry that is not the newest, from git history.
+
+WHICH COMMIT, and why not "the commit of the step": the definition a box must pin for a step is the one
+the catalog SERVED while that step was the head — the NEWEST commit whose compose images equal the
+step's `to`, i.e. the step's images WITH every fix that flowed after they moved. The commit that MOVED
+the image can be the broken one: romm's 15f9ebf (5.0.0 -> 5.3.0, 512M, four workers) OOM-looped on
+demo-hp; the working definition is the same images under f4eb94f's template (768M, two workers).
+
+Prints one line per step: app, entry, step key, the commit it came from. Never overwrites a step file.
+"""
+import os
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import ladder  # noqa: E402
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def git(*a):
+    return subprocess.run(["git", "-C", ROOT] + list(a), capture_output=True, text=True, check=True).stdout
+
+
+def main():
+    wrote = 0
+    for app in sorted(os.listdir(os.path.join(ROOT, "templates"))):
+        d = os.path.join(ROOT, "templates", app)
+        fy = os.path.join(d, ".felhom.yml")
+        if not os.path.isfile(fy):
+            continue
+        entries, _, errs = ladder.parse(open(fy, encoding="utf-8").read())
+        if errs or len(entries) < 2:
+            continue
+        rel = "templates/%s/docker-compose.yml" % app
+        commits = git("log", "--format=%H", "--", rel).split()   # newest first
+        for i, e in enumerate(entries[:-1]):
+            dst = os.path.join(d, ladder.step_file(e["to"]))
+            if os.path.exists(dst):
+                print("SKIP  %-10s entry %d: %s exists" % (app, i + 1, ladder.step_file(e["to"])))
+                continue
+            found = None
+            for c in commits:
+                body = git("show", "%s:%s" % (c, rel))
+                if ladder.images_in(body) == e["to"]:
+                    found = (c, body)
+                    break          # newest first: the last definition served for this step
+            if not found:
+                print("MISS  %-10s entry %d: no commit's compose names %s" % (app, i + 1, e["to"]))
+                continue
+            os.makedirs(os.path.dirname(dst), exist_ok=True)
+            open(dst, "w", encoding="utf-8").write(found[1])
+            print("WROTE %-10s entry %d/%d -> %s from %s (%s)" % (app, i + 1, len(entries), ladder.step_file(e["to"]),
+                                                                 found[0][:12], git("log", "-1", "--format=%s", found[0]).strip()[:70]))
+            wrote += 1
+    print("steps_backfill: %d step definition(s) written" % wrote)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/test_gate_decoys.py

@@ -94,6 +94,7 @@ def commit(clone, msg):
 
 def reset(clone):
     sh(["git", "reset", "-q", "--hard", "HEAD"], cwd=clone)
+    sh(["git", "clean", "-fdq"], cwd=clone)   # a case may ADD a file (a steps/ definition)
 
 
 def case(name, clone, edits, expect_rc, must_contain=(), gate="check-engine-major.py"):
@@ -283,6 +284,25 @@ def swap_image(service, frm, to):
 
 
 
+def cur_image(clone, relpath, service):
+    """The service's current image in the clone — read, never typed (R-663)."""
+    sys.path.insert(0, os.path.join(ROOT, "scripts"))
+    import ladder as _l
+    return _l.images_in(io.open(os.path.join(clone, relpath), encoding="utf-8").read())[service]
+
+
+def strip_ladder(t):
+    """The template WITHOUT its update_ladder block (and the header comment the writer puts above it).
+    The writer appends the block at the END of the file (ladder.append_entry), so everything from its
+    first line on goes — a case then builds exactly the ladder it describes, whatever the live catalog
+    has recorded since (R-663)."""
+    lines = t.splitlines()
+    for i, l in enumerate(lines):
+        if l.startswith("# update_ladder") or l.startswith("update_ladder:"):
+            return "\n".join(lines[:i]).rstrip("\n") + "\n"
+    return t
+
+
 # ── test record (09 §3 decision 13) ────────────────────────────────────────────────────────────
 TR_D1 = "sha256:" + "a" * 64
 TR_D2 = "sha256:" + "b" * 64
@@ -299,8 +319,12 @@ def tr_entry(frm, to, digest, verdict="proven", peak=41.0, tight=False, **extra)
 
 
 def tr_append(line, header=True):
+    """Append one entry line. A template that already HAS a ladder (the writer puts it at the end of
+    the file) gets the line appended to it; one without gets the block (R-663: navidrome gained a
+    ladder on 2026-09-23 night, and a second `update_ladder:` key is its own conviction)."""
     def _fn(t):
-        block = ("\nupdate_ladder:\n" if header else "") + line + "\n"
+        has = any(l.startswith("update_ladder:") for l in t.splitlines())
+        block = ("\nupdate_ladder:\n" if header and not has else "") + line + "\n"
         return t.rstrip("\n") + "\n" + block
     return _fn
 
@@ -349,7 +373,10 @@ def case_tr_static(name, clone, edits, expect_rc, must_contain=(), apps=("navidr
 
 def test_record_cases(clone):
     NC, NF = "templates/navidrome/docker-compose.yml", "templates/navidrome/.felhom.yml"
-    old, new = "deluan/navidrome:0.64.0", "deluan/navidrome:0.64.1"
+    # READ, never typed (R-663): the live pin moves with every proven step.
+    old = cur_image(clone, NC, "navidrome")
+    new = "deluan/navidrome:0.99.1"
+    prev = "deluan/navidrome:0.1.0"   # an invented older step, for the static cases
     frm, to = {"navidrome": old}, {"navidrome": new}
     move = (NC, swap_image("navidrome", old, new))
     good = tr_entry(frm, to, {"navidrome": TR_D1})
@@ -367,7 +394,7 @@ def test_record_cases(clone):
     case_tr_move("INCONCLUSIVE: the registry cannot be asked", clone,
                  [move, (NF, tr_append(good))], 2, ("could not be asked",), {})
     case_tr_move("DECOY: the entry only in a COMMENT under update_ladder", clone,
-                 [move, (NF, lambda t: t.rstrip("\n") + "\nupdate_ladder:\n  # " + good.strip() + "\n")], 1,
+                 [move, (NF, lambda t: strip_ladder(t).rstrip("\n") + "\nupdate_ladder:\n  # " + good.strip() + "\n")], 1,
                  ("holds no entry",), table)
     case_tr_move("DECOY: the entry only in README.md", clone,
                  [move, ("README.md", lambda t: t + "\n" + good + "\n")], 1,
@@ -385,21 +412,42 @@ def test_record_cases(clone):
     case_tr_move("DECOY: a ref moves in a compose COMMENT only", clone,
                  [(NC, lambda t: t + "\n# was: deluan/navidrome:0.63.2\n")], 0, (), table)
     print("-- test-record (static): the newest step IS the compose")
-    case_tr_static("GENUINE: a ladder whose head is the compose", clone,
-                   [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))], 0)
+    def ladder_of(*lines):
+        """Replace the template's ladder with exactly these entry lines."""
+        return lambda t: tr_append("\n".join(lines))(strip_ladder(t))
+    head = tr_entry({"navidrome": prev}, frm, {"navidrome": TR_D1})
+    case_tr_static("GENUINE: a ladder whose head is the compose", clone, [(NF, ladder_of(head))], 0)
     case_tr_static("FACT: the compose moved past the ladder's head", clone,
-                   [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}))),
-                    move], 1, ("not the ladder's newest step",))
+                   [(NF, ladder_of(head)), move], 1, ("not the ladder's newest step",))
     case_tr_static("FACT: a line that is not one JSON entry", clone,
-                   [(NF, lambda t: t + "\nupdate_ladder:\n  - from: x\n")], 1, ("not a one-line JSON entry",))
+                   [(NF, lambda t: strip_ladder(t) + "\nupdate_ladder:\n  - from: x\n")], 1, ("not a one-line JSON entry",))
     case_tr_static("FACT: a gap between steps", clone,
-                   [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.62.0"}, {"navidrome": "deluan/navidrome:0.63.0"}, {"navidrome": TR_D1})
-                                   + "\n" + tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))],
+                   [(NF, ladder_of(tr_entry({"navidrome": "deluan/navidrome:0.0.1"}, {"navidrome": "deluan/navidrome:0.0.2"}, {"navidrome": TR_D1}), head))],
                    1, ("the ladder has a gap",))
     case_tr_static("FACT: a failed verdict sits in the ladder", clone,
-                   [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}, verdict="failed")))],
+                   [(NF, ladder_of(tr_entry({"navidrome": prev}, frm, {"navidrome": TR_D1}, verdict="failed")))],
                    1, ("not allowed in a ladder",))
 
+    # ── `09` §6.4 part 5 (v0.268.0 on the box): every step but the newest carries its OWN definition
+    # at steps/<StepKey(to)>.yml — the box climbs one step at a time and pins that file. ──────────
+    print("-- test-record (static): every intermediate step carries its own definition")
+    sys.path.insert(0, os.path.join(ROOT, "scripts"))
+    import ladder as _l
+    mid = {"navidrome": "deluan/navidrome:0.2.0"}
+    two = ladder_of(tr_entry({"navidrome": prev}, mid, {"navidrome": TR_D1}),
+                    tr_entry(mid, frm, {"navidrome": TR_D1}))
+    step_rel = "templates/navidrome/" + _l.step_file(mid)
+    step_body = lambda t: swap_image("navidrome", old, mid["navidrome"])(io.open(os.path.join(clone, NC), encoding="utf-8").read())
+    case_tr_static("FACT: a two-step ladder with NO steps/ file for the first step", clone,
+                   [(NF, two)], 1, ("has no definition",))
+    case_tr_static("GENUINE: a two-step ladder whose first step carries its definition", clone,
+                   [(NF, two), (step_rel, step_body)], 0)
+    case_tr_static("DECOY: the steps/ file has the right NAME and names the head's image", clone,
+                   [(NF, two), (step_rel, lambda t: io.open(os.path.join(clone, NC), encoding="utf-8").read())],
+                   1, ("names",))
+    case_tr_static("DECOY: the step's definition sits beside the template under another name", clone,
+                   [(NF, two), ("templates/navidrome/steps/0.2.0.yml", step_body)], 1, ("has no definition",))
+
 def main():
     gate = os.path.join(ROOT, "scripts", "check-engine-major.py")
     if not os.path.isfile(gate):
@@ -409,6 +457,12 @@ def main():
     try:
         KIMAI = "templates/kimai/docker-compose.yml"
         DOCMOST = "templates/docmost/docker-compose.yml"
+        # The engine ref is READ from the clone, not typed: the night of 2026-09-23 moved kimai-db
+        # 11.6 -> 11.8 through its own test record, and a literal here broke every case below
+        # ("fixture drifted") without a single gate changing. R-663.
+        KDB = cur_image(clone, KIMAI, "kimai-db")
+        if not KDB.startswith("mariadb:11."):
+            raise SystemExit("kimai-db is %s — the cases below assume a MariaDB 11 line; fixture drifted" % KDB)
 
         # ── THE FACTS: these must be refused ─────────────────────────────────────────────────
         out = case("FACT: docmost-postgres 16-alpine -> 17-alpine bundled with the app bump", clone,
@@ -424,20 +478,20 @@ def main():
         # bookstack 0b73e5e shape — two migrations behind one edge — and stays refused.
         case("FACT: kimai-db 11.6 -> 12.3 BUNDLED with the kimai app bump", clone,
              [(KIMAI, lambda t: swap_image("kimai", "kimai/kimai2:apache-2.57.0", "kimai/kimai2:apache-2.58.0")(
-                 swap_image("kimai-db", "mariadb:11.6", "mariadb:12.3")(t)))],
+                 swap_image("kimai-db", KDB, "mariadb:12.3")(t)))],
              expect_rc=1, must_contain=("IN THE SAME COMMIT as kimai", "OWN EDGE", "R-450"))
         case("FACT: kimai-db mariadb:11.6 -> mariadb:lts (major unreadable)", clone,
-             [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:lts"))],
+             [(KIMAI, swap_image("kimai-db", KDB, "mariadb:lts"))],
              expect_rc=2, must_contain=("INCONCLUSIVE",))
 
         # ── THE GENUINE ARTICLES: these must pass ────────────────────────────────────────────
-        case("GENUINE: kimai-db mariadb:11.6 -> 11.8 (within major)", clone,
-             [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:11.8"))],
+        case("GENUINE: kimai-db mariadb:11.x -> 11.99 (within major)", clone,
+             [(KIMAI, swap_image("kimai-db", KDB, "mariadb:11.99"))],
              expect_rc=0, must_contain=("engine-major gate OK",))
         # R-469: the LIFT itself. A MariaDB major ALONE in its template is now permitted, and the
         # gate says so by name rather than passing in silence.
         case("GENUINE: kimai-db mariadb:11.6 -> 12.3 ALONE (the R-469 lift)", clone,
-             [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:12.3"))],
+             [(KIMAI, swap_image("kimai-db", KDB, "mariadb:12.3"))],
              expect_rc=0, must_contain=("ALLOWED", "kimai-db", "mariadb 11 -> 12", "R-469"))
 
         # ── THE DECOYS: the label moves, the fact does not — these must pass ─────────────────

scripts/test_ladder_writer.py

@@ -28,7 +28,14 @@ spec.loader.exec_module(ut)
 D = "sha256:" + "c" * 64
 
 
-def bench(verdict="proven", peak=0.41, marks=(), frm="0.64.0", to="0.64.1"):
+# READ, never typed (R-663): navidrome moved 0.64.0 -> 0.64.1 on 2026-09-23 night and a literal here
+# broke two of these tests without the writer changing.
+LIVE = ladder.images_in(open(os.path.join(ROOT, "templates", "navidrome", "docker-compose.yml")).read())["navidrome"].split(":")[1]
+NEXT = LIVE + "-next"
+AFTER = LIVE + "-after"
+
+
+def bench(verdict="proven", peak=0.41, marks=(), frm=LIVE, to=NEXT):
     return {"harness_version": 3, "app": "navidrome", "verdict": verdict,
             "from": {"navidrome": "deluan/navidrome:" + frm}, "to": {"navidrome": "deluan/navidrome:" + to},
             "measured_at": "2026-09-23T22:00:00Z", "marks": list(marks),
@@ -41,11 +48,12 @@ class WriterTest(unittest.TestCase):
         shutil.copytree(os.path.join(ROOT, "templates", "navidrome"),
                         os.path.join(self.tmp, "templates", "navidrome"))
         self.fy = os.path.join(self.tmp, "templates", "navidrome", ".felhom.yml")
-        # start from a template WITHOUT a ladder, at 0.64.0 (the live pin tonight)
+        # start from a template WITHOUT a ladder, at the live pin
         text = open(self.fy).read()
         if "update_ladder:" in text:
             text = text[:text.index("\n# update_ladder")] + "\n"
             open(self.fy, "w").write(text)
+        shutil.rmtree(os.path.join(self.tmp, "templates", "navidrome", "steps"), ignore_errors=True)
         self._orig = image_digest.resolve
         image_digest.resolve = lambda ref: (D, None)
 
@@ -67,11 +75,11 @@ class WriterTest(unittest.TestCase):
         self.assertEqual(rc, 0, out)
         entries, _, errs = ladder.parse(open(self.fy).read())
         self.assertEqual(errs, [])
-        self.assertEqual(entries[-1]["to"], {"navidrome": "deluan/navidrome:0.64.1"})
+        self.assertEqual(entries[-1]["to"], {"navidrome": "deluan/navidrome:" + NEXT})
         self.assertEqual(entries[-1]["memory_peak_pct"], 41.0)   # a PERCENT, from the watch's fraction
         self.assertEqual(entries[-1]["digest"], {"navidrome": D})
         comp = open(os.path.join(self.tmp, "templates", "navidrome", "docker-compose.yml")).read()
-        self.assertEqual(ladder.images_in(comp), {"navidrome": "deluan/navidrome:0.64.1"})
+        self.assertEqual(ladder.images_in(comp), {"navidrome": "deluan/navidrome:" + NEXT})
         import subprocess
         r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp,
                             "navidrome"], capture_output=True, text=True)
@@ -88,7 +96,7 @@ class WriterTest(unittest.TestCase):
         self.assertNotIn("update_ladder:", open(self.fy).read())
 
     def test_refuses_when_the_template_is_not_at_from(self):
-        rc, out = self.run_writer(bench(frm="0.63.2"))
+        rc, out = self.run_writer(bench(frm="0.0.1"))
         self.assertEqual(rc, 1)
         self.assertIn("the template is at", out)
 
@@ -98,6 +106,27 @@ class WriterTest(unittest.TestCase):
         e = ladder.parse(open(self.fy).read())[0][-1]
         self.assertEqual(e["marks"], {"files_may_change": True, "needs_person": None, "memory_tight": True})
 
+    def test_a_second_step_keeps_the_first_steps_definition(self):
+        """`09` §6.4 part 5: writing step 2 turns step 1 into an intermediate step — its OWN definition
+        (the compose as it stood, fixes included) is kept at steps/<key>.yml, and the gate accepts the
+        two-step ladder. RED-PROOF (REPORT.md): drop the STEP block in write_ladder — the gate refuses
+        at "has no definition"."""
+        rc, out = self.run_writer(bench())
+        self.assertEqual(rc, 0, out)
+        comp_p = os.path.join(self.tmp, "templates", "navidrome", "docker-compose.yml")
+        # a fix flows into the step-1 definition after it was published (the romm f4eb94f shape)
+        open(comp_p, "a").write("# a fix that flowed after step 1\n")
+        at_step1 = open(comp_p).read()
+        rc, out = self.run_writer(bench(frm=NEXT, to=AFTER))
+        self.assertEqual(rc, 0, out)
+        sp = os.path.join(self.tmp, "templates", "navidrome", ladder.step_file({"navidrome": "deluan/navidrome:" + NEXT}))
+        self.assertTrue(os.path.isfile(sp), out)
+        self.assertEqual(open(sp).read(), at_step1, "the step file must be the definition AS SERVED, fixes included")
+        import subprocess
+        r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp,
+                            "navidrome"], capture_output=True, text=True)
+        self.assertEqual(r.returncode, 0, r.stdout)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

scripts/test_upgrade_bench.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""test_upgrade_bench.py — the test bench's two night-of-2026-09-23 faults (R-653, R-656). No Docker.
+
+    python3 scripts/test_upgrade_bench.py
+"""
+import importlib.util
+import os
+import shutil
+import tempfile
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+spec = importlib.util.spec_from_file_location("upgrade_test_mod", os.path.join(HERE, "upgrade-test.py"))
+ut = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(ut)
+
+
+class LoadVerdict(unittest.TestCase):
+    """R-653. RED-PROOF (REPORT.md): make load_verdict always return "reached" — the ghost case fails."""
+
+    def test_every_request_errored_is_inconclusive(self):
+        # ghost's first watch, 2026-09-23 night: 11 797 requests, all `err`
+        self.assertEqual(ut.load_verdict(11797, {"err": 11797}), "inconclusive")
+
+    def test_answers_of_any_code_are_the_app_answering(self):
+        self.assertEqual(ut.load_verdict(11429, {"200": 5712, "401": 5717}), "reached")
+        self.assertEqual(ut.load_verdict(10, {"302": 5, "err": 5}), "reached")
+
+    def test_under_half_is_inconclusive_and_none_is_inconclusive(self):
+        self.assertEqual(ut.load_verdict(10, {"200": 4, "err": 6}), "inconclusive")
+        self.assertEqual(ut.load_verdict(0, {}), "inconclusive")
+
+
+class ClearScratch(unittest.TestCase):
+    """R-656. RED-PROOF (REPORT.md): make clear_scratch_folders return [] without removing — the first
+    test fails with the last run's config still there."""
+
+    def setUp(self):
+        self.root = tempfile.mkdtemp(prefix="bench-scratch-")
+        self.hdd = os.path.join(self.root, "hdd")
+        self.env = {"HDD_PATH": self.hdd, "USERDATA_PATH": self.hdd + "/userdata"}
+        self.said = []
+
+    def tearDown(self):
+        shutil.rmtree(self.root, ignore_errors=True)
+
+    def plant(self, rel):
+        p = os.path.join(self.hdd, rel, "config", "config.php")
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        open(p, "w").write("last run")
+        return p
+
+    def test_the_apps_own_folders_are_cleared_and_said(self):
+        mine = self.plant("appdata/nextcloud")
+        other = self.plant("appdata/immich")
+        comp = "services:\n  nextcloud:\n    volumes:\n      - ${HDD_PATH}/appdata/nextcloud:/var/www/html\n"
+        got = ut.clear_scratch_folders(comp, self.env, self.said.append)
+        self.assertFalse(os.path.exists(mine), "the last run's files are still there")
+        self.assertTrue(os.path.exists(other), "another app's folder was touched")
+        self.assertEqual(len(got), 1)
+        self.assertTrue(any("cleared before FROM" in s for s in self.said))
+
+    def test_never_a_bare_root_never_outside(self):
+        keep = self.plant("appdata/x")
+        comp = ("    volumes:\n      - ${HDD_PATH}:/data\n      - ${HDD_PATH}/:/d2\n"
+                "      - ${HDD_PATH}/../escape:/e\n")
+        got = ut.clear_scratch_folders(comp, self.env, self.said.append)
+        self.assertEqual(got, [])
+        self.assertTrue(os.path.exists(keep))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

scripts/upgrade-test.py

@@ -465,12 +465,50 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P
     tight = [n for n, p in per.items() if _tight_pct(p) is not None and _tight_pct(p) > MEMORY_TIGHT]
     rec = {"soak_s": round(time.time() - t0, 1), "requested_s": seconds, "requests": hits["n"],
            "codes": hits["codes"], "first_kill": first_bad, "containers": per,
-           "unmeasured": [n for n, p in per.items() if not p["measured"]]}
+           "unmeasured": [n for n, p in per.items() if not p["measured"]],
+           "load": load_verdict(hits["n"], hits["codes"])}
     marks = ["memory_tight"] if tight and not killed else []
     say(f"memory watch: killed={killed} tight={tight} requests={hits['n']} codes={hits['codes']}")
     return rec, killed, marks
 
 
+def load_verdict(requests: int, codes: dict) -> str:
+    """R-653: did the watch's load REACH the app? `reached` when at least half its requests got any HTTP
+    answer (a 401 or a 30x is the app answering); `inconclusive` otherwise. Measured 2026-09-23 night:
+    ghost's and nextcloud's first ten-minute watches sent 11 797 and 9 427 requests and EVERY one was
+    `err` — memory measured, app idle — and the harness wrote `proven` over it."""
+    if requests <= 0:
+        return "inconclusive"
+    answered = sum(n for c, n in codes.items() if c != "err")
+    return "reached" if answered * 2 >= requests else "inconclusive"
+
+
+def clear_scratch_folders(compose_text: str, env: dict, say) -> list:
+    """R-656: before FROM, remove the app's OWN scratch drive folders — every `${HDD_PATH}/…`,
+    `${USERDATA_PATH}/…` and `${IMPORT_PATH}/…` bind the compose names — so a re-run of the same app does
+    not start on the last run's files (nextcloud's second run never installed, 2026-09-23 night: `occ
+    status: installed: false` over the first run's config/). `compose down -v` removes named volumes, not
+    bind-mounted host folders. NEVER a bare root, never a path outside the scratch roots. Returns what was
+    removed, and says so either way."""
+    roots = {k: env.get(k) for k in ("HDD_PATH", "USERDATA_PATH", "IMPORT_PATH") if env.get(k)}
+    removed = []
+    for var, rel in re.findall(r"\$\{(HDD_PATH|USERDATA_PATH|IMPORT_PATH)\}(/[^:\s\"']*)", compose_text):
+        root = roots.get(var)
+        if not root:
+            continue
+        rel = rel.strip("/")
+        target = os.path.realpath(os.path.join(root, rel))
+        safe_root = os.path.realpath(root)
+        if not rel or target == safe_root or not target.startswith(safe_root + os.sep):
+            say(f"scratch folder NOT cleared (outside or equal to {var}={root}): {rel!r}")
+            continue
+        if os.path.exists(target):
+            shutil.rmtree(target, ignore_errors=False)
+            removed.append(target)
+    say(f"scratch drive folders cleared before FROM (R-656): {removed or 'none existed'}")
+    return removed
+
+
 MIGRATION_RE = re.compile(
     r"migrat|upgrad|schema|alter table|CREATE TABLE|InnoDB: Upgrad|mysql_upgrade|"
     r"mariadb-upgrade|Running .* migration|Applying|db:migrate",
@@ -558,6 +596,7 @@ def run_edge(edge_id: str) -> dict:
 
     try:
         # --- 1. FROM ---
+        rec["scratch_cleared"] = clear_scratch_folders(compose_text, env, say)
         say(f"{edge_id}: deploying {app} at FROM {e['frm']}")
         render(app, e["frm"], workdir, env, e.get("template"))
         up = compose(workdir, project, "up", "-d")
@@ -652,6 +691,11 @@ def run_edge(edge_id: str) -> dict:
             if killed:
                 rec["verdict"] = "failed"
                 say("VERDICT -> failed: the new version was OOM-killed or restarted under light load")
+            elif mem.get("load") != "reached":
+                rec["verdict"] = "inconclusive"
+                rec["abort_detail"] = (f"memory watch: fewer than half of its {mem.get('requests')} requests reached "
+                                       f"the app ({mem.get('codes')}) — memory measured on an idle app (R-653)")
+                say("VERDICT -> inconclusive: " + rec["abort_detail"])
 
         # --- 6. the ABORT ---
         say(f"{edge_id}: ABORT — putting the FROM images back")
@@ -778,6 +822,16 @@ def write_ladder(argv) -> int:
     if probs:
         print(f"REFUSED {app}: the entry would not be well-formed: {probs}")
         return 1
+    # `09` §6.4 part 5: the step being SUPERSEDED keeps its own definition. When the ladder's head is the
+    # compose as it stands, that compose — the head's images with every fix that flowed since — becomes
+    # steps/<step_key(head.to)>.yml, the file a box one step behind will pin (check-test-record.py rule 4).
+    prior, _, _ = ladder.parse(fy_p.read_text())
+    if prior and prior[-1].get("to") == cur:
+        sp = tdir / ladder.step_file(cur)
+        if not sp.exists():
+            sp.parent.mkdir(parents=True, exist_ok=True)
+            sp.write_text(comp)
+            print(f"STEP {app}: the superseded step {cur} keeps its definition at {ladder.step_file(cur)}")
     # move the compose, per service, on that service's own image: line
     out, svc = [], None
     for line in comp.splitlines():