#!/usr/bin/env python3 """walk28.py — ONE of the twenty-eight, the full rotation walk, through the product. Reuses `update-night-2026-09-21/walk.py` wholesale (R-161: do not rewrite the method). What is NEW here versus the update night is step 6: the app is RESTORED from its own backup and the seed read back a second time. The update night skipped that half. Every app gets a line even when it cannot be deployed. `inconclusive` is never collapsed into `failed` or `proven`, and an app with no non-browser seed route is `inconclusive` WITH WHAT WAS TRIED — never faked. """ import argparse, json, os, re, sys, time, traceback from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) NIGHT = os.path.join(os.path.dirname(HERE), "update-night-2026-09-21") sys.path.insert(0, NIGHT) sys.path.insert(0, HERE) import walk as w # noqa: E402 from fixtures28 import FIXTURES28 # noqa: E402 EDGES = json.load(open(os.path.join(HERE, "edges.json"))) if os.path.exists( os.path.join(HERE, "edges.json")) else {} META = json.load(open("/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/" "d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad/drift28/meta.json")) LOCK = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad/drill.lock" def drill_lock(fn, *a, **k): """The drill repo is ONE git clone; two workers pushing at once corrupt each other's index.""" for _ in range(600): try: fd = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY) os.close(fd) break except FileExistsError: time.sleep(1) try: return fn(*a, **k) finally: try: os.unlink(LOCK) except OSError: pass def main(): ap = argparse.ArgumentParser() ap.add_argument("app") ap.add_argument("--cap", type=int, default=1800) a = ap.parse_args() app = a.app d = os.path.join(HERE, "apps", app) os.makedirs(d, exist_ok=True) # per-worker session files: walk.py keeps the cookie in SC, and two workers racing on one # cookie file log each other out. Assigning the module global is enough — every helper reads # it at CALL time. w.SC = os.path.join("/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/" "d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad/w", app) os.makedirs(w.SC, exist_ok=True) import shutil shutil.copy("/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/" "d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad/.ctlpw", w.SC + "/.ctlpw") t0 = time.time() m = META.get(app, {}) fx = FIXTURES28.get(app) sub = getattr(fx, "sub", None) or app edge = EDGES.get(app) rec = {"harness_version": 2, "app": app, "venue": "guest 9202 demo-hp-scratch, controller 0.261.0, drill catalog", "class": m.get("class") or ("db" if m.get("engines") else "file-leg"), "from": {}, "to": {}, "verdict": "inconclusive", "deployed": False, "seed_route": None, "seed_read_before": False, "seed_read_after_update": False, "backup": None, "restore_verdict": "not-attempted", "seed_read_after_restore": False, "healthy_after": False, "migration_observed": None, "removed_clean": None, "edge": edge, "duration_s": 0, "measured_at": datetime.now(timezone.utc).isoformat(), "evidence": f"the-28-2026-09-22/apps/{app}/", "notes": []} w.say(f"==== {app} (sub={sub}, class={rec['class']}, edge={edge or 'none'})") try: w.login() # ---- 1 deploy at the LIVE pin ------------------------------------------------------- ok = w.deploy(app, sub) st = w.stack(app) rec["deployed"] = bool(st.get("deployed")) rec["from"] = (st.get("app_config") or {}).get("pinned_images") or {} if not rec["deployed"]: rec["verdict"] = "could-not-deploy" rec["notes"].append("deploy never reached `deployed` with a pin") raise SystemExit(0) front = w.app_curl(sub, "/") rec["front_door_after_deploy"] = {"rc": front[0], "code": front[1]} w.say(f" [1] front door {front[1]} controller state={st.get('state')}") # ---- 2/3 seed through the app's OWN front door, then read it back (C1) --------------- tok = None if fx is None: rec["seed_route"] = "none written" rec["notes"].append("no fixture: no non-browser seed route was written for this app") w.say(" [2] NO FIXTURE — recorded inconclusive for the data half, not faked") else: tok = fx.seed(w, sub, w.say) rec["seed_route"] = getattr(fx, "route", fx.__class__.__name__) if tok is None: rec["notes"].append("fixture ran and found no non-browser seed route: " + (getattr(fx, "tried", "") or "see log")) w.say(" [2] fixture found no route — inconclusive for the data half") else: rec["seed_read_before"] = bool(fx.verify(w, sub, tok, w.say)) w.say(f" [3] C1 readback before = {rec['seed_read_before']}") # ---- 4 „Mentés most" ----------------------------------------------------------------- w.backup_now(app) snaps = w.snapshots(app) rec["backup"] = {"snapshots_offered": len(snaps), "first": (snaps[0] if snaps else None)} w.say(f" [4] backups page offers {len(snaps)} restorable copy(ies)") # ---- 5 the edge, if one exists -------------------------------------------------------- if edge: h = drill_lock(w.drill_bump, app, edge["from"], edge["to"]) rec["drill_commit"] = h secs = w.sync_rescan(app, edge["to"].split(",")[0]) rec["badge_seconds"] = secs rec["badges"] = w.badges(app) w.say(f" [5] badge {rec['badges']}") ph = w.press_update(app) rec["phases"] = ph # R-621: the hold destroys the app's logs, so they are captured DURING the wait, # not after. press_update polls; this is the best moment available to a caller. logs = w.app_logs(app, 300) open(os.path.join(d, "app-logs-during-after.txt"), "w").write(logs or "") for line in (logs or "").split("\n"): if any(k in line.lower() for k in ("migrat", "upgrad", "schema", "alter table")): rec["migration_observed"] = line.strip()[:300] break obs = w.observables(app) rec["observables"] = obs rec["to"] = (w.stack(app).get("app_config") or {}).get("pinned_images") or {} # The Update can be REFUSED before any phase exists — `409 busy` while a backup or a # restore is in flight, which is the product guarding itself and is a RESULT, not an # error. An empty phase list then made `[-1]` raise, which cost rallly its whole walk. plist = ph if isinstance(ph, list) else (ph.get("phases") or []) last = plist[-1].get("phase") if plist else None if not last and isinstance(ph, dict) and ph.get("http") == "409": last = "refused-" + str((ph.get("refusal") or {}).get("data", {}).get("reason") or "409") rec["update_refusal"] = (ph.get("refusal") or {}).get("error") rec["update_phase_final"] = last if tok is not None: rec["seed_read_after_update"] = bool(fx.verify(w, sub, tok, w.say)) st = w.stack(app) rec["healthy_after"] = st.get("state") in ("running",) if last == "done" and (tok is None or rec["seed_read_after_update"]): rec["verdict"] = "proven" if tok is not None else "inconclusive" elif last == "failed": rec["verdict"] = "failed" else: rec["verdict"] = "no-edge" # ---- 6 RESTORE — the half the update night skipped ------------------------------------- if snaps: r = w.restore(app) rec["restore_raw"] = str(r)[:400] # A REFUSAL IS NOT A FAILURE, and collapsing the two would have mis-recorded the very # behaviour `07` §6.2 predicts. A class-A app's local copy holds no readable file leg, # and the product refuses rather than restoring a database over files it does not have # — naming the action that DOES work. The refusal travels as a `flash_error` on the # redirect, so it is read from there and quoted, urldecoded, in the record. from urllib.parse import unquote_plus loc = " ".join(r.get("location") or []) if isinstance(r, dict) else "" # NOT `m` — that name already holds this app's META entry, and shadowing it made the # teardown fall over with `'re.Match' object has no attribute 'get'` on paperless-ngx. fm = re.search(r"flash_error=([^&\s]+)", loc) rec["restore_refusal"] = unquote_plus(fm.group(1)) if fm else None time.sleep(15) st = w.stack(app) if rec["restore_refusal"]: rec["restore_verdict"] = "refused-with-a-sentence" elif st.get("state") in ("running", "unhealthy"): rec["restore_verdict"] = "ok" else: rec["restore_verdict"] = "failed" rec["restore_state_seen"] = st.get("state") if tok is not None: rec["seed_read_after_restore"] = bool(fx.verify(w, sub, tok, w.say)) w.say(f" [6] restore -> {rec['restore_verdict']}, seed back = " f"{rec['seed_read_after_restore']}") # WAIT FOR THE RESTORE TO SETTLE BEFORE REMOVING. `POST /backup/restore` answers 302 # and works in the background; a remove sent while its `compose up` is still running # tears down what exists and the restore then RE-CREATES it — the controller records # the app as removed and a container keeps restarting with a live traefik route. # Measured on gokapi at 11:34 (R-633). Leaving the race in would mean every later app # measured the harness rather than the product. for _ in range(36): s = w.stack(app) if not s.get("restore_running") and s.get("state") in ( "running", "unhealthy", "degraded", "stopped", "not_deployed", None): break time.sleep(5) time.sleep(10) else: rec["restore_verdict"] = "no-snapshot-offered" except SystemExit: pass except Exception as e: rec["notes"].append(f"harness error: {type(e).__name__}: {e}") open(os.path.join(d, "traceback.txt"), "w").write(traceback.format_exc()) w.say(f" !! {type(e).__name__}: {e}") finally: # ---- 7 remove, then R-626: check 60 s later that nothing came back -------------------- try: w.remove(app) time.sleep(60) back = w.guest(f"docker ps -a --format '{{{{.Names}}}}' | grep -x '{app}' || true; " f"ls /opt/docker/stacks/{app}/app.yaml 2>/dev/null || true") rec["removed_clean"] = (back.strip() == "") rec["remove_leftovers"] = back.strip()[:300] w.say(f" [7] 60 s after remove: clean={rec['removed_clean']} {back.strip()[:120]!r}") # drop the app's images BY NAME — never `prune` (rule 3); 9202's root disk is 28 GB imgs = [v for v in (m.get("all_images") or {}).values() if v] if imgs: w.guest("docker image rm -f " + " ".join(imgs) + " 2>&1 | tail -2") except Exception as e: rec["notes"].append(f"teardown error: {type(e).__name__}: {e}") rec["duration_s"] = round(time.time() - t0, 1) json.dump(rec, open(os.path.join(d, "verdict.json"), "w"), ensure_ascii=False, indent=2) open(os.path.join(d, "log.txt"), "w").write("\n".join(w.LOG) + "\n") w.say(f" [9] verdict {rec['verdict']} ({rec['duration_s']}s) -> apps/{app}/verdict.json") if __name__ == "__main__": main()