#!/usr/bin/env python3 """r1177_night.py — D22 proof on scratch guest 9202 (night 2026-10-21), built on the data-loss hunt's r1_proof.py (C3). Cases: 3 normal restore (unchanged), 5 damaged unit (put back), 7 controller killed between the volume step and the replay (put back on start). The earlier docstring follows. r1_proof.py — C3 proof of candidate R1/D4 on scratch guest 9202 (data-loss hunt 2026-10-20). Claim under test: a LOCAL (Tier-1 unit) restore that fails after it has started replacing the app's volumes restarts the app on whatever is left, and keeps NO copy of the live state it replaced — so the household's newer data is gone and nothing can put it back (unlike the off-site path, which got „put back exactly" in v0.336.0). Method (mechanism proof — markers are written with psql inside the app's own database container, as the 2026-10-19 R-893 measurement did; the household-facing acts are the dashboard's own endpoints): 1 deploy Rallly (throwaway) through POST /api/stacks/rallly/deploy 2 marker A; „Mentés most" = POST /api/backup/run (whole box, scratch); wait 3 POSITIVE CONTROL: marker B (live), unit restore (POST /backup/restore) -> A present, B gone (expected rewind) 4 marker C (the household's newest live row); THE FAULT: truncate the unit's DB volume tar to half 5 unit restore again -> observe: the answer, the app's state, what the DB holds, any undo/safety copy on disk 6 teardown: remove Rallly with its data and backups through the product; verify nothing left Env: CTL_BASE, CTL_DOMAIN, CTL_PW_ENV (see tools/ctl.py). Prints a timestamped log (no secrets). """ import json, os, re, subprocess, sys, time sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/night-2026-10-21/tools") import ctl # noqa: E402 APP = "rallly" DOMAIN = os.environ["CTL_DOMAIN"] def say(*a): print(time.strftime("%H:%M:%S", time.gmtime()) + "Z " + " ".join(str(x) for x in a), flush=True) def guest(script, timeout=600): r = subprocess.run(["ssh", "hp", "export LC_ALL=C; cat > /tmp/r1p.sh; pct push 9202 /tmp/r1p.sh /tmp/r1p.sh;" " pct exec 9202 -- bash /tmp/r1p.sh; pct exec 9202 -- rm -f /tmp/r1p.sh; rm -f /tmp/r1p.sh"], input=script, capture_output=True, text=True, timeout=timeout) return (r.stdout + r.stderr).strip() def psql(sql): return guest(f"""C=$(docker ps --format '{{{{.Names}}}}' | grep -E '^rallly.*(postgres|db)' | head -1) U=$(docker exec $C printenv POSTGRES_USER); D=$(docker exec $C printenv POSTGRES_DB) docker exec -i $C psql -v ON_ERROR_STOP=1 -U "$U" -d "$D" -tA <<'SQL' {sql} SQL""") def stack(): code, d = ctl.call("GET", f"/api/stacks/{APP}") return (d.get("data") or {}) if isinstance(d, dict) else {} def wait_backup(t_cap=1200): t0 = time.time() time.sleep(5) while time.time() - t0 < t_cap: code, d = ctl.call("GET", "/api/backup/status") dd = (d.get("data") or {}) if isinstance(d, dict) else {} if not dd.get("running"): return dd time.sleep(5) return {"timeout": True} def restore(tag): code, d = ctl.call("GET", f"/api/backup/snapshots?stack={APP}") data = d.get("data") if isinstance(d, dict) else None snaps = data if isinstance(data, list) else next((data[k] for k in ("snapshots", "items", "restore_points") if isinstance(data, dict) and isinstance(data.get(k), list)), []) say(f"[{tag}] restore points offered: {json.dumps(snaps, ensure_ascii=False)[:400]}") sid = snaps[0].get("id") or snaps[0].get("snapshot_id") or snaps[0].get("short_id") out = subprocess.run(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", ctl.HOST, "-H", "Cookie: " + ctl._SESS, "-X", "POST", "--data-urlencode", "_csrf=" + ctl._CSRF, "--data-urlencode", "stack_name=" + APP, "--data-urlencode", "snapshot_id=" + sid, ctl.BASE + "/backup/restore"], capture_output=True, text=True).stdout say(f"[{tag}] POST /backup/restore -> {out.splitlines()[0] if out else '?'} " f"{[l for l in out.splitlines() if l.lower().startswith('location')][:1]}") t0, last = time.time(), None while time.time() - t0 < 900: code, d = ctl.call("GET", "/api/backup/restore-status") dd = (d.get("data") or {}) if isinstance(d, dict) else {} cur = json.dumps(dd, ensure_ascii=False)[:600] if cur != last: say(f"[{tag}] restore-status: {cur}") last = cur if not dd.get("running") and time.time() - t0 > 8: break time.sleep(3) time.sleep(20) st = stack() say(f"[{tag}] after: state={st.get('state')} hold={st.get('hold_reason')!r}") def markers(tag): say(f"[{tag}] DB markers: {psql('SELECT string_agg(m, \',\' ORDER BY m) FROM r1_marker;')!r}") def main(): say("== 1 deploy") if not stack().get("deployed"): code, d = ctl.call("GET", f"/api/stacks/{APP}/deploy-fields") fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or [] vals = {"DOMAIN": DOMAIN, "SUBDOMAIN": "r1night"} for f in fields: if f.get("env_var") in vals: continue if f.get("type") == "password": import secrets vals[f["env_var"]] = "R1-" + secrets.token_hex(10) elif f.get("required") and f.get("default"): vals[f["env_var"]] = f["default"] code, d = ctl.call("POST", f"/api/stacks/{APP}/deploy", {"values": vals}) say("deploy ->", code, str(d)[:160]) for _ in range(90): time.sleep(5) st = stack() if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") and st.get("state") in ("running", "unhealthy", "degraded"): break say("state:", stack().get("state")) time.sleep(30) say("== 2 marker A + backup") say(psql("CREATE TABLE IF NOT EXISTS r1_marker(m text primary key); INSERT INTO r1_marker VALUES ('A') ON CONFLICT DO NOTHING;")) markers("2") code, d = ctl.call("POST", "/api/backup/run") say("POST /api/backup/run ->", code, str(d)[:200]) say("backup finished:", json.dumps(wait_backup(), ensure_ascii=False)[:500]) unit = guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do " f"[ -d $d ] && echo UNIT $d && find $d -maxdepth 3 -type f -printf '%TY-%Tm-%Td %TH:%TM %s %p\\n'; done") say("unit on disk:\n" + unit) say("== 3 POSITIVE CONTROL: marker B (live) then a normal unit restore") say(psql("INSERT INTO r1_marker VALUES ('B');")) markers("3-before") restore("3") time.sleep(15) markers("3-after") say("== 4 marker C (the newest live row), then THE FAULT: the unit's DB volume tar truncated to half") say(psql("INSERT INTO r1_marker VALUES ('C');")) markers("4-live") say(guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do " f"[ -d $d ] || continue; f=$(find $d -name '*.tar' -path '*volume*' | grep -iE 'postgres|db' | head -1); " f"[ -n \"$f\" ] || f=$(find $d -name '*.tar' | head -1); s=$(stat -c %s \"$f\"); " f"truncate -s $((s/2)) \"$f\"; echo \"TRUNCATED $f $s -> $(stat -c %s \"$f\")\"; done")) say("== 5 the unit restore over the damaged copy (D22: the live data must come back, the app must run)") restore("5") time.sleep(30) markers("5-after (want A,C)") say("containers:\n" + guest("docker ps -a --format '{{.Names}} {{.Status}}' | grep -i rallly")) say("undo copies left on disk:\n" + guest("find /mnt/felhom-drives /mnt/sys_drive /var/lib/felhom -path '*restore-undo*' 2>/dev/null | head")) say("controller log (restore lines):\n" + guest("docker logs --since 10m felhom-controller 2>&1 | grep -iE 'restore|rallly|R-1177|R-893|put' | grep -vi 'GET /api' | tail -30")) say("== 6 a fresh backup (the damaged unit is replaced) and marker D (the newest live row)") code, d = ctl.call("POST", "/api/backup/run") say("POST /api/backup/run ->", code, str(d)[:200]) say("backup finished:", json.dumps(wait_backup(), ensure_ascii=False)[:300]) say(psql("INSERT INTO r1_marker VALUES ('D');")) markers("6-live (want A,C,D)") say("== 7 the restore, the controller KILLED (kill -9) right after the volume step, before the replay") restore_start("7") say("killer:\n" + guest("""for i in $(seq 1 1200); do if docker logs --since 3m felhom-controller 2>&1 | grep -q 'Restored 1 Docker volume(s) for rallly'; then P=$(docker inspect -f '{{.State.Pid}}' felhom-controller); kill -9 $P; echo "KILLED pid $P at $(date -u +%T.%N)"; break; fi sleep 0.25; done""", timeout=400)) say("waiting for the controller to come back") for _ in range(60): time.sleep(5) try: ctl._SESS = None code, d = ctl.call("GET", "/api/backup/restore-status", timeout=20) if code == "200": break except SystemExit: continue time.sleep(40) code, d = ctl.call("GET", "/api/backup/restore-status") say("[7] restore-status after the restart:", json.dumps((d.get("data") or {}) if isinstance(d, dict) else d, ensure_ascii=False)[:700]) markers("7-after (want A,C,D)") say("containers:\n" + guest("docker ps -a --format '{{.Names}} {{.Status}}' | grep -i rallly")) say("undo copies left on disk:\n" + guest("find /mnt/felhom-drives /mnt/sys_drive /var/lib/felhom -path '*restore-undo*' 2>/dev/null | head")) say("controller log after the restart (appstop / restore):\n" + guest("docker logs --since 5m felhom-controller 2>&1 | grep -iE 'appstop|crash recovery|put back|R-1195|interrupted|restore record' | tail -30")) def restore_start(tag): code, d = ctl.call("GET", f"/api/backup/snapshots?stack={APP}") data = d.get("data") if isinstance(d, dict) else None snaps = data if isinstance(data, list) else next((data[k] for k in ("snapshots", "items", "restore_points") if isinstance(data, dict) and isinstance(data.get(k), list)), []) sid = snaps[0].get("id") or snaps[0].get("snapshot_id") or snaps[0].get("short_id") out = subprocess.run(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", ctl.HOST, "-H", "Cookie: " + ctl._SESS, "-X", "POST", "--data-urlencode", "_csrf=" + ctl._CSRF, "--data-urlencode", "stack_name=" + APP, "--data-urlencode", "snapshot_id=" + sid, ctl.BASE + "/backup/restore"], capture_output=True, text=True).stdout say(f"[{tag}] POST /backup/restore -> {out.splitlines()[0] if out else '?'}") if __name__ == "__main__": main()