197 lines
11 KiB
Python
197 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""r1177_night.py — D22 proof on scratch guest 9202 (night 2026-10-21), built on the data-loss hunt's r1_proof.py (C3).
|
|
Cases: 3 normal restore (unchanged), 5 damaged unit (put back), 7 controller killed between the volume step and the
|
|
replay (put back on start). The earlier docstring follows.
|
|
|
|
r1_proof.py — C3 proof of candidate R1/D4 on scratch guest 9202 (data-loss hunt 2026-10-20).
|
|
|
|
Claim under test: a LOCAL (Tier-1 unit) restore that fails after it has started replacing the app's volumes
|
|
restarts the app on whatever is left, and keeps NO copy of the live state it replaced — so the household's newer
|
|
data is gone and nothing can put it back (unlike the off-site path, which got „put back exactly" in v0.336.0).
|
|
|
|
Method (mechanism proof — markers are written with psql inside the app's own database container, as the
|
|
2026-10-19 R-893 measurement did; the household-facing acts are the dashboard's own endpoints):
|
|
1 deploy Rallly (throwaway) through POST /api/stacks/rallly/deploy
|
|
2 marker A; „Mentés most" = POST /api/backup/run (whole box, scratch); wait
|
|
3 POSITIVE CONTROL: marker B (live), unit restore (POST /backup/restore) -> A present, B gone (expected rewind)
|
|
4 marker C (the household's newest live row); THE FAULT: truncate the unit's DB volume tar to half
|
|
5 unit restore again -> observe: the answer, the app's state, what the DB holds, any undo/safety copy on disk
|
|
6 teardown: remove Rallly with its data and backups through the product; verify nothing left
|
|
Env: CTL_BASE, CTL_DOMAIN, CTL_PW_ENV (see tools/ctl.py). Prints a timestamped log (no secrets).
|
|
"""
|
|
import json, os, re, subprocess, sys, time
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/night-2026-10-21/tools")
|
|
import ctl # noqa: E402
|
|
|
|
APP = "rallly"
|
|
DOMAIN = os.environ["CTL_DOMAIN"]
|
|
|
|
|
|
def say(*a):
|
|
print(time.strftime("%H:%M:%S", time.gmtime()) + "Z " + " ".join(str(x) for x in a), flush=True)
|
|
|
|
|
|
def guest(script, timeout=600):
|
|
r = subprocess.run(["ssh", "hp", "export LC_ALL=C; cat > /tmp/r1p.sh; pct push 9202 /tmp/r1p.sh /tmp/r1p.sh;"
|
|
" pct exec 9202 -- bash /tmp/r1p.sh; pct exec 9202 -- rm -f /tmp/r1p.sh; rm -f /tmp/r1p.sh"],
|
|
input=script, capture_output=True, text=True, timeout=timeout)
|
|
return (r.stdout + r.stderr).strip()
|
|
|
|
|
|
def psql(sql):
|
|
return guest(f"""C=$(docker ps --format '{{{{.Names}}}}' | grep -E '^rallly.*(postgres|db)' | head -1)
|
|
U=$(docker exec $C printenv POSTGRES_USER); D=$(docker exec $C printenv POSTGRES_DB)
|
|
docker exec -i $C psql -v ON_ERROR_STOP=1 -U "$U" -d "$D" -tA <<'SQL'
|
|
{sql}
|
|
SQL""")
|
|
|
|
|
|
def stack():
|
|
code, d = ctl.call("GET", f"/api/stacks/{APP}")
|
|
return (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
|
|
|
|
def wait_backup(t_cap=1200):
|
|
t0 = time.time()
|
|
time.sleep(5)
|
|
while time.time() - t0 < t_cap:
|
|
code, d = ctl.call("GET", "/api/backup/status")
|
|
dd = (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
if not dd.get("running"):
|
|
return dd
|
|
time.sleep(5)
|
|
return {"timeout": True}
|
|
|
|
|
|
def restore(tag):
|
|
code, d = ctl.call("GET", f"/api/backup/snapshots?stack={APP}")
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
snaps = data if isinstance(data, list) else next((data[k] for k in ("snapshots", "items", "restore_points")
|
|
if isinstance(data, dict) and isinstance(data.get(k), list)), [])
|
|
say(f"[{tag}] restore points offered: {json.dumps(snaps, ensure_ascii=False)[:400]}")
|
|
sid = snaps[0].get("id") or snaps[0].get("snapshot_id") or snaps[0].get("short_id")
|
|
out = subprocess.run(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", ctl.HOST, "-H", "Cookie: " + ctl._SESS,
|
|
"-X", "POST", "--data-urlencode", "_csrf=" + ctl._CSRF, "--data-urlencode", "stack_name=" + APP,
|
|
"--data-urlencode", "snapshot_id=" + sid, ctl.BASE + "/backup/restore"],
|
|
capture_output=True, text=True).stdout
|
|
say(f"[{tag}] POST /backup/restore -> {out.splitlines()[0] if out else '?'} "
|
|
f"{[l for l in out.splitlines() if l.lower().startswith('location')][:1]}")
|
|
t0, last = time.time(), None
|
|
while time.time() - t0 < 900:
|
|
code, d = ctl.call("GET", "/api/backup/restore-status")
|
|
dd = (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
cur = json.dumps(dd, ensure_ascii=False)[:600]
|
|
if cur != last:
|
|
say(f"[{tag}] restore-status: {cur}")
|
|
last = cur
|
|
if not dd.get("running") and time.time() - t0 > 8:
|
|
break
|
|
time.sleep(3)
|
|
time.sleep(20)
|
|
st = stack()
|
|
say(f"[{tag}] after: state={st.get('state')} hold={st.get('hold_reason')!r}")
|
|
|
|
|
|
def markers(tag):
|
|
say(f"[{tag}] DB markers: {psql('SELECT string_agg(m, \',\' ORDER BY m) FROM r1_marker;')!r}")
|
|
|
|
|
|
def main():
|
|
say("== 1 deploy")
|
|
if not stack().get("deployed"):
|
|
code, d = ctl.call("GET", f"/api/stacks/{APP}/deploy-fields")
|
|
fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
|
|
vals = {"DOMAIN": DOMAIN, "SUBDOMAIN": "r1night"}
|
|
for f in fields:
|
|
if f.get("env_var") in vals:
|
|
continue
|
|
if f.get("type") == "password":
|
|
import secrets
|
|
vals[f["env_var"]] = "R1-" + secrets.token_hex(10)
|
|
elif f.get("required") and f.get("default"):
|
|
vals[f["env_var"]] = f["default"]
|
|
code, d = ctl.call("POST", f"/api/stacks/{APP}/deploy", {"values": vals})
|
|
say("deploy ->", code, str(d)[:160])
|
|
for _ in range(90):
|
|
time.sleep(5)
|
|
st = stack()
|
|
if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") and st.get("state") in ("running", "unhealthy", "degraded"):
|
|
break
|
|
say("state:", stack().get("state"))
|
|
time.sleep(30)
|
|
say("== 2 marker A + backup")
|
|
say(psql("CREATE TABLE IF NOT EXISTS r1_marker(m text primary key); INSERT INTO r1_marker VALUES ('A') ON CONFLICT DO NOTHING;"))
|
|
markers("2")
|
|
code, d = ctl.call("POST", "/api/backup/run")
|
|
say("POST /api/backup/run ->", code, str(d)[:200])
|
|
say("backup finished:", json.dumps(wait_backup(), ensure_ascii=False)[:500])
|
|
unit = guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do "
|
|
f"[ -d $d ] && echo UNIT $d && find $d -maxdepth 3 -type f -printf '%TY-%Tm-%Td %TH:%TM %s %p\\n'; done")
|
|
say("unit on disk:\n" + unit)
|
|
say("== 3 POSITIVE CONTROL: marker B (live) then a normal unit restore")
|
|
say(psql("INSERT INTO r1_marker VALUES ('B');"))
|
|
markers("3-before")
|
|
restore("3")
|
|
time.sleep(15)
|
|
markers("3-after")
|
|
say("== 4 marker C (the newest live row), then THE FAULT: the unit's DB volume tar truncated to half")
|
|
say(psql("INSERT INTO r1_marker VALUES ('C');"))
|
|
markers("4-live")
|
|
say(guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do "
|
|
f"[ -d $d ] || continue; f=$(find $d -name '*.tar' -path '*volume*' | grep -iE 'postgres|db' | head -1); "
|
|
f"[ -n \"$f\" ] || f=$(find $d -name '*.tar' | head -1); s=$(stat -c %s \"$f\"); "
|
|
f"truncate -s $((s/2)) \"$f\"; echo \"TRUNCATED $f $s -> $(stat -c %s \"$f\")\"; done"))
|
|
say("== 5 the unit restore over the damaged copy (D22: the live data must come back, the app must run)")
|
|
restore("5")
|
|
time.sleep(30)
|
|
markers("5-after (want A,C)")
|
|
say("containers:\n" + guest("docker ps -a --format '{{.Names}} {{.Status}}' | grep -i rallly"))
|
|
say("undo copies left on disk:\n" + guest("find /mnt/felhom-drives /mnt/sys_drive /var/lib/felhom -path '*restore-undo*' 2>/dev/null | head"))
|
|
say("controller log (restore lines):\n" + guest("docker logs --since 10m felhom-controller 2>&1 | grep -iE 'restore|rallly|R-1177|R-893|put' | grep -vi 'GET /api' | tail -30"))
|
|
say("== 6 a fresh backup (the damaged unit is replaced) and marker D (the newest live row)")
|
|
code, d = ctl.call("POST", "/api/backup/run")
|
|
say("POST /api/backup/run ->", code, str(d)[:200])
|
|
say("backup finished:", json.dumps(wait_backup(), ensure_ascii=False)[:300])
|
|
say(psql("INSERT INTO r1_marker VALUES ('D');"))
|
|
markers("6-live (want A,C,D)")
|
|
say("== 7 the restore, the controller KILLED (kill -9) right after the volume step, before the replay")
|
|
restore_start("7")
|
|
say("killer:\n" + guest("""for i in $(seq 1 1200); do
|
|
if docker logs --since 3m felhom-controller 2>&1 | grep -q 'Restored 1 Docker volume(s) for rallly'; then
|
|
P=$(docker inspect -f '{{.State.Pid}}' felhom-controller); kill -9 $P; echo "KILLED pid $P at $(date -u +%T.%N)"; break; fi
|
|
sleep 0.25; done""", timeout=400))
|
|
say("waiting for the controller to come back")
|
|
for _ in range(60):
|
|
time.sleep(5)
|
|
try:
|
|
ctl._SESS = None
|
|
code, d = ctl.call("GET", "/api/backup/restore-status", timeout=20)
|
|
if code == "200":
|
|
break
|
|
except SystemExit:
|
|
continue
|
|
time.sleep(40)
|
|
code, d = ctl.call("GET", "/api/backup/restore-status")
|
|
say("[7] restore-status after the restart:", json.dumps((d.get("data") or {}) if isinstance(d, dict) else d, ensure_ascii=False)[:700])
|
|
markers("7-after (want A,C,D)")
|
|
say("containers:\n" + guest("docker ps -a --format '{{.Names}} {{.Status}}' | grep -i rallly"))
|
|
say("undo copies left on disk:\n" + guest("find /mnt/felhom-drives /mnt/sys_drive /var/lib/felhom -path '*restore-undo*' 2>/dev/null | head"))
|
|
say("controller log after the restart (appstop / restore):\n" + guest("docker logs --since 5m felhom-controller 2>&1 | grep -iE 'appstop|crash recovery|put back|R-1195|interrupted|restore record' | tail -30"))
|
|
|
|
|
|
def restore_start(tag):
|
|
code, d = ctl.call("GET", f"/api/backup/snapshots?stack={APP}")
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
snaps = data if isinstance(data, list) else next((data[k] for k in ("snapshots", "items", "restore_points")
|
|
if isinstance(data, dict) and isinstance(data.get(k), list)), [])
|
|
sid = snaps[0].get("id") or snaps[0].get("snapshot_id") or snaps[0].get("short_id")
|
|
out = subprocess.run(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", ctl.HOST, "-H", "Cookie: " + ctl._SESS,
|
|
"-X", "POST", "--data-urlencode", "_csrf=" + ctl._CSRF, "--data-urlencode", "stack_name=" + APP,
|
|
"--data-urlencode", "snapshot_id=" + sid, ctl.BASE + "/backup/restore"],
|
|
capture_output=True, text=True).stdout
|
|
say(f"[{tag}] POST /backup/restore -> {out.splitlines()[0] if out else '?'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|