154 lines
8.0 KiB
Python
154 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
|
"""r1_proof.py — C3 proof of candidate R1/D4 on scratch guest 9202 (data-loss hunt 2026-10-20).
|
|
|
|
Claim under test: a LOCAL (Tier-1 unit) restore that fails after it has started replacing the app's volumes
|
|
restarts the app on whatever is left, and keeps NO copy of the live state it replaced — so the household's newer
|
|
data is gone and nothing can put it back (unlike the off-site path, which got „put back exactly" in v0.336.0).
|
|
|
|
Method (mechanism proof — markers are written with psql inside the app's own database container, as the
|
|
2026-10-19 R-893 measurement did; the household-facing acts are the dashboard's own endpoints):
|
|
1 deploy Rallly (throwaway) through POST /api/stacks/rallly/deploy
|
|
2 marker A; „Mentés most" = POST /api/backup/run (whole box, scratch); wait
|
|
3 POSITIVE CONTROL: marker B (live), unit restore (POST /backup/restore) -> A present, B gone (expected rewind)
|
|
4 marker C (the household's newest live row); THE FAULT: truncate the unit's DB volume tar to half
|
|
5 unit restore again -> observe: the answer, the app's state, what the DB holds, any undo/safety copy on disk
|
|
6 teardown: remove Rallly with its data and backups through the product; verify nothing left
|
|
Env: CTL_BASE, CTL_DOMAIN, CTL_PW_ENV (see tools/ctl.py). Prints a timestamped log (no secrets).
|
|
"""
|
|
import json, os, re, subprocess, sys, time
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/rehearsal-2026-10-20/tools")
|
|
import ctl # noqa: E402
|
|
|
|
APP = "rallly"
|
|
DOMAIN = os.environ["CTL_DOMAIN"]
|
|
|
|
|
|
def say(*a):
|
|
print(time.strftime("%H:%M:%S", time.gmtime()) + "Z " + " ".join(str(x) for x in a), flush=True)
|
|
|
|
|
|
def guest(script, timeout=600):
|
|
r = subprocess.run(["ssh", "demo-hp", "export LC_ALL=C; cat > /tmp/r1p.sh; pct push 9202 /tmp/r1p.sh /tmp/r1p.sh;"
|
|
" pct exec 9202 -- bash /tmp/r1p.sh; pct exec 9202 -- rm -f /tmp/r1p.sh; rm -f /tmp/r1p.sh"],
|
|
input=script, capture_output=True, text=True, timeout=timeout)
|
|
return (r.stdout + r.stderr).strip()
|
|
|
|
|
|
def psql(sql):
|
|
return guest(f"""C=$(docker ps --format '{{{{.Names}}}}' | grep -E '^rallly.*(postgres|db)' | head -1)
|
|
U=$(docker exec $C printenv POSTGRES_USER); D=$(docker exec $C printenv POSTGRES_DB)
|
|
docker exec -i $C psql -v ON_ERROR_STOP=1 -U "$U" -d "$D" -tA <<'SQL'
|
|
{sql}
|
|
SQL""")
|
|
|
|
|
|
def stack():
|
|
code, d = ctl.call("GET", f"/api/stacks/{APP}")
|
|
return (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
|
|
|
|
def wait_backup(t_cap=1200):
|
|
t0 = time.time()
|
|
time.sleep(5)
|
|
while time.time() - t0 < t_cap:
|
|
code, d = ctl.call("GET", "/api/backup/status")
|
|
dd = (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
if not dd.get("running"):
|
|
return dd
|
|
time.sleep(5)
|
|
return {"timeout": True}
|
|
|
|
|
|
def restore(tag):
|
|
code, d = ctl.call("GET", f"/api/backup/snapshots?stack={APP}")
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
snaps = data if isinstance(data, list) else next((data[k] for k in ("snapshots", "items", "restore_points")
|
|
if isinstance(data, dict) and isinstance(data.get(k), list)), [])
|
|
say(f"[{tag}] restore points offered: {json.dumps(snaps, ensure_ascii=False)[:400]}")
|
|
sid = snaps[0].get("id") or snaps[0].get("snapshot_id") or snaps[0].get("short_id")
|
|
out = subprocess.run(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", ctl.HOST, "-H", "Cookie: " + ctl._SESS,
|
|
"-X", "POST", "--data-urlencode", "_csrf=" + ctl._CSRF, "--data-urlencode", "stack_name=" + APP,
|
|
"--data-urlencode", "snapshot_id=" + sid, ctl.BASE + "/backup/restore"],
|
|
capture_output=True, text=True).stdout
|
|
say(f"[{tag}] POST /backup/restore -> {out.splitlines()[0] if out else '?'} "
|
|
f"{[l for l in out.splitlines() if l.lower().startswith('location')][:1]}")
|
|
t0, last = time.time(), None
|
|
while time.time() - t0 < 900:
|
|
code, d = ctl.call("GET", "/api/backup/restore-status")
|
|
dd = (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
cur = json.dumps(dd, ensure_ascii=False)[:600]
|
|
if cur != last:
|
|
say(f"[{tag}] restore-status: {cur}")
|
|
last = cur
|
|
if not dd.get("running") and time.time() - t0 > 8:
|
|
break
|
|
time.sleep(3)
|
|
time.sleep(20)
|
|
st = stack()
|
|
say(f"[{tag}] after: state={st.get('state')} hold={st.get('hold_reason')!r}")
|
|
|
|
|
|
def markers(tag):
|
|
say(f"[{tag}] DB markers: {psql('SELECT string_agg(m, \',\' ORDER BY m) FROM r1_marker;')!r}")
|
|
|
|
|
|
def main():
|
|
say("== 1 deploy")
|
|
if not stack().get("deployed"):
|
|
code, d = ctl.call("GET", f"/api/stacks/{APP}/deploy-fields")
|
|
fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
|
|
vals = {"DOMAIN": DOMAIN, "SUBDOMAIN": "r1poll"}
|
|
for f in fields:
|
|
if f.get("env_var") in vals:
|
|
continue
|
|
if f.get("type") == "password":
|
|
import secrets
|
|
vals[f["env_var"]] = "R1-" + secrets.token_hex(10)
|
|
elif f.get("required") and f.get("default"):
|
|
vals[f["env_var"]] = f["default"]
|
|
code, d = ctl.call("POST", f"/api/stacks/{APP}/deploy", {"values": vals})
|
|
say("deploy ->", code, str(d)[:160])
|
|
for _ in range(90):
|
|
time.sleep(5)
|
|
st = stack()
|
|
if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") and st.get("state") in ("running", "unhealthy", "degraded"):
|
|
break
|
|
say("state:", stack().get("state"))
|
|
time.sleep(30)
|
|
say("== 2 marker A + backup")
|
|
say(psql("CREATE TABLE IF NOT EXISTS r1_marker(m text primary key); INSERT INTO r1_marker VALUES ('A') ON CONFLICT DO NOTHING;"))
|
|
markers("2")
|
|
code, d = ctl.call("POST", "/api/backup/run")
|
|
say("POST /api/backup/run ->", code, str(d)[:200])
|
|
say("backup finished:", json.dumps(wait_backup(), ensure_ascii=False)[:500])
|
|
unit = guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do "
|
|
f"[ -d $d ] && echo UNIT $d && find $d -maxdepth 3 -type f -printf '%TY-%Tm-%Td %TH:%TM %s %p\\n'; done")
|
|
say("unit on disk:\n" + unit)
|
|
say("== 3 POSITIVE CONTROL: marker B (live) then a normal unit restore")
|
|
say(psql("INSERT INTO r1_marker VALUES ('B');"))
|
|
markers("3-before")
|
|
restore("3")
|
|
time.sleep(15)
|
|
markers("3-after")
|
|
say("== 4 marker C (the newest live row), then THE FAULT: the unit's DB volume tar truncated to half")
|
|
say(psql("INSERT INTO r1_marker VALUES ('C');"))
|
|
markers("4-live")
|
|
say(guest(f"for d in /mnt/felhom-drives/*/backups/primary/{APP} /mnt/sys_drive/felhom-data/backups/primary/{APP}; do "
|
|
f"[ -d $d ] || continue; f=$(find $d -name '*.tar' -path '*volume*' | grep -iE 'postgres|db' | head -1); "
|
|
f"[ -n \"$f\" ] || f=$(find $d -name '*.tar' | head -1); s=$(stat -c %s \"$f\"); "
|
|
f"truncate -s $((s/2)) \"$f\"; echo \"TRUNCATED $f $s -> $(stat -c %s \"$f\")\"; done"))
|
|
say("== 5 the unit restore over the damaged copy")
|
|
restore("5")
|
|
time.sleep(30)
|
|
markers("5-after")
|
|
say("containers:\n" + guest("docker ps -a --format '{{.Names}} {{.Status}}' | grep -i rallly"))
|
|
say("db container log tail:\n" + guest("C=$(docker ps -a --format '{{.Names}}' | grep -E '^rallly.*(postgres|db)' | head -1); docker logs --tail 15 $C 2>&1"))
|
|
say("any undo / safety copy of the live state on disk?\n" + guest(
|
|
"find / -xdev \\( -path /proc -o -path /sys \\) -prune -o \\( -iname '*restore-undo*' -o -iname 'pre-restore-*' \\) -print 2>/dev/null | head; "
|
|
"for d in /mnt/felhom-drives/*/backups /mnt/sys_drive/felhom-data/backups; do find $d -maxdepth 4 -iname '*rallly*' -newer /etc/hostname 2>/dev/null | head; done"))
|
|
say("controller log (restore lines):\n" + guest("docker logs --since 30m felhom-controller 2>&1 | grep -iE 'restore|rallly' | grep -vi 'GET /api' | tail -40"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|