#!/usr/bin/env python3 """Campaign 10 — shared driver library. Runs on DooPlex, NOT inside the VM: the soak's atoms kill the controller, the agent and the VM itself, so anything running inside would die with the thing it is testing. Canary model (brief A4): every app carries a marker naming the backup GENERATION its data came from. Without it, a restore that returns stale or empty data is indistinguishable from a good one — this arc produced six checks that passed for the wrong reason. rallly -> a row in a cc_proof TABLE in postgres, read over the app's real network path (docker run --network container:rallly ... psql -h rallly-postgres). NOT 127.0.0.1 inside the postgres container, which postgres trusts and which produced D5's false pass. others -> a marker file inside the app's own docker volume (what gets dumped). """ import json, os, subprocess, time HOME = os.path.expanduser("~/.config/campaign10") VM_IP = "192.168.0.105" JUMP = "demo-hp" VMID = "311" GUEST = "9201" HOSTPW = json.load(open(os.path.join(HOME, "host-recovery.json")))["password"] SECRETS = json.load(open(os.path.join(HOME, "app-secrets.json"))) # app -> (container, volume mount dir) for the file-marker apps FILE_APPS = { "grafana": ("grafana", "/var/lib/grafana"), "homebox": ("homebox", "/data"), "papra": ("papra", "/app/data"), } ALL_APPS = ["rallly", "homebox", "grafana", "papra"] def _run(args, timeout=300): env = dict(os.environ, SSHPASS=HOSTPW, LC_ALL="C") try: p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, env=env) return p.returncode, p.stdout, p.stderr except subprocess.TimeoutExpired: return 124, "", "TIMEOUT" def box(cmd, timeout=300): """Run a command on the campaign PVE host (the VM), via the demo-hp jump.""" return _run(["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=20", "-J", JUMP, f"root@{VM_IP}", "LC_ALL=C " + cmd], timeout) def guest(cmd, timeout=300): """Run a command inside guest 9201.""" return box(f"pct exec {GUEST} -- bash -c {shq(cmd)}", timeout) def hp(cmd, timeout=180): """Run a command on demo-hp itself (for qm: reset, stop, disk detach...).""" return _run(["ssh", "-o", "ConnectTimeout=20", JUMP, "LC_ALL=C " + cmd], timeout) def shq(s): return "'" + s.replace("'", "'\\''") + "'" def api(method, path, body=None, timeout=300): """Authenticated controller API call through the on-guest helper.""" if body is None: rc, out, _ = guest(f"/root/c10api.sh {method} {path} 2>/dev/null", timeout) return out import base64 b64 = base64.b64encode(json.dumps(body).encode()).decode() rc, out, _ = guest( f"echo {b64} | base64 -d > /tmp/.b && /root/c10api.sh {method} {path} " f'"$(cat /tmp/.b)" --json; shred -u /tmp/.b 2>/dev/null', timeout) return out def jparse(raw): try: return json.loads(raw[raw.index("{"):raw.rindex("}") + 1]) except Exception: return None # ---------------------------------------------------------------- canaries def _psql(sql, timeout=180): """Read/write rallly's DB over the path DATABASE_URL actually names.""" pw = SECRETS["rallly"]["values"]["DB_PASSWORD"] url = f"postgresql://rallly:{pw}@rallly-postgres:5432/rallly" cmd = ("docker run --rm --network container:rallly postgres:16-alpine " f"psql {shq(url)} -tAc {shq(sql)}") rc, out, err = guest(cmd, timeout) return rc, out.strip(), err.strip() def seed(gen, verify=True): """Write the generation marker into every app. Returns {app: ok}. With verify=True the value is READ BACK — a write that reported success but did not land is the failure mode that turned a dead app into a fake 'stale restore' in run 2a.""" res = {} _psql("CREATE TABLE IF NOT EXISTS cc_proof (id serial primary key, gen text, at timestamptz default now())") rc, out, err = _psql(f"INSERT INTO cc_proof (gen) VALUES ('{gen}') RETURNING gen") res["rallly"] = (gen in out) for app, (cont, d) in FILE_APPS.items(): # -u 0: papra's volume is root-owned while the container runs as nonroot (R-156), # so the app itself cannot write there. Seeding as root keeps the canary in the # volume the BACKUP dumps, which is what I7 needs to observe. rc, out, err = guest( f"docker exec -u 0 {cont} sh -c {shq(f'echo {gen} > {d}/cc_proof.txt && cat {d}/cc_proof.txt')}", 180) res[app] = (gen in out) if verify: back = read_canaries() for a in list(res): res[a] = res[a] and (back.get(a) == gen) return res def read_canaries(): """Read each app's marker back. Returns {app: value-or-None}.""" res = {} rc, out, err = _psql("SELECT gen FROM cc_proof ORDER BY id DESC LIMIT 1") res["rallly"] = out.strip() or None for app, (cont, d) in FILE_APPS.items(): rc, out, err = guest(f"docker exec {cont} sh -c {shq(f'cat {d}/cc_proof.txt 2>/dev/null')}", 180) v = out.strip() res[app] = v or None return res def apps_ready(names=None, timeout=600): """Wait until the named app containers are RUNNING (healthy where they report health). Load-bearing: a canary lives inside an app container, so reading one before the app is back does not measure the product — it measures the harness. Two false violations in run 2a came from exactly that (a seed that never landed, then a 'stale' restore).""" names = names or (ALL_APPS + [DRIVE_APP]) t0 = time.time() while time.time() - t0 < timeout: c = containers() ok = True for n in names: probe = "rallly-postgres" if n == "rallly" else n st = c.get(n) or "" st2 = c.get(probe) or "" if not st or "Up" not in st or "starting" in st: ok = False break if n == "rallly" and ("Up" not in st2 or "starting" in st2): ok = False break if ok: return True time.sleep(10) return False DRIVE_APP = "calibre-web" def containers(): rc, out, _ = guest("docker ps --format '{{.Names}}|{{.Status}}'", 120) d = {} for line in out.strip().splitlines(): if "|" in line: n, s = line.split("|", 1) d[n.strip()] = s.strip() return d def guest_init_pid(): """I4: a stale-device bind must repair WITHOUT the guest restarting.""" rc, out, _ = box(f"pct exec {GUEST} -- cat /proc/1/stat 2>/dev/null", 120) rc2, boot, _ = box(f"pct exec {GUEST} -- stat -c %Y /proc/1 2>/dev/null", 120) return boot.strip()