felhom.eu/documentation/tests/campaign10-evidence-2026-07-31/runner/c10lib.py

#!/usr/bin/env python3
"""Campaign 10 — shared driver library. Runs on DooPlex, NOT inside the VM:
the soak's atoms kill the controller, the agent and the VM itself, so anything
running inside would die with the thing it is testing.

Canary model (brief A4): every app carries a marker naming the backup GENERATION
its data came from. Without it, a restore that returns stale or empty data is
indistinguishable from a good one — this arc produced six checks that passed for
the wrong reason.

  rallly  -> a row in a cc_proof TABLE in postgres, read over the app's real
             network path (docker run --network container:rallly ... psql -h
             rallly-postgres). NOT 127.0.0.1 inside the postgres container,
             which postgres trusts and which produced D5's false pass.
  others  -> a marker file inside the app's own docker volume (what gets dumped).
"""
import json, os, subprocess, time

HOME = os.path.expanduser("~/.config/campaign10")
VM_IP = "192.168.0.105"
JUMP = "demo-hp"
VMID = "311"
GUEST = "9201"

HOSTPW = json.load(open(os.path.join(HOME, "host-recovery.json")))["password"]
SECRETS = json.load(open(os.path.join(HOME, "app-secrets.json")))

# app -> (container, volume mount dir) for the file-marker apps
FILE_APPS = {
    "grafana": ("grafana", "/var/lib/grafana"),
    "homebox": ("homebox", "/data"),
    "papra":   ("papra", "/app/data"),
}
ALL_APPS = ["rallly", "homebox", "grafana", "papra"]


def _run(args, timeout=300):
    env = dict(os.environ, SSHPASS=HOSTPW, LC_ALL="C")
    try:
        p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, env=env)
        return p.returncode, p.stdout, p.stderr
    except subprocess.TimeoutExpired:
        return 124, "", "TIMEOUT"


def box(cmd, timeout=300):
    """Run a command on the campaign PVE host (the VM), via the demo-hp jump."""
    return _run(["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no",
                 "-o", "ConnectTimeout=20", "-J", JUMP, f"root@{VM_IP}",
                 "LC_ALL=C " + cmd], timeout)


def guest(cmd, timeout=300):
    """Run a command inside guest 9201."""
    return box(f"pct exec {GUEST} -- bash -c {shq(cmd)}", timeout)


def hp(cmd, timeout=180):
    """Run a command on demo-hp itself (for qm: reset, stop, disk detach...)."""
    return _run(["ssh", "-o", "ConnectTimeout=20", JUMP, "LC_ALL=C " + cmd], timeout)


def shq(s):
    return "'" + s.replace("'", "'\\''") + "'"


def api(method, path, body=None, timeout=300):
    """Authenticated controller API call through the on-guest helper."""
    if body is None:
        rc, out, _ = guest(f"/root/c10api.sh {method} {path} 2>/dev/null", timeout)
        return out
    import base64
    b64 = base64.b64encode(json.dumps(body).encode()).decode()
    rc, out, _ = guest(
        f"echo {b64} | base64 -d > /tmp/.b && /root/c10api.sh {method} {path} "
        f'"$(cat /tmp/.b)" --json; shred -u /tmp/.b 2>/dev/null', timeout)
    return out


def jparse(raw):
    try:
        return json.loads(raw[raw.index("{"):raw.rindex("}") + 1])
    except Exception:
        return None


# ---------------------------------------------------------------- canaries

def _psql(sql, timeout=180):
    """Read/write rallly's DB over the path DATABASE_URL actually names."""
    pw = SECRETS["rallly"]["values"]["DB_PASSWORD"]
    url = f"postgresql://rallly:{pw}@rallly-postgres:5432/rallly"
    cmd = ("docker run --rm --network container:rallly postgres:16-alpine "
           f"psql {shq(url)} -tAc {shq(sql)}")
    rc, out, err = guest(cmd, timeout)
    return rc, out.strip(), err.strip()


def seed(gen, verify=True):
    """Write the generation marker into every app. Returns {app: ok}.

    With verify=True the value is READ BACK — a write that reported success but did not
    land is the failure mode that turned a dead app into a fake 'stale restore' in run 2a."""
    res = {}
    _psql("CREATE TABLE IF NOT EXISTS cc_proof (id serial primary key, gen text, at timestamptz default now())")
    rc, out, err = _psql(f"INSERT INTO cc_proof (gen) VALUES ('{gen}') RETURNING gen")
    res["rallly"] = (gen in out)
    for app, (cont, d) in FILE_APPS.items():
        # -u 0: papra's volume is root-owned while the container runs as nonroot (R-156),
        # so the app itself cannot write there. Seeding as root keeps the canary in the
        # volume the BACKUP dumps, which is what I7 needs to observe.
        rc, out, err = guest(
            f"docker exec -u 0 {cont} sh -c {shq(f'echo {gen} > {d}/cc_proof.txt && cat {d}/cc_proof.txt')}", 180)
        res[app] = (gen in out)
    if verify:
        back = read_canaries()
        for a in list(res):
            res[a] = res[a] and (back.get(a) == gen)
    return res


def read_canaries():
    """Read each app's marker back. Returns {app: value-or-None}."""
    res = {}
    rc, out, err = _psql("SELECT gen FROM cc_proof ORDER BY id DESC LIMIT 1")
    res["rallly"] = out.strip() or None
    for app, (cont, d) in FILE_APPS.items():
        rc, out, err = guest(f"docker exec {cont} sh -c {shq(f'cat {d}/cc_proof.txt 2>/dev/null')}", 180)
        v = out.strip()
        res[app] = v or None
    return res


def apps_ready(names=None, timeout=600):
    """Wait until the named app containers are RUNNING (healthy where they report health).

    Load-bearing: a canary lives inside an app container, so reading one before the app is
    back does not measure the product — it measures the harness. Two false violations in
    run 2a came from exactly that (a seed that never landed, then a 'stale' restore)."""
    names = names or (ALL_APPS + [DRIVE_APP])
    t0 = time.time()
    while time.time() - t0 < timeout:
        c = containers()
        ok = True
        for n in names:
            probe = "rallly-postgres" if n == "rallly" else n
            st = c.get(n) or ""
            st2 = c.get(probe) or ""
            if not st or "Up" not in st or "starting" in st:
                ok = False
                break
            if n == "rallly" and ("Up" not in st2 or "starting" in st2):
                ok = False
                break
        if ok:
            return True
        time.sleep(10)
    return False


DRIVE_APP = "calibre-web"


def containers():
    rc, out, _ = guest("docker ps --format '{{.Names}}|{{.Status}}'", 120)
    d = {}
    for line in out.strip().splitlines():
        if "|" in line:
            n, s = line.split("|", 1)
            d[n.strip()] = s.strip()
    return d


def guest_init_pid():
    """I4: a stale-device bind must repair WITHOUT the guest restarting."""
    rc, out, _ = box(f"pct exec {GUEST} -- cat /proc/1/stat 2>/dev/null", 120)
    rc2, boot, _ = box(f"pct exec {GUEST} -- stat -c %Y /proc/1 2>/dev/null", 120)
    return boot.strip()