felhom.eu/documentation/audits/update-night-2026-09-21/walk.py

#!/usr/bin/env python3
"""walk.py — ONE app's full update walk on guest 9202, through the product's own endpoints.

EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses:
  POST /api/stacks/<n>/deploy · POST /api/backup/run · POST /api/sync · POST /api/stacks/rescan
  POST /api/stacks/<n>/update · POST /api/stacks/<n>/remove
and reads GET /api/stacks/<n>. No controller code exists for it.

The walk, per `09` §6.4 and the update-night brief §4:
  1 deploy from the DRILL catalog at the LIVE pin
  2 seed through the app's OWN front door (R-156: never a volume, never SQL)
  3 read the seed back            <- control C1; a fixture that cannot prove itself proves nothing
  4 „Mentés most"
  5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages
  6 press the guarded Update, record every phase with timestamps
  7 read the seed back through the front door
  8 the four version observables side by side
  9 write the verdict record in `09`'s JSON shape

`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`.
"""
import argparse, json, os, re, subprocess, sys, time
from datetime import datetime, timezone

SC = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad"
EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/update-night-2026-09-21"
DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
BASE = "https://192.168.0.114"
HOSTHDR = "Host: felhom.enkisfelhom.hu"
DOMAIN = "enkisfelhom.hu"
HP = "demo-hp"

LOG = []


def say(*a):
    line = " ".join(str(x) for x in a)
    ts = datetime.now().strftime("%H:%M:%S")
    print(f"{ts} {line}", flush=True)
    LOG.append(f"{ts} {line}")


def sh(args, timeout=300, inp=None):
    try:
        return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp)
    except (subprocess.TimeoutExpired, OSError) as e:
        return subprocess.CompletedProcess(args, 124, "", f"{e}")


def guest(script, timeout=600):
    """Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting)."""
    r = sh(["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new", HP,
            "cat > /tmp/w.sh; pct push 9202 /tmp/w.sh /tmp/w.sh >/dev/null 2>&1; "
            "pct exec 9202 -- bash /tmp/w.sh; rm -f /tmp/w.sh"],
           timeout=timeout, inp=script)
    return r.stdout or ""


def login():
    pw = open(f"{SC}/.ctlpw").read().strip()
    sh(["curl", "-sk", "-D", f"{SC}/hdr.txt", "-o", "/dev/null", "-H", HOSTHDR,
        "-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"])
    h = open(f"{SC}/hdr.txt").read()
    m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I)
    if not m:
        sys.exit("login failed: no session cookie")
    open(f"{SC}/sess.txt", "w").write(m.group(0))
    r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"])
    c = re.search(r'<meta name="csrf-token" content="([^"]+)"', r.stdout or "")
    if not c:
        sys.exit("login failed: no csrf token")
    open(f"{SC}/csrf.txt", "w").write(c.group(1))


def ctl(method, path, data=None, raw=False, tries=2):
    """One controller API call. Re-logs in once on a 302/401 — the controller's session store is
    in memory, so any controller restart during the night invalidates it silently."""
    for attempt in range(tries):
        sess = open(f"{SC}/sess.txt").read().strip()
        csrf = open(f"{SC}/csrf.txt").read().strip()
        args = ["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", "-w", "\n%{http_code}"]
        if method != "GET":
            args += ["-H", f"X-CSRF-Token: {csrf}", "-H", "Content-Type: application/json",
                     "-X", method]
            if data is not None:
                args += ["--data", json.dumps(data)]
        args.append(f"{BASE}{path}")
        r = sh(args)
        body, _, code = (r.stdout or "").rpartition("\n")
        if code.strip() in ("302", "401") and attempt + 1 < tries:
            login()
            continue
        if raw:
            return code.strip(), body
        try:
            return code.strip(), json.loads(body)
        except Exception:
            return code.strip(), {"_raw": body[:600]}
    return code.strip(), {"_raw": body[:600]}


def page(path):
    sess = open(f"{SC}/sess.txt").read().strip()
    r = sh(["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", f"{BASE}{path}"])
    return r.stdout or ""


def app_curl(sub, path, *extra, method=None, data=None, timeout=45):
    """A call to the APP's own front door on 9202 — the household's route, not ours."""
    args = ["curl", "-sSk", "--max-time", str(timeout), "-H", f"Host: {sub}.{DOMAIN}",
            "-w", "\n%{http_code}"]
    if method:
        args += ["-X", method]
    if data is not None:
        args += ["--data-binary", "@-"]
    args += list(extra) + [f"{BASE}{path}"]
    r = sh(args, timeout=timeout + 30, inp=data)
    body, _, code = (r.stdout or "").rpartition("\n")
    return r.returncode, code.strip(), body


def stack(name):
    _, d = ctl("GET", f"/api/stacks/{name}")
    return (d.get("data") or {}) if isinstance(d, dict) else {}


def wait_app(sub, path="/", want=("200", "302", "303", "401", "403"), tries=60, delay=5):
    """Settling says the container runs; this says the APP answers. Not the same thing."""
    last = None
    for _ in range(tries):
        rc, code, _ = app_curl(sub, path, timeout=15)
        last = (rc, code)
        if rc == 0 and code in want:
            return True
        time.sleep(delay)
    say(f"    app never answered on {sub}{path} (last rc={last[0]} code={last[1]})")
    return False


# ------------------------------------------------------------------ the walk


DRIVE = "/mnt/felhom-drives/scratch_hdd/userdata"

# What THIS run generated for a deploy, per app. Deploy secrets are ENCRYPTED AT REST in
# `app.yaml` (`ENC:…`), which is right and which means a fixture cannot read an app's admin
# password back off the box — the household sees it once. So the value the harness itself
# generated is kept here for the life of the run, and nowhere else.
GENERATED = {}


def deploy_values(name, sub):
    """Fill EVERY required deploy field the way the wizard would, by asking the box what this app
    asks for — `GET /api/stacks/<n>/deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN.

    Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because
    a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password
    (grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only
    reason this was safe to discover by running it (live-probes rule).

    A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on
    the scratch drive first — the same act the drive browser performs for a household.
    """
    code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields")
    fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
    values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub}
    made = []
    for f in fields:
        ev, ty = f.get("env_var"), f.get("type")
        if ev in values:
            continue
        # `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses
        # when the caller sends none, deliberately ("the user needs to know their password"),
        # while `.felhom.yml` declares `required: false` and the API serves that verbatim. A
        # caller that trusts the contract gets a 400. Measured tonight on grafana; filed.
        if not f.get("required") and ty != "password":
            continue                      # the controller generates the optional secrets itself
        if ty == "path":
            p = f"{DRIVE}/{name}"
            values[ev] = p
            made.append(p)
        elif ty in ("secret", "password"):
            import secrets as _s
            values[ev] = "Drill-" + _s.token_hex(12)
            GENERATED.setdefault(name, {})[ev] = values[ev]
        elif f.get("default"):
            values[ev] = f["default"]
        else:
            values[ev] = f"drill-{name}"
    if made:
        guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made))
        say(f"  [1] made the drive paths this app requires: {made}")
    extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")]
    if extra:
        say(f"  [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}")
    return values


def deploy(name, sub, extra_values=None):
    st = stack(name)
    if st.get("deployed"):
        say(f"  [1] {name} already deployed — reusing")
        return True
    values = deploy_values(name, sub)
    if extra_values:
        values.update(extra_values)
    code, d = ctl("POST", f"/api/stacks/{name}/deploy", {"values": values})
    say(f"  [1] deploy -> {code} {str(d)[:120]}")
    if code != "202":
        return False
    # WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the
    # container `healthy` while the controller's own state read `unhealthy` — a gate on `running`
    # alone therefore times out on an app that is up. The state is RECORDED rather than required;
    # the real gate is the fixture's own `wait_app`, which asks whether the APP answers.
    seen = None
    for _ in range(90):
        time.sleep(5)
        st = stack(name)
        seen = st.get("state")
        # `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21:
        # tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm
        # read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the
        # deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark
        # (`runComposeDeploy` writes it), so that is what to wait for.
        pins = (st.get("app_config") or {}).get("pinned_images")
        if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"):
            say(f"  [1] deployed, controller state={seen}, "
                f"pinned={(st.get('app_config') or {}).get('pinned_images')}")
            if seen != "running":
                say(f"  [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, "
                    f"not treated as a failure; the fixture's front-door wait is the real gate")
            return True
    say(f"  [1] never became deployed (last controller state={seen!r})")
    return False


def backup_now(name):
    code, d = ctl("POST", "/api/backup/run")
    say(f"  [4] „Mentés most\" -> {code} {str(d)[:160]}")
    for _ in range(90):
        time.sleep(5)
        c2, s = ctl("GET", "/api/backup/status")
        dd = s.get("data") or {}
        if not dd.get("running", False):
            say(f"  [4] backup idle; last={dd.get('last_run') or dd.get('last_db_dump')}")
            return True
    say("  [4] backup still running after 7.5 min — carrying on")
    return False


def drill_bump(app, frm, to, service_hint=None):
    """Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates).

    `frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in
    two images (adventurelog's backend and frontend) moves both in one edge, while its engine
    sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move
    every service that carries the app's own version and no others.
    """
    comp = f"{DRILL}/templates/{app}/docker-compose.yml"
    fy = f"{DRILL}/templates/{app}/.felhom.yml"
    s = open(comp).read()
    froms = [x.strip() for x in frm.split(",") if x.strip()]
    tos = [x.strip() for x in to.split(",") if x.strip()]
    if len(froms) != len(tos):
        say(f"  [5] from/to lists differ in length: {froms} vs {tos}")
        return None
    for f1, t1 in zip(froms, tos):
        if f"image: {f1}" not in s:
            say(f"  [5] FROM ref not found in compose: {f1}")
            return None
        s = s.replace(f"image: {f1}", f"image: {t1}")
    open(comp, "w").write(s)
    f = open(fy).read()
    today = datetime.now().strftime("%Y-%m-%d")
    f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M)
    open(fy, "w").write(f)
    sh(["git", "-C", DRILL, "add", "-A"])
    sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"])
    r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120)
    h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip()
    say(f"  [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})")
    return h


def sync_rescan():
    ctl("POST", "/api/sync")
    time.sleep(2)
    ctl("POST", "/api/stacks/rescan")   # R-607: ALWAYS before reading a badge
    time.sleep(2)


def badges(name):
    out = {}
    for lang, suffix in (("hu", ""), ("en", "?lang=en")):
        h = page(f"/apps/{name}{suffix}")
        m = re.findall(r'<span class="tag tag-[^"]*"[^>]*title="([^"]*)"[^>]*>([^<]*)<', h)
        out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3]
    return out


def press_update(name, poll=1.0, cap_s=1800):
    code, d = ctl("POST", f"/api/stacks/{name}/update")
    say(f"  [6] Update -> {code} {str(d)[:220]}")
    if code not in ("202", "200"):
        return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0}
    phases, seen, t0 = [], None, time.time()
    while time.time() - t0 < cap_s:
        st = stack(name)
        ph = st.get("update_phase")
        if ph != seen:
            seen = ph
            rec = {"t": round(time.time() - t0, 1), "phase": ph,
                   "label": st.get("update_phase_label"), "updating": st.get("updating"),
                   "error": st.get("update_error"), "hold": st.get("hold_reason")}
            phases.append(rec)
            say(f"      +{rec['t']:>6.1f}s  phase={ph}  label={rec['label']}  "
                f"err={rec['error']}  hold={rec['hold']}")
        if not st.get("updating") and ph in ("done", "failed", None) and time.time() - t0 > 3:
            break
        time.sleep(poll)
    st = stack(name)
    return {"accepted": True, "http": code, "phases": phases,
            "duration_s": round(time.time() - t0, 1),
            "final_phase": st.get("update_phase"), "update_error": st.get("update_error"),
            "hold_reason": st.get("hold_reason"), "state": st.get("state")}


def observables(name):
    st = stack(name)
    ac = st.get("app_config") or {}
    live = guest(f"""
grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//'
echo '---inspect---'
for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do
  echo -n "$c  "; docker inspect "$c" --format '{{{{.Config.Image}}}}  running={{{{.State.Running}}}}  restarts={{{{.RestartCount}}}}'
done
""")
    a, _, b = live.partition("---inspect---")
    return {
        "pinned_images": ac.get("pinned_images"),
        "installed_images": {k: (v.get("ref") if isinstance(v, dict) else v)
                             for k, v in (ac.get("installed_images") or {}).items()},
        "catalog_images": st.get("catalog_images"),
        "live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()],
        "docker_inspect": [x for x in b.strip().splitlines() if x.strip()],
    }


def app_logs(name, lines=400):
    """The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is
    one string with escaped newlines — a scan over the envelope sees a single enormous line and
    finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96
    rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines."""
    code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}")
    if isinstance(d, dict):
        data = d.get("data")
        if isinstance(data, dict) and isinstance(data.get("logs"), str):
            return data["logs"]
        if isinstance(d.get("_raw"), str):
            return d["_raw"]
    return str(d)


def write_verdict(rec, appdir):
    os.makedirs(appdir, exist_ok=True)
    p = os.path.join(appdir, "verdict.json")
    json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False)
    say(f"  [9] verdict {rec['verdict']}  -> {p}")


def remove(name):
    """Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint
    refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up."""
    c1, d1 = ctl("POST", f"/api/stacks/{name}/stop")
    say(f"  [X] stop -> {c1} {str(d1)[:100]}")
    for _ in range(24):
        time.sleep(5)
        if stack(name).get("state") != "running":
            break
    code, d = ctl("POST", f"/api/stacks/{name}/remove",
                  {"remove_hdd_data": True, "remove_backups": True})
    say(f"  [X] remove (with drive data) -> {code} {str(d)[:160]}")
    if code == "409":
        # R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive
        # path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202
        # `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits
        # this. The household's other choice — remove the app, KEEP the data — is accepted, and the
        # harness takes it, then tidies its own directory by name at teardown.
        say("  [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)"
            " — removing the app and KEEPING the drive data instead")
        code, d = ctl("POST", f"/api/stacks/{name}/remove",
                      {"remove_hdd_data": False, "remove_backups": True})
        say(f"  [X] remove (keeping drive data) -> {code} {str(d)[:160]}")
    time.sleep(5)
    st = stack(name)
    left = guest(f"ls -d /opt/docker/stacks/{name} 2>/dev/null; "
                 f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'")
    say(f"  [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}")
    return code


def app_env(name, key):
    """Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the
    app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller
    shows them the same value. Data still goes in through the app's own front door."""
    out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1")
    if ":" in out:
        return out.split(":", 1)[1].strip().strip('"').strip("'")
    return ""


def snapshots(name):
    """The restorable copies the backups page offers for this app."""
    code, d = ctl("GET", f"/api/backup/snapshots?stack={name}")
    data = d.get("data") if isinstance(d, dict) else None
    if isinstance(data, dict):
        for k in ("snapshots", "items", "restore_points"):
            if isinstance(data.get(k), list):
                return data[k]
    return data if isinstance(data, list) else []


def restore(name, snapshot_id=None, wait_s=1200):
    """The household's own way out: the „Visszaállítás a mentésből" button on the backups page.

    A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`,
    `snapshot_id` — because that is the button the sentence tells them to press.
    """
    snaps = snapshots(name)
    if snapshot_id is None:
        if not snaps:
            say(f"  [R] no restorable copy offered for {name}")
            return {"ok": False, "why": "no snapshot offered", "snapshots": snaps}
        first = snaps[0]
        snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id")
    say(f"  [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)")
    sess = open(f"{SC}/sess.txt").read().strip()
    csrf = open(f"{SC}/csrf.txt").read().strip()
    r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}",
            "-X", "POST",
            "--data-urlencode", f"_csrf={csrf}",
            "--data-urlencode", f"stack_name={name}",
            "--data-urlencode", f"snapshot_id={snapshot_id}",
            f"{BASE}/backup/restore"], timeout=180)
    head = (r.stdout or "").split("\n")[0].strip()
    loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")]
    say(f"  [R] POST /backup/restore -> {head} {loc[:1]}")
    t0 = time.time()
    last = None
    while time.time() - t0 < wait_s:
        code, d = ctl("GET", "/api/backup/restore-status")
        dd = d.get("data") or {}
        cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message"))
        if cur != last:
            say(f"      +{round(time.time()-t0,1):>6.1f}s restore {cur}")
            last = cur
        if not dd.get("running", False) and time.time() - t0 > 5:
            break
        time.sleep(2)
    st = stack(name)
    say(f"  [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} "
        f"phase={st.get('update_phase')}")
    return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps,
            "http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1),
            "state_after": st.get("state"), "hold_after": st.get("hold_reason"),
            "observables_after": observables(name)}