#!/usr/bin/env python3
"""walk.py — ONE app's full update walk on guest 9202, through the product's own endpoints.

EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses:
  POST /api/stacks/<n>/deploy · POST /api/backup/run · POST /api/sync · POST /api/stacks/rescan
  POST /api/stacks/<n>/update · POST /api/stacks/<n>/remove
and reads GET /api/stacks/<n>. No controller code exists for it.

The walk, per `09` §6.4 and the update-night brief §4:
  1 deploy from the DRILL catalog at the LIVE pin
  2 seed through the app's OWN front door (R-156: never a volume, never SQL)
  3 read the seed back            <- control C1; a fixture that cannot prove itself proves nothing
  4 „Mentés most"
  5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages
  6 press the guarded Update, record every phase with timestamps
  7 read the seed back through the front door
  8 the four version observables side by side
  9 write the verdict record in `09`'s JSON shape

`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`.
"""
import argparse, json, os, re, subprocess, sys, time
from datetime import datetime, timezone

SC = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/d029e2e6-1762-440e-956d-0760c8aea4b3/scratchpad"
EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/update-night-2026-09-21"
DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
BASE = "https://192.168.0.114"
HOSTHDR = "Host: felhom.enkisfelhom.hu"
DOMAIN = "enkisfelhom.hu"
HP = "demo-hp"

LOG = []


def say(*a):
    line = " ".join(str(x) for x in a)
    ts = datetime.now().strftime("%H:%M:%S")
    print(f"{ts} {line}", flush=True)
    LOG.append(f"{ts} {line}")


def sh(args, timeout=300, inp=None):
    try:
        return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp)
    except (subprocess.TimeoutExpired, OSError) as e:
        return subprocess.CompletedProcess(args, 124, "", f"{e}")


def guest(script, timeout=600):
    """Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting)."""
    r = sh(["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new", HP,
            "cat > /tmp/w.sh; pct push 9202 /tmp/w.sh /tmp/w.sh >/dev/null 2>&1; "
            "pct exec 9202 -- bash /tmp/w.sh; rm -f /tmp/w.sh"],
           timeout=timeout, inp=script)
    return r.stdout or ""


def login():
    pw = open(f"{SC}/.ctlpw").read().strip()
    sh(["curl", "-sk", "-D", f"{SC}/hdr.txt", "-o", "/dev/null", "-H", HOSTHDR,
        "-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"])
    h = open(f"{SC}/hdr.txt").read()
    m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I)
    if not m:
        sys.exit("login failed: no session cookie")
    open(f"{SC}/sess.txt", "w").write(m.group(0))
    r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"])
    c = re.search(r'<meta name="csrf-token" content="([^"]+)"', r.stdout or "")
    if not c:
        sys.exit("login failed: no csrf token")
    open(f"{SC}/csrf.txt", "w").write(c.group(1))


def ctl(method, path, data=None, raw=False, tries=2):
    """One controller API call. Re-logs in once on a 302/401 — the controller's session store is
    in memory, so any controller restart during the night invalidates it silently."""
    for attempt in range(tries):
        sess = open(f"{SC}/sess.txt").read().strip()
        csrf = open(f"{SC}/csrf.txt").read().strip()
        args = ["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", "-w", "\n%{http_code}"]
        if method != "GET":
            args += ["-H", f"X-CSRF-Token: {csrf}", "-H", "Content-Type: application/json",
                     "-X", method]
            if data is not None:
                args += ["--data", json.dumps(data)]
        args.append(f"{BASE}{path}")
        r = sh(args)
        body, _, code = (r.stdout or "").rpartition("\n")
        if code.strip() in ("302", "401") and attempt + 1 < tries:
            login()
            continue
        if raw:
            return code.strip(), body
        try:
            return code.strip(), json.loads(body)
        except Exception:
            return code.strip(), {"_raw": body[:600]}
    return code.strip(), {"_raw": body[:600]}


def page(path):
    sess = open(f"{SC}/sess.txt").read().strip()
    r = sh(["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", f"{BASE}{path}"])
    return r.stdout or ""


def app_curl(sub, path, *extra, method=None, data=None, timeout=45):
    """A call to the APP's own front door on 9202 — the household's route, not ours."""
    args = ["curl", "-sSk", "--max-time", str(timeout), "-H", f"Host: {sub}.{DOMAIN}",
            "-w", "\n%{http_code}"]
    if method:
        args += ["-X", method]
    if data is not None:
        args += ["--data-binary", "@-"]
    args += list(extra) + [f"{BASE}{path}"]
    r = sh(args, timeout=timeout + 30, inp=data)
    body, _, code = (r.stdout or "").rpartition("\n")
    return r.returncode, code.strip(), body


def stack(name):
    _, d = ctl("GET", f"/api/stacks/{name}")
    return (d.get("data") or {}) if isinstance(d, dict) else {}


def wait_app(sub, path="/", want=("200", "302", "303", "401", "403"), tries=60, delay=5):
    """Settling says the container runs; this says the APP answers. Not the same thing."""
    last = None
    for _ in range(tries):
        rc, code, _ = app_curl(sub, path, timeout=15)
        last = (rc, code)
        if rc == 0 and code in want:
            return True
        time.sleep(delay)
    say(f"    app never answered on {sub}{path} (last rc={last[0]} code={last[1]})")
    return False


# ------------------------------------------------------------------ the walk


DRIVE = "/mnt/felhom-drives/scratch_hdd/userdata"

# What THIS run generated for a deploy, per app. Deploy secrets are ENCRYPTED AT REST in
# `app.yaml` (`ENC:…`), which is right and which means a fixture cannot read an app's admin
# password back off the box — the household sees it once. So the value the harness itself
# generated is kept here for the life of the run, and nowhere else.
GENERATED = {}


def deploy_values(name, sub):
    """Fill EVERY required deploy field the way the wizard would, by asking the box what this app
    asks for — `GET /api/stacks/<n>/deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN.

    Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because
    a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password
    (grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only
    reason this was safe to discover by running it (live-probes rule).

    A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on
    the scratch drive first — the same act the drive browser performs for a household.
    """
    code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields")
    fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
    values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub}
    made = []
    for f in fields:
        ev, ty = f.get("env_var"), f.get("type")
        if ev in values:
            continue
        # `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses
        # when the caller sends none, deliberately ("the user needs to know their password"),
        # while `.felhom.yml` declares `required: false` and the API serves that verbatim. A
        # caller that trusts the contract gets a 400. Measured tonight on grafana; filed.
        if not f.get("required") and ty != "password":
            continue                      # the controller generates the optional secrets itself
        if ty == "path":
            p = f"{DRIVE}/{name}"
            values[ev] = p
            made.append(p)
        elif ty in ("secret", "password"):
            import secrets as _s
            values[ev] = "Drill-" + _s.token_hex(12)
            GENERATED.setdefault(name, {})[ev] = values[ev]
        elif f.get("default"):
            values[ev] = f["default"]
        else:
            values[ev] = f"drill-{name}"
    if made:
        guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made))
        say(f"  [1] made the drive paths this app requires: {made}")
    extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")]
    if extra:
        say(f"  [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}")
    return values


def deploy(name, sub, extra_values=None):
    st = stack(name)
    if st.get("deployed"):
        say(f"  [1] {name} already deployed — reusing")
        return True
    values = deploy_values(name, sub)
    if extra_values:
        values.update(extra_values)
    code, d = ctl("POST", f"/api/stacks/{name}/deploy", {"values": values})
    say(f"  [1] deploy -> {code} {str(d)[:120]}")
    if code != "202":
        return False
    # WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the
    # container `healthy` while the controller's own state read `unhealthy` — a gate on `running`
    # alone therefore times out on an app that is up. The state is RECORDED rather than required;
    # the real gate is the fixture's own `wait_app`, which asks whether the APP answers.
    seen = None
    for _ in range(90):
        time.sleep(5)
        st = stack(name)
        seen = st.get("state")
        # `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21:
        # tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm
        # read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the
        # deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark
        # (`runComposeDeploy` writes it), so that is what to wait for.
        pins = (st.get("app_config") or {}).get("pinned_images")
        if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"):
            say(f"  [1] deployed, controller state={seen}, "
                f"pinned={(st.get('app_config') or {}).get('pinned_images')}")
            if seen != "running":
                say(f"  [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, "
                    f"not treated as a failure; the fixture's front-door wait is the real gate")
            return True
    say(f"  [1] never became deployed (last controller state={seen!r})")
    return False


def backup_now(name):
    code, d = ctl("POST", "/api/backup/run")
    say(f"  [4] „Mentés most\" -> {code} {str(d)[:160]}")
    for _ in range(90):
        time.sleep(5)
        c2, s = ctl("GET", "/api/backup/status")
        dd = s.get("data") or {}
        if not dd.get("running", False):
            say(f"  [4] backup idle; last={dd.get('last_run') or dd.get('last_db_dump')}")
            return True
    say("  [4] backup still running after 7.5 min — carrying on")
    return False


def drill_bump(app, frm, to, service_hint=None):
    """Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates).

    `frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in
    two images (adventurelog's backend and frontend) moves both in one edge, while its engine
    sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move
    every service that carries the app's own version and no others.
    """
    comp = f"{DRILL}/templates/{app}/docker-compose.yml"
    fy = f"{DRILL}/templates/{app}/.felhom.yml"
    s = open(comp).read()
    froms = [x.strip() for x in frm.split(",") if x.strip()]
    tos = [x.strip() for x in to.split(",") if x.strip()]
    if len(froms) != len(tos):
        say(f"  [5] from/to lists differ in length: {froms} vs {tos}")
        return None
    for f1, t1 in zip(froms, tos):
        if f"image: {f1}" not in s:
            say(f"  [5] FROM ref not found in compose: {f1}")
            return None
        s = s.replace(f"image: {f1}", f"image: {t1}")
    open(comp, "w").write(s)
    f = open(fy).read()
    today = datetime.now().strftime("%Y-%m-%d")
    f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M)
    open(fy, "w").write(f)
    sh(["git", "-C", DRILL, "add", "-A"])
    sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"])
    r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120)
    h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip()
    say(f"  [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})")
    return h


def sync_rescan(expect_app=None, expect_ref=None, tries=12, delay=5):
    """Sync, rescan, and — when told what to expect — WAIT FOR THE BADGE TO CATCH UP.

    R-607: `POST /api/sync` answers "nincs valtozas" while the catalog HAS moved, and
    `catalog_images` stays stale until a separate rescan. Tonight showed the rescan alone is not
    enough either: mealie's badge read "Naprakesz" seconds after its bump was pushed, and the
    Update that followed moved nothing and still reported "Frissitve". So when the caller knows
    which reference should appear, this polls for it and SAYS HOW LONG IT TOOK — which is the
    NUMBER R-607 asks for and has never had.
    """
    t0 = time.time()
    ctl("POST", "/api/sync")
    time.sleep(2)
    ctl("POST", "/api/stacks/rescan")
    time.sleep(2)
    if not expect_app or not expect_ref:
        return None
    for i in range(tries):
        cat = stack(expect_app).get("catalog_images") or {}
        if expect_ref in cat.values():
            waited = round(time.time() - t0, 1)
            if i:
                say(f"  [sync] the badge needed {waited}s and {i+1} sync+rescan rounds to catch up "
                    f"to {expect_ref} — R-607's window, measured")
            return waited
        time.sleep(delay)
        ctl("POST", "/api/sync")
        time.sleep(1)
        ctl("POST", "/api/stacks/rescan")
    say(f"  [sync] the badge NEVER caught up to {expect_ref} in {round(time.time()-t0,1)}s — "
        f"catalog_images = {stack(expect_app).get('catalog_images')}")
    return None


def badges(name):
    out = {}
    for lang, suffix in (("hu", ""), ("en", "?lang=en")):
        h = page(f"/apps/{name}{suffix}")
        m = re.findall(r'<span class="tag tag-[^"]*"[^>]*title="([^"]*)"[^>]*>([^<]*)<', h)
        out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3]
    return out


def press_update(name, poll=1.0, cap_s=1800):
    code, d = ctl("POST", f"/api/stacks/{name}/update")
    say(f"  [6] Update -> {code} {str(d)[:220]}")
    if code not in ("202", "200"):
        return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0}
    phases, seen, t0 = [], None, time.time()
    while time.time() - t0 < cap_s:
        st = stack(name)
        ph = st.get("update_phase")
        if ph != seen:
            seen = ph
            rec = {"t": round(time.time() - t0, 1), "phase": ph,
                   "label": st.get("update_phase_label"), "updating": st.get("updating"),
                   "error": st.get("update_error"), "hold": st.get("hold_reason")}
            phases.append(rec)
            say(f"      +{rec['t']:>6.1f}s  phase={ph}  label={rec['label']}  "
                f"err={rec['error']}  hold={rec['hold']}")
        if not st.get("updating") and ph in ("done", "failed", None) and time.time() - t0 > 3:
            break
        time.sleep(poll)
    st = stack(name)
    return {"accepted": True, "http": code, "phases": phases,
            "duration_s": round(time.time() - t0, 1),
            "final_phase": st.get("update_phase"), "update_error": st.get("update_error"),
            "hold_reason": st.get("hold_reason"), "state": st.get("state")}


def observables(name):
    st = stack(name)
    ac = st.get("app_config") or {}
    live = guest(f"""
grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//'
echo '---inspect---'
for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do
  echo -n "$c  "; docker inspect "$c" --format '{{{{.Config.Image}}}}  running={{{{.State.Running}}}}  restarts={{{{.RestartCount}}}}'
done
""")
    a, _, b = live.partition("---inspect---")
    return {
        "pinned_images": ac.get("pinned_images"),
        "installed_images": {k: (v.get("ref") if isinstance(v, dict) else v)
                             for k, v in (ac.get("installed_images") or {}).items()},
        "catalog_images": st.get("catalog_images"),
        "live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()],
        "docker_inspect": [x for x in b.strip().splitlines() if x.strip()],
    }


def app_logs(name, lines=400):
    """The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is
    one string with escaped newlines — a scan over the envelope sees a single enormous line and
    finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96
    rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines."""
    code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}")
    if isinstance(d, dict):
        data = d.get("data")
        if isinstance(data, dict) and isinstance(data.get("logs"), str):
            return data["logs"]
        if isinstance(d.get("_raw"), str):
            return d["_raw"]
    return str(d)


def write_verdict(rec, appdir):
    os.makedirs(appdir, exist_ok=True)
    p = os.path.join(appdir, "verdict.json")
    json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False)
    say(f"  [9] verdict {rec['verdict']}  -> {p}")


def remove(name):
    """Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint
    refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up."""
    c1, d1 = ctl("POST", f"/api/stacks/{name}/stop")
    say(f"  [X] stop -> {c1} {str(d1)[:100]}")
    for _ in range(24):
        time.sleep(5)
        if stack(name).get("state") != "running":
            break
    code, d = ctl("POST", f"/api/stacks/{name}/remove",
                  {"remove_hdd_data": True, "remove_backups": True})
    say(f"  [X] remove (with drive data) -> {code} {str(d)[:160]}")
    if code == "409":
        # R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive
        # path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202
        # `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits
        # this. The household's other choice — remove the app, KEEP the data — is accepted, and the
        # harness takes it, then tidies its own directory by name at teardown.
        say("  [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)"
            " — removing the app and KEEPING the drive data instead")
        code, d = ctl("POST", f"/api/stacks/{name}/remove",
                      {"remove_hdd_data": False, "remove_backups": True})
        say(f"  [X] remove (keeping drive data) -> {code} {str(d)[:160]}")
    time.sleep(5)
    st = stack(name)
    left = guest(f"ls -d /opt/docker/stacks/{name} 2>/dev/null; "
                 f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'")
    say(f"  [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}")
    return code


def app_env(name, key):
    """Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the
    app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller
    shows them the same value. Data still goes in through the app's own front door."""
    out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1")
    if ":" in out:
        return out.split(":", 1)[1].strip().strip('"').strip("'")
    return ""


def snapshots(name):
    """The restorable copies the backups page offers for this app."""
    code, d = ctl("GET", f"/api/backup/snapshots?stack={name}")
    data = d.get("data") if isinstance(d, dict) else None
    if isinstance(data, dict):
        for k in ("snapshots", "items", "restore_points"):
            if isinstance(data.get(k), list):
                return data[k]
    return data if isinstance(data, list) else []


def restore(name, snapshot_id=None, wait_s=1200):
    """The household's own way out: the „Visszaállítás a mentésből" button on the backups page.

    A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`,
    `snapshot_id` — because that is the button the sentence tells them to press.
    """
    snaps = snapshots(name)
    if snapshot_id is None:
        if not snaps:
            say(f"  [R] no restorable copy offered for {name}")
            return {"ok": False, "why": "no snapshot offered", "snapshots": snaps}
        first = snaps[0]
        snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id")
    say(f"  [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)")
    sess = open(f"{SC}/sess.txt").read().strip()
    csrf = open(f"{SC}/csrf.txt").read().strip()
    r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}",
            "-X", "POST",
            "--data-urlencode", f"_csrf={csrf}",
            "--data-urlencode", f"stack_name={name}",
            "--data-urlencode", f"snapshot_id={snapshot_id}",
            f"{BASE}/backup/restore"], timeout=180)
    head = (r.stdout or "").split("\n")[0].strip()
    loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")]
    say(f"  [R] POST /backup/restore -> {head} {loc[:1]}")
    t0 = time.time()
    last = None
    while time.time() - t0 < wait_s:
        code, d = ctl("GET", "/api/backup/restore-status")
        dd = d.get("data") or {}
        cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message"))
        if cur != last:
            say(f"      +{round(time.time()-t0,1):>6.1f}s restore {cur}")
            last = cur
        if not dd.get("running", False) and time.time() - t0 > 5:
            break
        time.sleep(2)
    st = stack(name)
    say(f"  [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} "
        f"phase={st.get('update_phase')}")
    return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps,
            "http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1),
            "state_after": st.get("state"), "hold_after": st.get("hold_reason"),
            "observables_after": observables(name)}