"""kp.py <step> [arg] — kept data + off-site restore, live, one step per call (2026-09-28, controller 0.277.0).

EVIDENCE, NOT PRODUCT. Every act goes through the product's own endpoints (the ones the pages call):
  POST /api/stacks/nextcloud/deploy (with and without kept_data) · /stop · /remove · POST /backups/window
  POST /kept-data/delete · GET /kept-data · the restore pages' form posts
and the app's OWN front doors for data: `occ user:add` inside nextcloud's container, and WebDAV as the
seeded user (R-156: nothing planted in a volume, no SQL).

Env: GUEST=9202|9201 (demo-hp), SC=<scratchpad> (seed tokens live there, never in the evidence tree),
OUT=<evidence file>. The throwaway app is nextcloud; the sub-domain is SUB (default c-nc).
"""
import json, os, re, subprocess, sys, time
import walk as w, fixtures

step = sys.argv[1]
arg = sys.argv[2] if len(sys.argv) > 2 else ""
APP = "nextcloud"
SUB = os.environ.get("SUB", "c-nc")
HDD = os.environ.get("HDD", {"9202": "/mnt/felhom-drives/scratch_hdd", "9201": "/mnt/felhom-drives/hdd_1"}[w.GUEST])
w.DRIVE = HDD + "/userdata"  # the helper hard-codes 9202's drive for the folders a deploy needs
TOK = os.path.join(w.SC, f"seed-tokens-{w.GUEST}.json")
OUT = open(os.environ["OUT"], "a", buffering=1)


def say(*a):
    w.say(*a)
    OUT.write(time.strftime("%H:%M:%S ") + " ".join(map(str, a)) + "\n")


def g(cmd, timeout=600):
    return w.guest(cmd, timeout=timeout)


def toks():
    try:
        return json.load(open(TOK))
    except Exception:
        return {}


def save_toks(t):
    old = os.umask(0o077)
    json.dump(t, open(TOK, "w"), default=str)
    os.umask(old)


def deploy(choice=None, lang=""):
    vals = w.deploy_values(APP, SUB)
    vals["HDD_PATH"] = HDD
    body = {"values": vals}
    if choice:
        body["kept_data"] = choice
    return w.ctl("POST", f"/api/stacks/{APP}/deploy" + ("?lang=en" if lang else ""), body)


def wait_deployed(limit=900):
    t0 = time.time()
    while time.time() - t0 < limit:
        st = w.stack(APP)
        if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") and st.get("state") in ("running", "unhealthy", "degraded"):
            return round(time.time() - t0, 1), st.get("state")
        time.sleep(5)
    return None, w.stack(APP).get("state")


def dav(mode, name=""):
    """A file through Nextcloud's OWN WebDAV as the seeded user."""
    t = toks()["nextcloud"]
    base = f"{w.BASE}/remote.php/dav/files/{t['uid']}/"
    a = ["curl", "-sk", "--max-time", "60", "-u", f"{t['uid']}:{t['pw']}", "-H", f"Host: {SUB}.{w.DOMAIN}", "-w", "\n%{http_code}"]
    if mode == "put":
        r = subprocess.run(a + ["-X", "PUT", "--data-binary", f"kept-offsite {name} {time.strftime('%FT%T')}", base + name], capture_output=True, text=True)
        return r.stdout.strip().splitlines()[-1] if r.stdout.strip() else "?"
    r = subprocess.run(a + ["-X", "PROPFIND", "-H", "Depth: 1", base], capture_output=True, text=True)
    lines = r.stdout.strip().splitlines()
    code = lines[-1] if lines else "?"
    names = sorted(set(re.findall(r"<d:href>[^<]*/([^/<]+)</d:href>", r.stdout)))
    return code, names


def files_report(expect_present, expect_absent):
    code, names = dav("ls")
    ok = code == "207" and all(n in names for n in expect_present) and not any(n in names for n in expect_absent)
    say(f"PROPFIND as the seeded user -> {code}; present {expect_present}: {[n in names for n in expect_present]}; "
        f"absent {expect_absent}: {[n not in names for n in expect_absent]}; listing={names}")
    return ok


def users_report(want_present, want_absent):
    nc = fixtures.FIXTURES["nextcloud"]
    res = {}
    for u in want_present + want_absent + ["nobody" + os.urandom(3).hex()]:
        got, _ = nc._info(w, u)
        res[u] = got
    ok = all(res[u] is True for u in want_present) and all(res[u] is False for u in want_absent)
    say(f"occ user:info — present {want_present}, absent {want_absent} (+ a uid that cannot exist, the control): {res}")
    return ok


def form(path, fields):
    """A FORM post the way the page sends it (session cookie + _csrf). Returns (status line, location)."""
    sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
    csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
    args = ["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
            "--data-urlencode", f"_csrf={csrf}"]
    for k, v in fields.items():
        args += ["--data-urlencode", f"{k}={v}"]
    r = w.sh(args + [f"{w.BASE}{path}"], timeout=300)
    lines = (r.stdout or "").split("\n")
    loc = [l.split(":", 1)[1].strip() for l in lines if l.lower().startswith("location:")]
    return lines[0].strip() if lines else "?", (loc[0] if loc else "")


def wait_restore(label, cap=3600):
    t0 = time.time(); last = None
    while time.time() - t0 < cap:
        d = w.ctl("GET", "/api/backup/restore-status")[1].get("data") or {}
        cur = (d.get("running"), d.get("op"), (d.get("last") or {}).get("ok"), (d.get("last") or {}).get("message"))
        if cur != last:
            say(f"  {label} +{round(time.time()-t0)}s restore-status running={cur[0]} op={cur[1]} last.ok={cur[2]} msg={str(cur[3])[:220]!r}")
            last = cur
        if not d.get("running") and time.time() - t0 > 5:
            return d
        time.sleep(3)
    return {}


w.login()
say(f"# kp {step} {arg} — {time.strftime('%FT%T%z')}; guest {w.GUEST}; controller {g('cat /etc/felhom-controller-image').strip()}")

if step == "deploy":
    st = w.stack(APP)
    say("before: deployed=", st.get("deployed"), "| appdata:", g(f"ls {HDD}/appdata 2>&1 | tr '\\n' ' '"))
    code, d = deploy()
    say(f"deploy (no choice) -> {code} {json.dumps(d, ensure_ascii=False)[:300]}")
    say("deployed after", wait_deployed())
elif step == "seed":
    t = toks()
    t["nextcloud"] = fixtures.FIXTURES["nextcloud"].seed(w, SUB, say)
    save_toks(t)
    say("seeded (the account):", t["nextcloud"] is not None, "uid=", (t["nextcloud"] or {}).get("uid"))
    say(f"PUT before-backup.txt -> {dav('put', 'before-backup.txt')}")
    files_report(["before-backup.txt"], [])
elif step == "marker":
    # Written AFTER the snapshot that will be restored: a second account + a file.
    nc = fixtures.FIXTURES["nextcloud"]
    uid = "marker" + os.urandom(3).hex()
    out = g(f"docker exec -u www-data -e OC_PASS=Marker-{os.urandom(8).hex()} nextcloud php occ user:add --password-from-env {uid} 2>&1")
    say(f"occ user:add {uid} :: {' '.join(out.split())[:160]}")
    t = toks(); t["marker"] = uid; save_toks(t)
    say(f"PUT after-snapshot.txt -> {dav('put', 'after-snapshot.txt')}")
    users_report([t["nextcloud"]["uid"], uid], [])
    files_report(["before-backup.txt", "after-snapshot.txt"], [])
elif step == "read":
    t = toks()
    present = [t["nextcloud"]["uid"]] + ([] if arg == "no-marker" else ([t["marker"]] if t.get("marker") else []))
    absent = [t["marker"]] if (arg == "no-marker" and t.get("marker")) else []
    users_report(present, absent)
    files_report(["before-backup.txt"], [])
    say("state:", w.stack(APP).get("state"), "images:", g(f"docker ps --filter label=com.docker.compose.project={APP} --format '{{{{.Names}}}}={{{{.Image}}}}' | tr '\\n' ' '"))
elif step == "ask":
    for lang in ("", "en"):
        code, d = deploy(None, lang)
        dd = d.get("data") or {}
        say(f"deploy, no choice, lang={lang or 'hu'} -> {code} use_offered={dd.get('use_offered')} use_desc={dd.get('use_desc')!r} use_off={dd.get('use_off')!r}")
    say("still not installed:", w.stack(APP).get("deployed"))
elif step == "use":
    since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
    code, d = deploy("use")
    say(f"deploy kept_data=use -> {code} {json.dumps(d, ensure_ascii=False)[:240]}")
    say("deployed after", wait_deployed(1500))
    for i in range(90):
        if int((g(f"docker logs --since {since} felhom-controller 2>&1 | grep -c -E 'after_load|kept load .* FAILED'").strip() or "0")) > 0:
            break
        time.sleep(5)
    time.sleep(15)
    say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'kept|after_load|restor|offbox|snapshot' | grep -v DEBUG | cut -c1-320 | head -50"))
    say("restore status:", json.dumps(w.ctl("GET", "/api/backup/restore-status")[1].get("data"), ensure_ascii=False)[:400])
elif step == "remove-keep":
    # arg: "keep-backups" (default) or "delete-backups"
    code, d = w.ctl("POST", f"/api/stacks/{APP}/stop"); say("stop", code)
    time.sleep(15)
    rb = arg == "delete-backups"
    code, d = w.ctl("POST", f"/api/stacks/{APP}/remove", {"remove_hdd_data": False, "remove_backups": rb})
    say(f"remove (keep drive data, remove_backups={rb}) -> {code} {json.dumps(d, ensure_ascii=False)[:400]}")
    time.sleep(8)
    say("after: deployed=", w.stack(APP).get("deployed"), "| appdata:", g(f"ls {HDD}/appdata | tr '\\n' ' '; echo; ls -la --time-style=+%FT%T {HDD}/backups/primary/{APP} {HDD}/backups/primary/{APP}/db-dumps 2>&1 | tail -8; ls -d /mnt/*/*/backups/secondary/{APP} /mnt/*/backups/secondary/{APP} 2>&1"))
elif step == "window":
    sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
    csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
    r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
              "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"window_start={arg}", f"{w.BASE}/backups/window"])
    say(f"POST /backups/window window_start={arg} -> {r.stdout}")
    time.sleep(3)
    say(g("docker logs --since 1m felhom-controller 2>&1 | grep -E 'rescheduled|window set' | tail -6"))
elif step == "logs":
    say(g(f"docker logs --since {arg} felhom-controller 2>&1 | grep -iE '{APP}|job|offbox|snapshot|kept' | grep -v DEBUG | cut -c1-300 | tail -80"))
elif step == "kept":
    for lang in ("", "en"):
        h = w.page("/kept-data" + ("?lang=en" if lang else ""))
        t = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", h))
        i = t.find("extcloud")
        say(f"GET /kept-data{'?lang=en' if lang else ''}: ...{t[max(0, i-200):i+500]}..." if i >= 0 else f"GET /kept-data{'?lang=en' if lang else ''}: no nextcloud row")
elif step == "snapshots":
    code, d = w.ctl("GET", "/backup/offbox/status")
    say("offbox status:", json.dumps(d, ensure_ascii=False)[:600])
    say(g("docker exec felhom-controller sh -c 'ls /opt/docker/felhom-controller/data/offbox' 2>&1 | tr '\\n' ' '"))
    say(g(f"docker logs --since {arg or '4h'} felhom-controller 2>&1 | grep -iE 'offbox|off-site|snapshot|nextcloud' | grep -v DEBUG | cut -c1-300 | tail -40"))
elif step == "offsite-restore":
    since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
    say("1 prepare (mode=full, no confirm) ->", form("/backup/offbox/restore", {"app": APP, "mode": "full"}))
    say("2 download (mode=full, confirm=1) ->", form("/backup/offbox/restore", {"app": APP, "mode": "full", "confirm": "1"}))
    wait_restore("download")
    st, loc = form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1"})
    say("3 reconstitute (confirm=1) ->", st, loc)
    if "ack_placement" in loc or "placement" in loc:
        say("   the page asked for the placement acknowledgement — sending it with confirm")
        say("3b reconstitute (confirm=1, ack_placement=1) ->", form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1", "ack_placement": "1"}))
    wait_restore("reconstitute")
    say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'offbox|reconstitut|restor|nextcloud|version' | grep -v DEBUG | cut -c1-320 | head -60"))
    st = w.stack(APP)
    say("after: state=", st.get("state"), "pinned=", (st.get("app_config") or {}).get("pinned_images"))
elif step == "offsite-run":
    say("E2 manual off-site run: the backup page's run-now button, POST /backup/offbox/run")
    say("apps before:", g("docker ps --format '{{.Names}} {{.Status}}' | sort | tr '\\n' ';'")[:900])
    since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
    say("POST /backup/offbox/run ->", form("/backup/offbox/run", {}))
    t0 = time.time(); last = None
    while time.time() - t0 < 3600:
        c, d = w.ctl("GET", "/backup/offbox/status")
        p = (d.get("progress") or {}); cur = (p.get("active"), p.get("phase"), p.get("current_app"))
        if cur != last:
            say(f"  +{round(time.time()-t0)}s active={cur[0]} phase={cur[1]} app={cur[2]}"); last = cur
        if not p.get("active") and time.time() - t0 > 20:
            break
        time.sleep(5)
    c, d = w.ctl("GET", "/backup/offbox/status")
    say("status after:", {k: d.get(k) for k in ("status", "last_run", "last_error", "snapshots", "last_duration")})
    say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'offbox|nextcloud|snapshot|dump' | grep -v DEBUG | cut -c1-260 | tail -40"))
    say("apps after:", g("docker ps --format '{{.Names}} {{.Status}}' | sort | tr '\\n' ';'")[:900])
elif step == "kept-delete":
    h = w.page("/kept-data")
    paths = sorted(set(re.findall(r'name="path" value="([^"]*nextcloud[^"]*)"', h)))
    say("kept nextcloud items:", paths)
    sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
    csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
    for p in paths:
        r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
                  "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"path={p}", "--data-urlencode", "confirm=Nextcloud",
                  f"{w.BASE}/kept-data/delete"])
        loc = [l for l in r.stdout.split("\n") if l.lower().startswith("location:")]
        say(f"POST /kept-data/delete {p} -> {r.stdout.splitlines()[0] if r.stdout else '?'} {loc[:1]}")
    say("appdata now:", g(f"ls {HDD}/appdata {HDD}/kept 2>&1 | tr '\\n' ' '"))
else:
    sys.exit(f"unknown step {step}")