admin
aedaab8944
gates / gates (push) Successful in 26s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
256 lines
14 KiB
Python
256 lines
14 KiB
Python
"""kp.py <step> [arg] — kept data + off-site restore, live, one step per call (2026-09-28, controller 0.277.0).
|
|
|
|
EVIDENCE, NOT PRODUCT. Every act goes through the product's own endpoints (the ones the pages call):
|
|
POST /api/stacks/nextcloud/deploy (with and without kept_data) · /stop · /remove · POST /backups/window
|
|
POST /kept-data/delete · GET /kept-data · the restore pages' form posts
|
|
and the app's OWN front doors for data: `occ user:add` inside nextcloud's container, and WebDAV as the
|
|
seeded user (R-156: nothing planted in a volume, no SQL).
|
|
|
|
Env: GUEST=9202|9201 (demo-hp), SC=<scratchpad> (seed tokens live there, never in the evidence tree),
|
|
OUT=<evidence file>. The throwaway app is nextcloud; the sub-domain is SUB (default c-nc).
|
|
"""
|
|
import json, os, re, subprocess, sys, time
|
|
import walk as w, fixtures
|
|
|
|
step = sys.argv[1]
|
|
arg = sys.argv[2] if len(sys.argv) > 2 else ""
|
|
APP = "nextcloud"
|
|
SUB = os.environ.get("SUB", "c-nc")
|
|
HDD = os.environ.get("HDD", {"9202": "/mnt/felhom-drives/scratch_hdd", "9201": "/mnt/felhom-drives/hdd_1"}[w.GUEST])
|
|
w.DRIVE = HDD + "/userdata" # the helper hard-codes 9202's drive for the folders a deploy needs
|
|
TOK = os.path.join(w.SC, f"seed-tokens-{w.GUEST}.json")
|
|
OUT = open(os.environ["OUT"], "a", buffering=1)
|
|
|
|
|
|
def say(*a):
|
|
w.say(*a)
|
|
OUT.write(time.strftime("%H:%M:%S ") + " ".join(map(str, a)) + "\n")
|
|
|
|
|
|
def g(cmd, timeout=600):
|
|
return w.guest(cmd, timeout=timeout)
|
|
|
|
|
|
def toks():
|
|
try:
|
|
return json.load(open(TOK))
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def save_toks(t):
|
|
old = os.umask(0o077)
|
|
json.dump(t, open(TOK, "w"), default=str)
|
|
os.umask(old)
|
|
|
|
|
|
def deploy(choice=None, lang=""):
|
|
vals = w.deploy_values(APP, SUB)
|
|
vals["HDD_PATH"] = HDD
|
|
body = {"values": vals}
|
|
if choice:
|
|
body["kept_data"] = choice
|
|
return w.ctl("POST", f"/api/stacks/{APP}/deploy" + ("?lang=en" if lang else ""), body)
|
|
|
|
|
|
def wait_deployed(limit=900):
|
|
t0 = time.time()
|
|
while time.time() - t0 < limit:
|
|
st = w.stack(APP)
|
|
if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") and st.get("state") in ("running", "unhealthy", "degraded"):
|
|
return round(time.time() - t0, 1), st.get("state")
|
|
time.sleep(5)
|
|
return None, w.stack(APP).get("state")
|
|
|
|
|
|
def dav(mode, name=""):
|
|
"""A file through Nextcloud's OWN WebDAV as the seeded user."""
|
|
t = toks()["nextcloud"]
|
|
base = f"{w.BASE}/remote.php/dav/files/{t['uid']}/"
|
|
a = ["curl", "-sk", "--max-time", "60", "-u", f"{t['uid']}:{t['pw']}", "-H", f"Host: {SUB}.{w.DOMAIN}", "-w", "\n%{http_code}"]
|
|
if mode == "put":
|
|
r = subprocess.run(a + ["-X", "PUT", "--data-binary", f"kept-offsite {name} {time.strftime('%FT%T')}", base + name], capture_output=True, text=True)
|
|
return r.stdout.strip().splitlines()[-1] if r.stdout.strip() else "?"
|
|
r = subprocess.run(a + ["-X", "PROPFIND", "-H", "Depth: 1", base], capture_output=True, text=True)
|
|
lines = r.stdout.strip().splitlines()
|
|
code = lines[-1] if lines else "?"
|
|
names = sorted(set(re.findall(r"<d:href>[^<]*/([^/<]+)</d:href>", r.stdout)))
|
|
return code, names
|
|
|
|
|
|
def files_report(expect_present, expect_absent):
|
|
code, names = dav("ls")
|
|
ok = code == "207" and all(n in names for n in expect_present) and not any(n in names for n in expect_absent)
|
|
say(f"PROPFIND as the seeded user -> {code}; present {expect_present}: {[n in names for n in expect_present]}; "
|
|
f"absent {expect_absent}: {[n not in names for n in expect_absent]}; listing={names}")
|
|
return ok
|
|
|
|
|
|
def users_report(want_present, want_absent):
|
|
nc = fixtures.FIXTURES["nextcloud"]
|
|
res = {}
|
|
for u in want_present + want_absent + ["nobody" + os.urandom(3).hex()]:
|
|
got, _ = nc._info(w, u)
|
|
res[u] = got
|
|
ok = all(res[u] is True for u in want_present) and all(res[u] is False for u in want_absent)
|
|
say(f"occ user:info — present {want_present}, absent {want_absent} (+ a uid that cannot exist, the control): {res}")
|
|
return ok
|
|
|
|
|
|
def form(path, fields):
|
|
"""A FORM post the way the page sends it (session cookie + _csrf). Returns (status line, location)."""
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
args = ["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}"]
|
|
for k, v in fields.items():
|
|
args += ["--data-urlencode", f"{k}={v}"]
|
|
r = w.sh(args + [f"{w.BASE}{path}"], timeout=300)
|
|
lines = (r.stdout or "").split("\n")
|
|
loc = [l.split(":", 1)[1].strip() for l in lines if l.lower().startswith("location:")]
|
|
return lines[0].strip() if lines else "?", (loc[0] if loc else "")
|
|
|
|
|
|
def wait_restore(label, cap=3600):
|
|
t0 = time.time(); last = None
|
|
while time.time() - t0 < cap:
|
|
d = w.ctl("GET", "/api/backup/restore-status")[1].get("data") or {}
|
|
cur = (d.get("running"), d.get("op"), (d.get("last") or {}).get("ok"), (d.get("last") or {}).get("message"))
|
|
if cur != last:
|
|
say(f" {label} +{round(time.time()-t0)}s restore-status running={cur[0]} op={cur[1]} last.ok={cur[2]} msg={str(cur[3])[:220]!r}")
|
|
last = cur
|
|
if not d.get("running") and time.time() - t0 > 5:
|
|
return d
|
|
time.sleep(3)
|
|
return {}
|
|
|
|
|
|
w.login()
|
|
say(f"# kp {step} {arg} — {time.strftime('%FT%T%z')}; guest {w.GUEST}; controller {g('cat /etc/felhom-controller-image').strip()}")
|
|
|
|
if step == "deploy":
|
|
st = w.stack(APP)
|
|
say("before: deployed=", st.get("deployed"), "| appdata:", g(f"ls {HDD}/appdata 2>&1 | tr '\\n' ' '"))
|
|
code, d = deploy()
|
|
say(f"deploy (no choice) -> {code} {json.dumps(d, ensure_ascii=False)[:300]}")
|
|
say("deployed after", wait_deployed())
|
|
elif step == "seed":
|
|
t = toks()
|
|
t["nextcloud"] = fixtures.FIXTURES["nextcloud"].seed(w, SUB, say)
|
|
save_toks(t)
|
|
say("seeded (the account):", t["nextcloud"] is not None, "uid=", (t["nextcloud"] or {}).get("uid"))
|
|
say(f"PUT before-backup.txt -> {dav('put', 'before-backup.txt')}")
|
|
files_report(["before-backup.txt"], [])
|
|
elif step == "marker":
|
|
# Written AFTER the snapshot that will be restored: a second account + a file.
|
|
nc = fixtures.FIXTURES["nextcloud"]
|
|
uid = "marker" + os.urandom(3).hex()
|
|
out = g(f"docker exec -u www-data -e OC_PASS=Marker-{os.urandom(8).hex()} nextcloud php occ user:add --password-from-env {uid} 2>&1")
|
|
say(f"occ user:add {uid} :: {' '.join(out.split())[:160]}")
|
|
t = toks(); t["marker"] = uid; save_toks(t)
|
|
say(f"PUT after-snapshot.txt -> {dav('put', 'after-snapshot.txt')}")
|
|
users_report([t["nextcloud"]["uid"], uid], [])
|
|
files_report(["before-backup.txt", "after-snapshot.txt"], [])
|
|
elif step == "read":
|
|
t = toks()
|
|
present = [t["nextcloud"]["uid"]] + ([] if arg == "no-marker" else ([t["marker"]] if t.get("marker") else []))
|
|
absent = [t["marker"]] if (arg == "no-marker" and t.get("marker")) else []
|
|
users_report(present, absent)
|
|
files_report(["before-backup.txt"], [])
|
|
say("state:", w.stack(APP).get("state"), "images:", g(f"docker ps --filter label=com.docker.compose.project={APP} --format '{{{{.Names}}}}={{{{.Image}}}}' | tr '\\n' ' '"))
|
|
elif step == "ask":
|
|
for lang in ("", "en"):
|
|
code, d = deploy(None, lang)
|
|
dd = d.get("data") or {}
|
|
say(f"deploy, no choice, lang={lang or 'hu'} -> {code} use_offered={dd.get('use_offered')} use_desc={dd.get('use_desc')!r} use_off={dd.get('use_off')!r}")
|
|
say("still not installed:", w.stack(APP).get("deployed"))
|
|
elif step == "use":
|
|
since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
|
|
code, d = deploy("use")
|
|
say(f"deploy kept_data=use -> {code} {json.dumps(d, ensure_ascii=False)[:240]}")
|
|
say("deployed after", wait_deployed(1500))
|
|
for i in range(90):
|
|
if int((g(f"docker logs --since {since} felhom-controller 2>&1 | grep -c -E 'after_load|kept load .* FAILED'").strip() or "0")) > 0:
|
|
break
|
|
time.sleep(5)
|
|
time.sleep(15)
|
|
say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'kept|after_load|restor|offbox|snapshot' | grep -v DEBUG | cut -c1-320 | head -50"))
|
|
say("restore status:", json.dumps(w.ctl("GET", "/api/backup/restore-status")[1].get("data"), ensure_ascii=False)[:400])
|
|
elif step == "remove-keep":
|
|
# arg: "keep-backups" (default) or "delete-backups"
|
|
code, d = w.ctl("POST", f"/api/stacks/{APP}/stop"); say("stop", code)
|
|
time.sleep(15)
|
|
rb = arg == "delete-backups"
|
|
code, d = w.ctl("POST", f"/api/stacks/{APP}/remove", {"remove_hdd_data": False, "remove_backups": rb})
|
|
say(f"remove (keep drive data, remove_backups={rb}) -> {code} {json.dumps(d, ensure_ascii=False)[:400]}")
|
|
time.sleep(8)
|
|
say("after: deployed=", w.stack(APP).get("deployed"), "| appdata:", g(f"ls {HDD}/appdata | tr '\\n' ' '; echo; ls -la --time-style=+%FT%T {HDD}/backups/primary/{APP} {HDD}/backups/primary/{APP}/db-dumps 2>&1 | tail -8; ls -d /mnt/*/*/backups/secondary/{APP} /mnt/*/backups/secondary/{APP} 2>&1"))
|
|
elif step == "window":
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"window_start={arg}", f"{w.BASE}/backups/window"])
|
|
say(f"POST /backups/window window_start={arg} -> {r.stdout}")
|
|
time.sleep(3)
|
|
say(g("docker logs --since 1m felhom-controller 2>&1 | grep -E 'rescheduled|window set' | tail -6"))
|
|
elif step == "logs":
|
|
say(g(f"docker logs --since {arg} felhom-controller 2>&1 | grep -iE '{APP}|job|offbox|snapshot|kept' | grep -v DEBUG | cut -c1-300 | tail -80"))
|
|
elif step == "kept":
|
|
for lang in ("", "en"):
|
|
h = w.page("/kept-data" + ("?lang=en" if lang else ""))
|
|
t = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", h))
|
|
i = t.find("extcloud")
|
|
say(f"GET /kept-data{'?lang=en' if lang else ''}: ...{t[max(0, i-200):i+500]}..." if i >= 0 else f"GET /kept-data{'?lang=en' if lang else ''}: no nextcloud row")
|
|
elif step == "snapshots":
|
|
code, d = w.ctl("GET", "/backup/offbox/status")
|
|
say("offbox status:", json.dumps(d, ensure_ascii=False)[:600])
|
|
say(g("docker exec felhom-controller sh -c 'ls /opt/docker/felhom-controller/data/offbox' 2>&1 | tr '\\n' ' '"))
|
|
say(g(f"docker logs --since {arg or '4h'} felhom-controller 2>&1 | grep -iE 'offbox|off-site|snapshot|nextcloud' | grep -v DEBUG | cut -c1-300 | tail -40"))
|
|
elif step == "offsite-restore":
|
|
since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
|
|
say("1 prepare (mode=full, no confirm) ->", form("/backup/offbox/restore", {"app": APP, "mode": "full"}))
|
|
say("2 download (mode=full, confirm=1) ->", form("/backup/offbox/restore", {"app": APP, "mode": "full", "confirm": "1"}))
|
|
wait_restore("download")
|
|
st, loc = form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1"})
|
|
say("3 reconstitute (confirm=1) ->", st, loc)
|
|
if "ack_placement" in loc or "placement" in loc:
|
|
say(" the page asked for the placement acknowledgement — sending it with confirm")
|
|
say("3b reconstitute (confirm=1, ack_placement=1) ->", form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1", "ack_placement": "1"}))
|
|
wait_restore("reconstitute")
|
|
say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'offbox|reconstitut|restor|nextcloud|version' | grep -v DEBUG | cut -c1-320 | head -60"))
|
|
st = w.stack(APP)
|
|
say("after: state=", st.get("state"), "pinned=", (st.get("app_config") or {}).get("pinned_images"))
|
|
elif step == "offsite-run":
|
|
say("E2 manual off-site run: the backup page's run-now button, POST /backup/offbox/run")
|
|
say("apps before:", g("docker ps --format '{{.Names}} {{.Status}}' | sort | tr '\\n' ';'")[:900])
|
|
since = g("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
|
|
say("POST /backup/offbox/run ->", form("/backup/offbox/run", {}))
|
|
t0 = time.time(); last = None
|
|
while time.time() - t0 < 3600:
|
|
c, d = w.ctl("GET", "/backup/offbox/status")
|
|
p = (d.get("progress") or {}); cur = (p.get("active"), p.get("phase"), p.get("current_app"))
|
|
if cur != last:
|
|
say(f" +{round(time.time()-t0)}s active={cur[0]} phase={cur[1]} app={cur[2]}"); last = cur
|
|
if not p.get("active") and time.time() - t0 > 20:
|
|
break
|
|
time.sleep(5)
|
|
c, d = w.ctl("GET", "/backup/offbox/status")
|
|
say("status after:", {k: d.get(k) for k in ("status", "last_run", "last_error", "snapshots", "last_duration")})
|
|
say("controller lines:\n" + g(f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'offbox|nextcloud|snapshot|dump' | grep -v DEBUG | cut -c1-260 | tail -40"))
|
|
say("apps after:", g("docker ps --format '{{.Names}} {{.Status}}' | sort | tr '\\n' ';'")[:900])
|
|
elif step == "kept-delete":
|
|
h = w.page("/kept-data")
|
|
paths = sorted(set(re.findall(r'name="path" value="([^"]*nextcloud[^"]*)"', h)))
|
|
say("kept nextcloud items:", paths)
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
for p in paths:
|
|
r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"path={p}", "--data-urlencode", "confirm=Nextcloud",
|
|
f"{w.BASE}/kept-data/delete"])
|
|
loc = [l for l in r.stdout.split("\n") if l.lower().startswith("location:")]
|
|
say(f"POST /kept-data/delete {p} -> {r.stdout.splitlines()[0] if r.stdout else '?'} {loc[:1]}")
|
|
say("appdata now:", g(f"ls {HDD}/appdata {HDD}/kept 2>&1 | tr '\\n' ' '"))
|
|
else:
|
|
sys.exit(f"unknown step {step}")
|