261 lines
13 KiB
Python
261 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""d63_live.py <phase> — D63 live proofs on Tester 1 (2026-10-29, attended), test build with `-tags faultinject`.
|
|
|
|
Phases:
|
|
setup deploy a throwaway Radicale, seed T1 through its own front door, run it at 3.8.0 (the stack folder's
|
|
compose edited, the household's Restart), switch it into the off-site set and press „Mentés most" for
|
|
off-site, then put the compose back at 3.8.1 and Restart — so the newest off-site snapshot holds 3.8.0's
|
|
data under 3.8.1's live definition.
|
|
pull pull the newest off-site snapshot of radicale to the restore scratch (the wizard's two steps).
|
|
h4 seed T2; arm defwrite-after-compose "0 1"; press the full restore. Want: the fault, then the live definition
|
|
written back BEFORE the start; the app runs 3.8.1; no hold; T1 and T2 read back.
|
|
d62 a full restore with no fault; the copy is listed on the restore page with its date; a wrong name is refused;
|
|
the right name deletes it; it is gone.
|
|
h5 a created-only container holds radicale's data volume (the restore cannot replace it → the put-back);
|
|
arm stop-radicale "1 1" (the restore's own stop passes, the put-back's stop fails). Want: nothing is put
|
|
back, the copy stays, the app is held with the D20 sentence; the page shows the held copy's button OFF.
|
|
teardown the blocker and the fault files go; the app is removed through the product.
|
|
State (passwords) stays in the scratchpad, never printed."""
|
|
import json, os, re, subprocess, sys, time
|
|
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad"
|
|
os.environ.setdefault("SC", f"{SCR}/rt-tester1")
|
|
os.environ.setdefault("TARGET", "tester-1")
|
|
import box_walk as w
|
|
import upgrade_fixtures_box as fb
|
|
|
|
EV = f"{SCR}/d63/ev"
|
|
os.makedirs(EV, exist_ok=True)
|
|
STATE = f"{SCR}/d63/.state.json"
|
|
APP, SUB = "radicale", "radicale"
|
|
fx = fb.FIXTURES[APP]
|
|
FAULTS = "/var/lib/docker/volumes/felhom-controller-data/_data/faults"
|
|
COMPOSE = f"/opt/docker/stacks/{APP}/docker-compose.yml"
|
|
OLD, NEW = "ghcr.io/kozea/radicale:3.8.0", "ghcr.io/kozea/radicale:3.8.1"
|
|
OLD_FULL = "ghcr.io/kozea/radicale:3.8.0@sha256:0d60d544710bda0fa83bfdea685db731b6cbf74e12d7f22b4997e9cff3a50468"
|
|
NEW_FULL = "ghcr.io/kozea/radicale:3.8.1@sha256:54d9406cd30f9e206a9dd6d61dfac48da26b9afc37200e14f23beb8fafa7d11c"
|
|
|
|
|
|
def state():
|
|
return json.load(open(STATE)) if os.path.exists(STATE) else {}
|
|
|
|
|
|
def save(st):
|
|
json.dump(st, open(STATE, "w"))
|
|
os.chmod(STATE, 0o600)
|
|
|
|
|
|
def form(path, fields, accept_json=False):
|
|
"""A dashboard form post with the session and its CSRF token — what the page's button sends."""
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
args = ["curl", "-sk", "-D", "-", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}"]
|
|
if accept_json:
|
|
args += ["-H", "Accept: application/json"]
|
|
for k, v in fields.items():
|
|
args += ["--data-urlencode", f"{k}={v}"]
|
|
bodyf = f"{SCR}/d63/.body"
|
|
r = w.sh(args + ["-o", bodyf, f"{w.BASE}{path}"], timeout=180)
|
|
out = r.stdout or ""
|
|
head = out.split("\n")[0].strip()
|
|
loc = [l.strip() for l in out.split("\n") if l.lower().startswith("location:")]
|
|
body = open(bodyf).read() if os.path.exists(bodyf) else ""
|
|
return head, loc[:1], body
|
|
|
|
|
|
def wait_op(label, cap=1800):
|
|
t0, last = time.time(), None
|
|
while time.time() - t0 < cap:
|
|
code, d = w.ctl("GET", "/api/backup/restore-status")
|
|
dd = d.get("data") or {}
|
|
cur = (dd.get("running"), dd.get("op"), dd.get("ok"), (dd.get("message") or "")[:300])
|
|
if cur != last:
|
|
w.say(f" +{round(time.time()-t0):>5}s {label}: {cur}")
|
|
last = cur
|
|
if not dd.get("running") and time.time() - t0 > 5:
|
|
return dd
|
|
time.sleep(3)
|
|
return {}
|
|
|
|
|
|
def ctl_log(since, pat):
|
|
return w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -E '{pat}' | tail -60")
|
|
|
|
|
|
def now():
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
def look():
|
|
st = w.stack(APP)
|
|
w.say(f" app: state={st.get('state')} hold={st.get('hold_reason')!r} deployed={st.get('deployed')}")
|
|
w.say(" compose image: " + w.guest(f"grep -E '^ +image:' {COMPOSE}").strip())
|
|
w.say(" container: " + w.guest(f"docker ps -a --filter name=^/{APP}$ --format '{{{{.Image}}}} {{{{.Status}}}}'").strip())
|
|
w.say(" copies: " + (w.guest("find /mnt/*/felhom-data/backups/restore-undo /opt/docker/felhom-data/backups/restore-undo -mindepth 2 -maxdepth 2 -type d 2>/dev/null | head").strip() or "(none)"))
|
|
return st
|
|
|
|
|
|
def set_image(img, label, full=False):
|
|
if full:
|
|
w.guest(f"sed -i -E 's#^( +image: )ghcr.io/kozea/radicale:[^ ]+$#\\1{img}#' {COMPOSE}")
|
|
img = img.split("@")[0]
|
|
else:
|
|
w.guest(f"sed -i -E 's#^( +image: )ghcr.io/kozea/radicale:[0-9.]+#\\1{img}#' {COMPOSE}")
|
|
c, d = w.ctl("POST", f"/api/stacks/{APP}/restart")
|
|
w.say(f" [{label}] compose → {img}; Restart → {c} {str(d)[:120]}")
|
|
for _ in range(40):
|
|
time.sleep(3)
|
|
if img in w.guest(f"docker ps --filter name=^/{APP}$ --format '{{{{.Image}}}}'"):
|
|
break
|
|
look()
|
|
|
|
|
|
w.login()
|
|
phase = sys.argv[1]
|
|
st = state()
|
|
if st.get("t1"): # the deploy's generated login lives only in the deploying process; the seed reads w.GENERATED
|
|
w.GENERATED[APP] = {"RADICALE_USER": st["t1"]["user"], "RADICALE_PASSWORD": st["t1"]["pw"]}
|
|
t_start = now()
|
|
w.say(f"=== phase {phase} at {t_start}")
|
|
w.say("controller image: " + w.guest("cat /etc/felhom-controller-image; docker ps --filter name=felhom-controller --format '{{.Image}} {{.Status}}'").strip())
|
|
|
|
if phase == "setup":
|
|
assert w.deploy(APP, SUB), "deploy failed"
|
|
t1 = fx.seed(w, SUB, w.say)
|
|
assert t1, "seed T1 failed"
|
|
st["t1"] = t1
|
|
save(st)
|
|
set_image(OLD, "setup")
|
|
w.say(f" T1 readback at 3.8.0: {fx.verify(w, SUB, t1, w.say)}")
|
|
w.say(" off-site toggle: " + str(form("/backup/offbox/toggle", {"app": APP, "enabled": "on"})[:2]))
|
|
w.say(" off-site run: " + str(form("/backup/offbox/run", {})[:2]))
|
|
t0 = time.time()
|
|
while time.time() - t0 < 3600:
|
|
time.sleep(15)
|
|
if "manual off-box backup" in ctl_log(t_start, "off-box backup|offbox.*(completed|finished|failed)") or \
|
|
re.search(r"(?i)off-?box (run|backup) (completed|finished|done)", ctl_log(t_start, "offbox|off-box")):
|
|
break
|
|
w.say(" off-site run log: " + ctl_log(t_start, "offbox|off-box|radicale").strip()[-3000:])
|
|
set_image(NEW, "setup")
|
|
w.say(f" T1 readback at 3.8.1: {fx.verify(w, SUB, t1, w.say)}")
|
|
|
|
elif phase in ("setup2", "setup3"):
|
|
# The off-site run pushes recovery units; a fresh app has none until a backup run made one ("skipped (no unit)").
|
|
# The household's „Mentés most" (whole box, the only backup press there is — R-648) makes radicale's unit at 3.8.0.
|
|
look()
|
|
c, d = w.ctl("POST", "/api/backup/run")
|
|
w.say(f" local backup press → {c} {str(d)[:200]}")
|
|
t0 = time.time()
|
|
while time.time() - t0 < 2400:
|
|
time.sleep(10)
|
|
lg = ctl_log(t_start, r"App-data backup completed|\[backup\].*(failed|FAILED)|backup run (finished|complete)")
|
|
if "App-data backup completed" in lg:
|
|
break
|
|
w.say(" local backup log: " + ctl_log(t_start, r"App-data backup completed|radicale").strip()[-2500:])
|
|
w.say(" off-site run: " + str(form("/backup/offbox/run", {})[:2]))
|
|
t1 = now()
|
|
t0 = time.time()
|
|
while time.time() - t0 < 2400:
|
|
time.sleep(10)
|
|
if re.search(r"\[offbox\] backup (OK|failed)", ctl_log(t1, r"\[offbox\] backup")):
|
|
break
|
|
w.say(" off-site run log: " + ctl_log(t1, r"\[offbox\] backup|radicale").strip()[-2500:])
|
|
if phase == "setup2":
|
|
set_image(NEW, "setup2")
|
|
w.say(f" T1 readback: {fx.verify(w, SUB, st['t1'], w.say)}")
|
|
w.say(" radicale's off-site snapshots: " + w.guest("ls -la /mnt/sys_drive/felhom-data/backups/primary/radicale 2>&1 | head -8"))
|
|
|
|
elif phase == "pull":
|
|
w.say(" prepare: " + str(form("/backup/offbox/restore", {"app": APP, "mode": "full"})[:2]))
|
|
w.say(" pull: " + str(form("/backup/offbox/restore", {"app": APP, "mode": "full", "confirm": "1"})[:2]))
|
|
wait_op("pull")
|
|
|
|
elif phase == "h4":
|
|
# the live definition moves to the TRUE 3.8.0 (its own digest) right before the restore — within one catalog-sync
|
|
# tick — so the snapshot (3.8.1) and the live definition differ and the restore writes the snapshot's definition
|
|
set_image(NEW_FULL, "h4", full=True) # T2 is seeded on 3.8.1 (09:05-09:08: the 401s were the harness's empty password in a new process, not 3.8.0)
|
|
t2 = None
|
|
for _ in range(6):
|
|
time.sleep(10)
|
|
t2 = fx.seed(w, SUB, w.say)
|
|
if t2:
|
|
break
|
|
assert t2, "seed T2 failed"
|
|
st["t2"] = t2
|
|
save(st)
|
|
set_image(OLD_FULL, "h4", full=True)
|
|
time.sleep(15)
|
|
look()
|
|
w.guest(f"mkdir -p {FAULTS} && echo '0 1' > {FAULTS}/defwrite-after-compose && cat {FAULTS}/defwrite-after-compose")
|
|
w.say(" armed: defwrite-after-compose = 0 1")
|
|
w.say(" restore: " + str(form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1"})[:2]))
|
|
rec = wait_op("h4 restore")
|
|
w.say(f" restore record: {json.dumps(rec)[:1200]}")
|
|
w.say(" controller log:\n" + ctl_log(t_start, "faultinject|radicale|R-1256|definition"))
|
|
look()
|
|
w.say(f" T1 readback: {fx.verify(w, SUB, st['t1'], w.say)}")
|
|
w.say(f" T2 readback: {fx.verify(w, SUB, t2, w.say)}")
|
|
w.say(" fault file left: " + (w.guest(f"ls {FAULTS} 2>/dev/null").strip() or "(none)"))
|
|
|
|
elif phase == "d62":
|
|
look()
|
|
w.say(" restore (no fault): " + str(form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1"})[:2]))
|
|
rec = wait_op("d62 restore")
|
|
# at once: the stable-start check releases the copy about a minute after a good restore (R-1270)
|
|
page = w.page("/backups/restore")
|
|
t_page = now()
|
|
rows = re.findall(r'<details class="kept-delete" data-restore-copy="([^"]+)">(.*?)</details>', page, re.S)
|
|
w.say(f" page at {t_page}: {len(rows)} deletable restore copy row(s); restore ok={(rec.get('last') or {}).get('ok')}")
|
|
m = re.search(r'data-restore-copies="true">(.*?)</div>\s*<script>', page, re.S)
|
|
if m:
|
|
w.say(" list text: " + re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", m.group(1))).strip()[:800])
|
|
if rows:
|
|
app_stamp = rows[0][0]
|
|
app, stamp = app_stamp.split("/", 1)
|
|
names = re.findall(r'<span class="app-row-name">([^<]+)</span>', page[:page.find('data-restore-copy="' + app_stamp)])
|
|
typed = names[-1].strip() if names else app
|
|
h, l, b = form("/backup/restore-copy/delete", {"app": app, "stamp": stamp, "confirm": typed + "x"}, accept_json=True)
|
|
w.say(f" wrong name ({typed + 'x'!r}) → {h} {b.strip()[:300]}")
|
|
h, l, b = form("/backup/restore-copy/delete", {"app": app, "stamp": stamp, "confirm": typed}, accept_json=True)
|
|
w.say(f" right name ({typed!r}) → {h} {b.strip()[:300]}")
|
|
look()
|
|
page2 = w.page("/backups/restore")
|
|
w.say(f" page after: rows={page2.count('data-restore-copy=')}")
|
|
w.say(" controller log: " + ctl_log(t_start, "restore copy DELETED|restore-copy delete|restore_copy_deleted|Event push|released"))
|
|
else:
|
|
w.say(" NO deletable copy listed — the D62 page proof did not happen")
|
|
rec2 = wait_op("settle", cap=5)
|
|
w.say(f" T1 readback: {fx.verify(w, SUB, st['t1'], w.say)}")
|
|
|
|
elif phase == "h5":
|
|
look()
|
|
vols = w.guest(f"docker volume ls --format '{{{{.Name}}}}' | grep -E '^{APP}' | sort").split()
|
|
w.say(f" volumes: {vols}")
|
|
datav = vols[0]
|
|
w.say(" blocker: " + w.guest(f"docker rm -f d63-blocker >/dev/null 2>&1; docker create --name d63-blocker -v {datav}:/x alpine true"))
|
|
w.say(" premise (rm -f with the blocker): " + w.guest(f"docker volume rm -f {datav} 2>&1; echo rc=$?").strip())
|
|
w.guest(f"mkdir -p {FAULTS} && echo '1 1' > {FAULTS}/stop-{APP}")
|
|
w.say(f" armed: stop-{APP} = 1 1")
|
|
w.say(" restore: " + str(form("/backup/offbox/reconstitute", {"app": APP, "confirm": "1"})[:2]))
|
|
rec = wait_op("h5 restore")
|
|
w.say(f" restore record: {json.dumps(rec)[:1500]}")
|
|
w.say(" controller log:\n" + ctl_log(t_start, "faultinject|radicale|R-1256|put-back|put back|Failed to remove|hold"))
|
|
st2 = look()
|
|
app_page = w.page(f"/apps/{APP}")
|
|
held = re.search(r'data-held="true">(.*?)</div>', app_page, re.S)
|
|
w.say(" app page hold sentence: " + (re.sub(r"<[^>]+>", " ", held.group(1)).strip()[:600] if held else "(none)"))
|
|
page = w.page("/backups/restore")
|
|
w.say(f" restore page: button off={'data-restore-copy-off' in page}, needed line="
|
|
f"{re.findall(r'data-restore-copy-needed=.true.>([^<]+)<', page)[:2]}, deletable rows={page.count('data-restore-copy=')}")
|
|
w.say(" fault file left: " + (w.guest(f"ls {FAULTS} 2>/dev/null").strip() or "(none)"))
|
|
|
|
elif phase == "teardown":
|
|
w.guest(f"docker rm -f d63-blocker >/dev/null 2>&1; rm -rf {FAULTS}")
|
|
look()
|
|
w.remove(APP)
|
|
look()
|
|
|
|
open(os.path.join(EV, f"{phase}.log"), "a").write("\n".join(w.LOG) + "\n")
|