#!/usr/bin/env python3 """live.py — controller v0.263.0's undo, proven on guest 9202 through the endpoints the UI invokes. Nothing here performs an undo: the PRODUCT does. This presses Update, reads GET /api/stacks/, fetches the app page in both languages, and reads the seeds back through each app's own front door. Seed C is written immediately before each Update, after every backup — so only the undo's own last-second copy can bring it back. """ import json, re, sys, time, html as H sys.path.insert(0, ".") import walk as w from spike import load, save, ts import bakeoff as b HEALTH = {"docmost": "/", "romm": "/api/heartbeat", "vikunja": "/api/v1/info"} def seed_c(app, s): return b.seed_b(app, s["sub"], s["seedA"]) def page_lines(app): out = {} for lang in ("hu", "en"): h = H.unescape(w.page(f"/apps/{app}?lang={lang}")) m = re.search(r'data-update-undone="true">([^<]*)<', h) hold = re.search(r'data-held="true">([^<]*)<', h) out[lang] = {"undone_line": m.group(1).strip() if m else None, "hold": hold.group(1).strip() if hold else None} return out def press(app, poll=0.5, on_phase=None): code, d = w.ctl("POST", f"/api/stacks/{app}/update") w.say(f" Update -> {code} {str(d)[:140]}") phases, seen, t0 = [], None, time.time() while time.time() - t0 < 1500: try: st = w.stack(app) except Exception as e: st = {} ph = st.get("update_phase") if ph != seen and ph is not None: seen = ph phases.append((round(time.time() - t0, 1), ph, st.get("update_phase_label"))) w.say(f" +{phases[-1][0]:6.1f}s phase={ph} label={st.get('update_phase_label')}") if on_phase and on_phase(ph): return phases, "interrupted" if st and not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - t0 > 2: break time.sleep(poll) st = w.stack(app) w.say(f" END phase={st.get('update_phase')} err={st.get('update_error')!r} hold={st.get('hold_reason')!r}") return phases, st def readback(app, s, with_c=True): sub = s["sub"] w.wait_app(sub, HEALTH[app], want=("200",), tries=60, delay=2) A = b.FX[app].verify(w, sub, s["seedA"], w.say) B = b.verify_b(app, sub, s["seedA"], s["seedB"]) C = b.verify_b(app, sub, s["seedA"], s["seedC"]) if with_c and s.get("seedC") else None return {"A": A, "B": B, "C": C} def stage_undo(app): s = load(app) w.say(f"=== {app}: live undo by the product") s["seedC"] = seed_c(app, s); s["seedC_at"] = ts() w.say(" seed C written right before the Update:", bool(s["seedC"])) before = b.db_state(app); w.say(" db before:", before) phases, st = press(app) obs = w.observables(app) w.say(" observables:", json.dumps(obs)) rb = readback(app, s) after = b.db_state(app) w.say(f" READBACK A={rb['A']} B={rb['B']} C={rb['C']} db after: {after} (before: {before})") pl = page_lines(app); w.say(" PAGE:", json.dumps(pl, ensure_ascii=False)) w.say(" leftover copies:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip()) s["live_undo"] = {"phases": phases, "end": {k: st.get(k) for k in ("update_phase", "update_error", "hold_reason")}, "readback": rb, "db_before": before, "db_after": after, "page": pl, "obs": obs} save(app, s) def set_box_language(lang): """POST /settings/language — the household's language switch (form + the dashboard's CSRF).""" sess = open(f"{w.SC}/sess.txt").read().strip(); csrf = open(f"{w.SC}/csrf.txt").read().strip() r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}", "--data-urlencode", f"lang={lang}", "--data-urlencode", f"gorilla.csrf.Token={csrf}", f"{w.BASE}/settings/language"]) w.say(f" box language -> {lang}: http {r.stdout.strip()}") def stage_powercut(app): """Press Update; the moment the phase reads `undoing`, cut the guest's power (`pct stop`, a hard stop); boot it again and let the controller resume the undo.""" import subprocess s = load(app) w.say(f"=== {app}: power cut DURING the undo") s["seedC"] = seed_c(app, s) cut = {} def on_phase(ph): if ph == "undoing": t = time.time() r = subprocess.run(["ssh", "demo-hp", "pct stop 9202"], capture_output=True, text=True, timeout=120) cut["at"] = ts(); cut["rc"] = r.returncode w.say(f" >>> POWER CUT (pct stop 9202) in phase undoing: rc={r.returncode} in {round(time.time()-t,1)}s") return True return False phases, _ = press(app, poll=0.3, on_phase=on_phase) if not cut: w.say(" the cut never landed in `undoing` — recorded as a MISS"); return r = subprocess.run(["ssh", "demo-hp", "pct start 9202"], capture_output=True, text=True, timeout=180) w.say(f" guest started again: rc={r.returncode}") for i in range(60): time.sleep(5) try: w.login(); st = w.stack(app) if st: break except SystemExit: continue w.say(w.guest("docker logs felhom-controller 2>&1 | grep -E 'update recovery|resuming the UNDO|UNDONE|UNDO failed' | head -6")) t0 = time.time() while time.time() - t0 < 600: st = w.stack(app) if not st.get("updating") and st.get("update_phase") in ("undone", "failed"): break time.sleep(3) w.say(f" after the restart: phase={st.get('update_phase')} hold={st.get('hold_reason')!r}") rb = readback(app, s) w.say(f" READBACK A={rb['A']} B={rb['B']} C={rb['C']} db: {b.db_state(app)}") w.say(" leftover copies:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip()) s["powercut"] = {"phases": phases, "cut": cut, "end": st.get("update_phase"), "readback": rb} save(app, s) def stage_cutoff(app): """Press Update; while the NEW version is in `verifying`, take the finished-marker away from one of the undo copies (a copy cut off mid-way looks exactly like this). The undo must refuse to pour it back and HOLD with the new prefix.""" s = load(app) w.say(f"=== {app}: a cut-off undo copy") done = {} def on_phase(ph): if ph == "verifying" and not done: out = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={app} | head -1); echo "copy: $c" docker run --rm -v $c:/c alpine sh -c 'ls -la /c; rm -f /c/felhom-undo-complete; ls /c'""") done["out"] = out w.say(" >>> finished-marker removed from one copy:\n" + out) return False phases, st = press(app, poll=0.5, on_phase=on_phase) before = b.db_state(app) pl = page_lines(app) w.say(" PAGE (box language hu):", json.dumps(pl, ensure_ascii=False)) set_box_language("en") pl_en = page_lines(app) w.say(" PAGE (box language en):", json.dumps(pl_en, ensure_ascii=False)) set_box_language("hu") w.say(" copies kept:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app}")) s["cutoff_live"] = {"phases": phases, "end": {k: st.get(k) for k in ("update_phase", "hold_reason")}, "page_hu_box": pl, "page_en_box": pl_en, "marker": done} save(app, s) def stage_fixprobe_and_press(app): """After an undo: the catalog fixes the new version's probe; a PERSON presses Update; it must work and end the undone note.""" s = load(app) frm, to, p0, p1 = b.EDGE[app] fy = f"{w.DRILL}/templates/{app}/.felhom.yml" f = open(fy).read().replace(f"port: {p1}", f"port: {p0}", 1); open(fy, "w").write(f) w.sh(["git", "-C", w.DRILL, "commit", "-qam", f"DRILL {app}: the probe fixed (port {p0}) — the step is now good"]) w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) w.sync_rescan(app, to) time.sleep(20) w.ctl("POST", "/api/sync"); time.sleep(3); w.ctl("POST", "/api/stacks/rescan") w.say(f"=== {app}: a person presses Update again after the undo (probe fixed in the catalog)") w.say(" before, the page:", json.dumps(page_lines(app), ensure_ascii=False)) phases, st = press(app) rb = readback(app, s) w.say(f" READBACK A={rb['A']} B={rb['B']} C={rb['C']} installed={w.observables(app)['installed_images']}") pl = page_lines(app); w.say(" after, the page:", json.dumps(pl, ensure_ascii=False)) w.say(" app.yaml last_update_undone:", w.guest(f"grep -c last_update_undone /opt/docker/stacks/{app}/app.yaml")) s["manual_after_undo"] = {"phases": phases, "end": st.get("update_phase"), "readback": rb, "page": pl} save(app, s) if __name__ == "__main__": w.login() {"undo": stage_undo, "powercut": stage_powercut, "cutoff": stage_cutoff, "fixpress": stage_fixprobe_and_press}[sys.argv[1]](sys.argv[2])