felhom.eu/documentation/audits/undo-live-2026-09-23/live.py

#!/usr/bin/env python3
"""live.py — controller v0.263.0's undo, proven on guest 9202 through the endpoints the UI invokes.

Nothing here performs an undo: the PRODUCT does. This presses Update, reads GET /api/stacks/<n>,
fetches the app page in both languages, and reads the seeds back through each app's own front door.
Seed C is written immediately before each Update, after every backup — so only the undo's own
last-second copy can bring it back.
"""
import json, re, sys, time, html as H
sys.path.insert(0, ".")
import walk as w
from spike import load, save, ts
import bakeoff as b

HEALTH = {"docmost": "/", "romm": "/api/heartbeat", "vikunja": "/api/v1/info"}


def seed_c(app, s):
    return b.seed_b(app, s["sub"], s["seedA"])


def page_lines(app):
    out = {}
    for lang in ("hu", "en"):
        h = H.unescape(w.page(f"/apps/{app}?lang={lang}"))
        m = re.search(r'data-update-undone="true">([^<]*)<', h)
        hold = re.search(r'data-held="true">([^<]*)<', h)
        out[lang] = {"undone_line": m.group(1).strip() if m else None, "hold": hold.group(1).strip() if hold else None}
    return out


def press(app, poll=0.5, on_phase=None):
    code, d = w.ctl("POST", f"/api/stacks/{app}/update")
    w.say(f"  Update -> {code} {str(d)[:140]}")
    phases, seen, t0 = [], None, time.time()
    while time.time() - t0 < 1500:
        try:
            st = w.stack(app)
        except Exception as e:
            st = {}
        ph = st.get("update_phase")
        if ph != seen and ph is not None:
            seen = ph
            phases.append((round(time.time() - t0, 1), ph, st.get("update_phase_label")))
            w.say(f"    +{phases[-1][0]:6.1f}s phase={ph} label={st.get('update_phase_label')}")
            if on_phase and on_phase(ph):
                return phases, "interrupted"
        if st and not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - t0 > 2:
            break
        time.sleep(poll)
    st = w.stack(app)
    w.say(f"  END phase={st.get('update_phase')} err={st.get('update_error')!r} hold={st.get('hold_reason')!r}")
    return phases, st


def readback(app, s, with_c=True):
    sub = s["sub"]
    w.wait_app(sub, HEALTH[app], want=("200",), tries=60, delay=2)
    A = b.FX[app].verify(w, sub, s["seedA"], w.say)
    B = b.verify_b(app, sub, s["seedA"], s["seedB"])
    C = b.verify_b(app, sub, s["seedA"], s["seedC"]) if with_c and s.get("seedC") else None
    return {"A": A, "B": B, "C": C}


def stage_undo(app):
    s = load(app)
    w.say(f"=== {app}: live undo by the product")
    s["seedC"] = seed_c(app, s); s["seedC_at"] = ts()
    w.say("  seed C written right before the Update:", bool(s["seedC"]))
    before = b.db_state(app); w.say("  db before:", before)
    phases, st = press(app)
    obs = w.observables(app)
    w.say("  observables:", json.dumps(obs))
    rb = readback(app, s)
    after = b.db_state(app)
    w.say(f"  READBACK A={rb['A']} B={rb['B']} C={rb['C']}   db after: {after}   (before: {before})")
    pl = page_lines(app); w.say("  PAGE:", json.dumps(pl, ensure_ascii=False))
    w.say("  leftover copies:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip())
    s["live_undo"] = {"phases": phases, "end": {k: st.get(k) for k in ("update_phase", "update_error", "hold_reason")},
                      "readback": rb, "db_before": before, "db_after": after, "page": pl, "obs": obs}
    save(app, s)




def set_box_language(lang):
    """POST /settings/language — the household's language switch (form + the dashboard's CSRF)."""
    sess = open(f"{w.SC}/sess.txt").read().strip(); csrf = open(f"{w.SC}/csrf.txt").read().strip()
    r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}",
              "-H", f"X-CSRF-Token: {csrf}", "--data-urlencode", f"lang={lang}", "--data-urlencode", f"gorilla.csrf.Token={csrf}",
              f"{w.BASE}/settings/language"])
    w.say(f"  box language -> {lang}: http {r.stdout.strip()}")


def stage_powercut(app):
    """Press Update; the moment the phase reads `undoing`, cut the guest's power (`pct stop`, a hard
    stop); boot it again and let the controller resume the undo."""
    import subprocess
    s = load(app)
    w.say(f"=== {app}: power cut DURING the undo")
    s["seedC"] = seed_c(app, s)
    cut = {}

    def on_phase(ph):
        if ph == "undoing":
            t = time.time()
            r = subprocess.run(["ssh", "demo-hp", "pct stop 9202"], capture_output=True, text=True, timeout=120)
            cut["at"] = ts(); cut["rc"] = r.returncode
            w.say(f"  >>> POWER CUT (pct stop 9202) in phase undoing: rc={r.returncode} in {round(time.time()-t,1)}s")
            return True
        return False
    phases, _ = press(app, poll=0.3, on_phase=on_phase)
    if not cut:
        w.say("  the cut never landed in `undoing` — recorded as a MISS"); return
    r = subprocess.run(["ssh", "demo-hp", "pct start 9202"], capture_output=True, text=True, timeout=180)
    w.say(f"  guest started again: rc={r.returncode}")
    for i in range(60):
        time.sleep(5)
        try:
            w.login(); st = w.stack(app)
            if st:
                break
        except SystemExit:
            continue
    w.say(w.guest("docker logs felhom-controller 2>&1 | grep -E 'update recovery|resuming the UNDO|UNDONE|UNDO failed' | head -6"))
    t0 = time.time()
    while time.time() - t0 < 600:
        st = w.stack(app)
        if not st.get("updating") and st.get("update_phase") in ("undone", "failed"):
            break
        time.sleep(3)
    w.say(f"  after the restart: phase={st.get('update_phase')} hold={st.get('hold_reason')!r}")
    rb = readback(app, s)
    w.say(f"  READBACK A={rb['A']} B={rb['B']} C={rb['C']}   db: {b.db_state(app)}")
    w.say("  leftover copies:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip())
    s["powercut"] = {"phases": phases, "cut": cut, "end": st.get("update_phase"), "readback": rb}
    save(app, s)


def stage_cutoff(app):
    """Press Update; while the NEW version is in `verifying`, take the finished-marker away from one of
    the undo copies (a copy cut off mid-way looks exactly like this). The undo must refuse to pour it
    back and HOLD with the new prefix."""
    s = load(app)
    w.say(f"=== {app}: a cut-off undo copy")
    done = {}

    def on_phase(ph):
        if ph == "verifying" and not done:
            out = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={app} | head -1); echo "copy: $c"
docker run --rm -v $c:/c alpine sh -c 'ls -la /c; rm -f /c/felhom-undo-complete; ls /c'""")
            done["out"] = out
            w.say("  >>> finished-marker removed from one copy:\n" + out)
        return False
    phases, st = press(app, poll=0.5, on_phase=on_phase)
    before = b.db_state(app)
    pl = page_lines(app)
    w.say("  PAGE (box language hu):", json.dumps(pl, ensure_ascii=False))
    set_box_language("en")
    pl_en = page_lines(app)
    w.say("  PAGE (box language en):", json.dumps(pl_en, ensure_ascii=False))
    set_box_language("hu")
    w.say("  copies kept:", w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app}"))
    s["cutoff_live"] = {"phases": phases, "end": {k: st.get(k) for k in ("update_phase", "hold_reason")}, "page_hu_box": pl, "page_en_box": pl_en, "marker": done}
    save(app, s)


def stage_fixprobe_and_press(app):
    """After an undo: the catalog fixes the new version's probe; a PERSON presses Update; it must work
    and end the undone note."""
    s = load(app)
    frm, to, p0, p1 = b.EDGE[app]
    fy = f"{w.DRILL}/templates/{app}/.felhom.yml"
    f = open(fy).read().replace(f"port: {p1}", f"port: {p0}", 1); open(fy, "w").write(f)
    w.sh(["git", "-C", w.DRILL, "commit", "-qam", f"DRILL {app}: the probe fixed (port {p0}) — the step is now good"])
    w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120)
    w.sync_rescan(app, to)
    time.sleep(20)
    w.ctl("POST", "/api/sync"); time.sleep(3); w.ctl("POST", "/api/stacks/rescan")
    w.say(f"=== {app}: a person presses Update again after the undo (probe fixed in the catalog)")
    w.say("  before, the page:", json.dumps(page_lines(app), ensure_ascii=False))
    phases, st = press(app)
    rb = readback(app, s)
    w.say(f"  READBACK A={rb['A']} B={rb['B']} C={rb['C']}  installed={w.observables(app)['installed_images']}")
    pl = page_lines(app); w.say("  after, the page:", json.dumps(pl, ensure_ascii=False))
    w.say("  app.yaml last_update_undone:", w.guest(f"grep -c last_update_undone /opt/docker/stacks/{app}/app.yaml"))
    s["manual_after_undo"] = {"phases": phases, "end": st.get("update_phase"), "readback": rb, "page": pl}
    save(app, s)


if __name__ == "__main__":
    w.login()
    {"undo": stage_undo, "powercut": stage_powercut, "cutoff": stage_cutoff,
     "fixpress": stage_fixprobe_and_press}[sys.argv[1]](sys.argv[2])