felhom.eu/documentation/audits/update-night-2026-09-21/phase3_legs.py

#!/usr/bin/env python3
"""Phase 3 — the bad days, legs B2..B8.  Usage: phase3_legs.py <leg>

Every leg records the SAME five things (the brief §6):
  what the household saw (BOTH languages) · what the box did by itself · time to steady ·
  which alarm or event fired and whether it was true · which should have fired and did not.

Nothing here is product code. Every act is a button a person can press.
"""
import json, os, re, subprocess, sys, time
from datetime import datetime

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import walk as w   # noqa: E402

BAD = os.path.join(HERE, "bad-days")


def out(leg):
    d = os.path.join(BAD, leg)
    os.makedirs(d, exist_ok=True)
    return d


def sentences(app):
    """The household's own sentences on the app page, both languages, ASCII-fragment matched
    (R-96 rule 8: Hungarian is searched by ASCII fragment, with a positive and a negative control)."""
    res = {}
    for lang, sfx in (("hu", ""), ("en", "?lang=en")):
        html = w.page(f"/apps/{app}{sfx}")
        # Strip <script> and <style> FIRST. Without this the page's inline JS becomes "sentences"
        # and the evidence is unreadable — measured on the nextcloud leg, fixed here.
        html = re.sub(r"<script.*?</script>", " ", html, flags=re.S)
        html = re.sub(r"<style.*?</style>", " ", html, flags=re.S)
        txt = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html))
        keep = [s.strip() for s in re.split(r"(?<=[.!?]) ", txt)
                if any(k in s.lower() for k in
                       ("friss", "update", "vissza", "restore", "ment", "backup",
                        "hib", "error", "megsz", "nem "))]
        res[lang] = keep[:12]
    return res


def events(n=40):
    for p in ("/api/events?limit=%d" % n, "/api/debug/dump"):
        code, d = w.ctl("GET", p)
        if code == "200":
            return {"path": p, "body": d}
    return {"path": None, "body": None}


def snap(app, label):
    st = w.stack(app)
    return {"label": label, "at": datetime.now().isoformat(timespec="seconds"),
            "state": st.get("state"), "updating": st.get("updating"),
            "update_phase": st.get("update_phase"), "update_error": st.get("update_error"),
            "hold_reason": st.get("hold_reason"), "observables": w.observables(app)}


# ---------------------------------------------------------------------------- B2
def b2():
    """B2 — the new tag cannot be pulled. Phase `pulling` fails; §6.1 says the pin AND the
    definition are PUT BACK and nothing ran."""
    APP, SUB = "bentopdf", "pdf"
    GOOD, GONE = "localhost:5000/drill/pdf:1.0.0", "localhost:5000/drill/pdf:1.0.1"
    d = out("B2-pull-fails")
    w.say("==== B2: the new tag cannot be pulled")
    st = w.stack(APP)
    if st.get("deployed"):
        w.remove(APP)
    for prev in ("ghcr.io/alam00000/bentopdf:v2.8.6", GONE, "ghcr.io/alam00000/bentopdf:v2.8.8"):
        if w.drill_bump(APP, prev, GOOD) is not None:
            break
    w.sync_rescan()
    if not w.deploy(APP, SUB):
        w.say("  bentopdf never came up on the drill image — B2 cannot run")
        return
    w.backup_now(APP)
    before = snap(APP, "before")
    w.drill_bump(APP, GOOD, GONE)
    w.sync_rescan()
    bdg = w.badges(APP)
    w.say(f"  badge HU {bdg['hu']}")
    t0 = time.time()
    res = w.press_update(APP)
    steady = round(time.time() - t0, 1)
    after = snap(APP, "after")
    # is the OLD version still serving? the household's own door
    rc, code, _ = w.app_curl(SUB, "/", timeout=20)
    serving = code in ("200", "302")
    # what does an unattended caller see on the wire?
    code2, ref = w.ctl("POST", f"/api/stacks/{APP}/update")
    w.say(f"  second press (to read the refusal on the wire): http={code2} {json.dumps(ref, ensure_ascii=False)[:300]}")
    rec = {"leg": "B2", "app": APP, "edge": f"{GOOD} -> {GONE} (tag absent from the store)",
           "badges": bdg, "phases": res, "before": before, "after": after,
           "old_version_still_serving": serving, "time_to_steady_s": steady,
           "sentences": sentences(APP), "second_press": {"http": code2, "body": ref},
           "events": events()}
    json.dump(rec, open(f"{d}/result.json", "w"), indent=2, ensure_ascii=False)
    open(f"{d}/log.txt", "w").write("\n".join(w.LOG) + "\n")
    w.say(f"  phases={[p['phase'] for p in res['phases']]} error={after['update_error']!r} "
          f"hold={after['hold_reason']!r} old_serving={serving}")
    w.say(f"  pinned after = {after['observables']['pinned_images']}")
    w.say(f"  compose after= {after['observables']['live_compose_image_lines']}")


# ---------------------------------------------------------------------------- B3
def b3(app="bentopdf"):
    """B3 — Update pressed WHILE a backup runs. Expect reason `busy`, then a later pass gets in."""
    d = out("B3-busy-during-backup")
    w.say("==== B3: Update pressed while a backup is running")
    code, msg = w.ctl("POST", "/api/backup/run")
    w.say(f"  „Mentés most\" -> {code} {str(msg)[:120]}")
    time.sleep(1)
    tries = []
    for i in range(6):
        c, b = w.ctl("POST", f"/api/stacks/{app}/update")
        reason = (b.get("data") or {}).get("reason") if isinstance(b, dict) else None
        tries.append({"i": i, "http": c, "reason": reason,
                      "error": b.get("error") if isinstance(b, dict) else None})
        w.say(f"  press {i}: http={c} reason={reason!r} :: "
              f"{(b.get('error') if isinstance(b,dict) else '') or ''}")
        if c != "409":
            break
        time.sleep(3)
    # then wait the backup out and press once more — the caller's "retry next pass"
    for _ in range(90):
        time.sleep(5)
        _, s = w.ctl("GET", "/api/backup/status")
        if not (s.get("data") or {}).get("running", False):
            break
    c, b = w.ctl("POST", f"/api/stacks/{app}/update")
    after_reason = (b.get("data") or {}).get("reason") if isinstance(b, dict) else None
    w.say(f"  after the backup finished: http={c} reason={after_reason!r} "
          f":: {(b.get('error') if isinstance(b,dict) else '') or ''}")
    # If it was accepted, FOLLOW it to the end. Leaving an update in flight would make the next
    # leg's first press read `updating` and measure this leg by accident.
    ran = None
    if c in ("200", "202"):
        t0 = time.time()
        while time.time() - t0 < 1200:
            st = w.stack(app)
            if not st.get("updating") and st.get("update_phase") in ("done", "failed"):
                ran = {"final_phase": st.get("update_phase"),
                       "update_error": st.get("update_error"),
                       "hold_reason": st.get("hold_reason"),
                       "seconds": round(time.time() - t0, 1),
                       "observables": w.observables(app)}
                break
            time.sleep(2)
        w.say(f"  the update that got through ended: {ran}")
    rec = {"leg": "B3", "app": app, "during_backup": tries,
           "after_backup": {"http": c, "reason": after_reason, "body": b},
           "the_update_that_got_through": ran,
           "sentences": sentences(app)}
    json.dump(rec, open(f"{d}/result.json", "w"), indent=2, ensure_ascii=False)
    open(f"{d}/log.txt", "w").write("\n".join(w.LOG) + "\n")


# ---------------------------------------------------------------------------- B4
def b4(apps):
    """B4 — two Updates within one second, then five. Single-flight, or do they run together?"""
    d = out("B4-concurrent-updates")
    w.say(f"==== B4: concurrent Updates on {apps}")
    import concurrent.futures as cf
    results = {}
    with cf.ThreadPoolExecutor(max_workers=len(apps)) as ex:
        fut = {ex.submit(w.ctl, "POST", f"/api/stacks/{a}/update"): a for a in apps}
        for f in cf.as_completed(fut):
            a = fut[f]
            c, b = f.result()
            results[a] = {"http": c, "reason": (b.get("data") or {}).get("reason")
                          if isinstance(b, dict) else None, "body": b}
            w.say(f"  {a}: http={c} reason={results[a]['reason']!r}")
    mem = w.guest("free -m | head -2; docker stats --no-stream --format "
                  "'{{.Name}} {{.MemUsage}}' | head -20")
    w.say("  memory during: " + " | ".join(mem.split("\n")[:3]))
    states = {}
    t0 = time.time()
    while time.time() - t0 < 1200:
        states = {a: w.stack(a) for a in apps}
        if all(not s.get("updating") for s in states.values()):
            break
        time.sleep(3)
    fin = {a: {"state": s.get("state"), "update_phase": s.get("update_phase"),
               "update_error": s.get("update_error"), "hold_reason": s.get("hold_reason"),
               "observables": w.observables(a)} for a, s in states.items()}
    for a, v in fin.items():
        w.say(f"  {a} ended phase={v['update_phase']} err={v['update_error']!r} hold={v['hold_reason']!r}")
    json.dump({"leg": "B4", "apps": apps, "presses": results, "memory_during": mem,
               "final": fin, "elapsed_s": round(time.time() - t0, 1)},
              open(f"{d}/result.json", "w"), indent=2, ensure_ascii=False)
    open(f"{d}/log.txt", "w").write("\n".join(w.LOG) + "\n")


# ---------------------------------------------------------------------------- B7
def b7(app="bentopdf"):
    """B7 — the 2 GB disk floor and the memory refusal. Both refuse BEFORE anything is recorded
    (§6.1 phase 0), so nothing moves — which is what makes them safe to probe at all."""
    d = out("B7-disk-and-memory-refusals")
    w.say("==== B7: the disk floor and the memory refusal")
    free_before = w.guest("df -h /var/lib/felhom | tail -1; df -B1 --output=avail /var/lib/felhom | tail -1")
    w.say(f"  free before: {' '.join(free_before.split())}")
    # eat the space down under the 2 GB floor with ONE file, on the scratch guest only
    fill = w.guest("""
AVAIL=$(df -B1 --output=avail /var/lib/felhom | tail -1 | tr -d ' ')
KEEP=1500000000                       # leave ~1.5 GB, i.e. UNDER the fixed 2 GB floor
EAT=$((AVAIL - KEEP))
echo "avail=$AVAIL eat=$EAT"
if [ "$EAT" -gt 0 ]; then fallocate -l "$EAT" /var/lib/felhom/DRILL-FILL.bin && echo filled; fi
df -h /var/lib/felhom | tail -1
""", timeout=600)
    w.say("  " + " | ".join(x for x in fill.split("\n") if x.strip()))
    c, b = w.ctl("POST", f"/api/stacks/{app}/update")
    reason = (b.get("data") or {}).get("reason") if isinstance(b, dict) else None
    w.say(f"  Update under the floor: http={c} reason={reason!r} :: "
          f"{(b.get('error') if isinstance(b,dict) else '') or ''}")
    snap_low = snap(app, "under-the-floor")
    sent = sentences(app)
    # english refusal too — the 409 body is localised by errText since v0.260.0
    c_en, b_en = w.ctl("POST", f"/api/stacks/{app}/update?lang=en")
    w.say(f"  refusal EN: http={c_en} :: {(b_en.get('error') if isinstance(b_en,dict) else '') or ''}")
    freed = w.guest("rm -f /var/lib/felhom/DRILL-FILL.bin; sync; df -h /var/lib/felhom | tail -1")
    w.say(f"  freed: {' '.join(freed.split())}")
    json.dump({"leg": "B7", "app": app, "free_before": free_before, "fill": fill,
               "refusal_hu": {"http": c, "reason": reason, "body": b},
               "refusal_en": {"http": c_en, "body": b_en},
               "state_under_floor": snap_low, "sentences": sent, "freed": freed},
              open(f"{d}/result.json", "w"), indent=2, ensure_ascii=False)
    open(f"{d}/log.txt", "w").write("\n".join(w.LOG) + "\n")


if __name__ == "__main__":
    w.login()
    leg = sys.argv[1]
    if leg == "b2":
        b2()
    elif leg == "b3":
        b3(*sys.argv[2:])
    elif leg == "b4":
        b4(sys.argv[2].split(","))
    elif leg == "b7":
        b7(*sys.argv[2:])
    else:
        sys.exit(f"unknown leg {leg}")