diff --git a/documentation/audits/DRILL-night-2026-09-24.md b/documentation/audits/DRILL-night-2026-09-24.md new file mode 100644 index 00000000..6d73505d --- /dev/null +++ b/documentation/audits/DRILL-night-2026-09-24.md @@ -0,0 +1,50 @@ +# DRILL — night 2026-09-24 (run 2026-09-24 from 11:07 CEST): the second drive brings a file app back whole; a crash loop is stopped; exact image fingerprints; automatic updates spiked; a chaos hour + +> Written in parts. The "Not done, or changed" section and the results are filled at the end; **the +> Part E schedule below was written and committed BEFORE round 1.** + +## Part E — the schedule (committed before round 1) + +Drawn by `night-2026-09-24/tools/chaos_schedule.py`, seed **20260924** (constraints: every action at least +once; controller killed at most 3 times, ≥ 20 min apart; Start-after-unhealthy after a stop; Remove-on-held +after a hold). Guest **9202** (scratch), controller v0.269.1, drill catalog. + +seed 20260924, 12 rounds +| round | action | accident | +|---|---|---| +| 1 | oom_app | power_cut | +| 2 | install | docker_restart | +| 3 | cutoff_undo_file_app | power_cut | +| 4 | failing_step | controller_killed | +| 5 | remove_held | none | +| 6 | failing_step | whole_box_backup | +| 7 | crash_loop | whole_box_backup | +| 8 | oom_app | disk_to_floor_plus_1g | +| 9 | remove_held | controller_killed | +| 10 | ladder_step | docker_restart | +| 11 | cutoff_undo_file_app | disk_to_floor_plus_1g | +| 12 | start_after_unhealthy | none | + +**The app each round acts on — fixed here, not drawn** (an app's state after round N decides what N+1 can do): + +| round | app | why | +|---|---|---| +| 1 oom_app | `chaosoom` (drill-only: alpine, 32M cap, a 200 MB eater in a loop) | out-of-memory storm → the box must stop it | +| 2 install | n8n at its ladder's FIRST `from` (2 steps behind), seeded through its front door | gives round 10 a step | +| 3 cutoff_undo_file_app | nextcloud (the file app), a failing step (redis 7-alpine → 7.4.0-alpine, probe 8999), the NEWEST undo copy's marker cut at verifying → the hold must name the second drive → its whole restore, then the seeds + files read back | decision 26 under a power cut | +| 4 failing_step | wishlist v0.67.1 → `:latest` with probe 8999 → undo | the undo under a controller kill | +| 5 remove_held | gokapi (held since A3: the box stopped its crash loop) — Remove with data | Remove on a held app | +| 6 failing_step | navidrome 0.64.1 → `:latest` with probe 8999 → undo | the undo under a backup run | +| 7 crash_loop | `chaoscrash` (drill-only: alpine that exits after 3 s) | the box must stop it | +| 8 oom_app | `chaosoomb` (a second eater) | the OOM stop under a full disk | +| 9 remove_held | `chaoscrash` (held since round 7) | Remove on a held app under a controller kill | +| 10 ladder_step | n8n, one step (2 → 1) | a good step under a docker restart | +| 11 cutoff_undo_file_app | nextcloud again | decision 26 with the disk 1 GB above the floor (a `disk` refusal is a valid outcome) | +| 12 start_after_unhealthy | `chaosoomb` (held since round 8) — Start → one more try → the repeat stop says support is told | decision 28's second half | + +**Changed before round 1:** the accident "a whole-box backup started on 9202" is the box's own full backup run +(`POST /api/backup/run`, as on 2026-09-23), not a vzdump: demo-hp's backup storage had ~4 GB free and the host +root was at 90 %, so a vzdump of 9202 could have filled the HOST (demo-hp also had R-672's full pool today). +**How accidents are fired:** the runner arms the round and waits; the session fires each host-level accident +as its own visible command (power cut = `pct stop 9202; pct start 9202`, kill -9 of the controller PID, +`systemctl restart docker` in the guest, a `fallocate` to leave 3 GB free on `/var/lib/docker`). diff --git a/documentation/audits/night-2026-09-24/tools/chaos.py b/documentation/audits/night-2026-09-24/tools/chaos.py new file mode 100644 index 00000000..8ea6736e --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/chaos.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +"""chaos.py — Part E, the chaos hour on tonight's work. Guest 9202 (scratch), controller v0.269.1, drill catalog. + + python3 chaos.py setup # seed wishlist + navidrome through their front doors; note nextcloud's seeds + python3 chaos.py round # one round of chaos_schedule.py (seed 20260924), evidence written as it happens + +EVIDENCE, NOT PRODUCT. Every ACT is the endpoint the UI invokes. The ACCIDENT is NOT fired by this script: +at the moment the round is armed it writes `E/arm-NN.json` and waits for `E/done-NN.json`, and the session +fires the accident as its own visible command (power cut = pct stop/start 9202; controller kill = kill -9 of +the controller's PID; docker restart; disk fill). The one exception is the backup accident, which is a +product endpoint (`POST /api/backup/run`) and is pressed here. + +CHANGED BEFORE ROUND 1 (said in the findings doc): the "whole-box backup" accident is the box's own full +backup run, not a vzdump — demo-hp's backup storage (`local`) had ~4 GB free and the host root was 90 % +full, so a vzdump of 9202 would have filled the HOST. + +The five things per round: the household's page in both languages, what the box did (its own log), time to +steady, which events fired (9202 has no hub: R-620's DROPPED lines are the witness), and — judged in the +findings doc against `08` — what should have fired and did not. Plus the data read back. +""" +import html as H, json, os, re, sys, threading, time +sys.path.insert(0, ".") +import walk as w +import fixtures as fx +import ncfiles as nf +from chaos_schedule import schedule + +HERE = os.path.dirname(os.path.abspath(__file__)) +EVD = os.path.join(HERE, "..", "E") +os.makedirs(EVD, exist_ok=True) +STATE = os.path.join(EVD, "chaos-state.json") + +TARGET = {1: "chaosoom", 2: "n8n", 3: "nextcloud", 4: "wishlist", 5: "gokapi", 6: "navidrome", 7: "chaoscrash", + 8: "chaosoomb", 9: "chaoscrash", 10: "n8n", 11: "nextcloud", 12: "chaosoomb"} +FAIL = { # failing steps: a real image move + the drill probe on a port the app does not answer + "wishlist": ("ghcr.io/cmintey/wishlist:v0.67.1", "ghcr.io/cmintey/wishlist:latest", 3000), + "navidrome": ("deluan/navidrome:0.64.1", "deluan/navidrome:latest", 4533), + "nextcloud": ("redis:7-alpine", "redis:7.4.0-alpine", 80), +} +FX = {"wishlist": fx.FIXTURES["wishlist"], "navidrome": fx.FIXTURES["navidrome"], "n8n": fx.FIXTURES["n8n"], + "nextcloud": fx.Nextcloud()} +SUB = {"wishlist": "c-wish", "navidrome": "c-navi", "n8n": "e-n8n", "nextcloud": "cloud-a1"} + + +def load(): + return json.load(open(STATE)) if os.path.exists(STATE) else {} + + +def save(s): + json.dump(s, open(STATE, "w"), indent=2, ensure_ascii=False) + + +# ------------------------------------------------------------------------------------------ observing +def household(app): + st = w.stack(app) + out = {"state": {k: st.get(k) for k in ("deployed", "state", "update_phase", "hold_reason", "hold_kind", + "update_error", "ladder_steps_left")}} + if not st.get("deployed"): + return out + out["badges"] = w.badges(app) + for lang in ("hu", "en"): + h = H.unescape(w.page(f"/apps/{app}?lang={lang}")) + lines = [] + for m in re.finditer(r'
]*)>(.*?)
', h, re.S): + t = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", m.group(2))).strip() + if t and "' +" not in t and "nem visszavonható" not in t and "cannot undo" not in t \ + and "Hub kapcsolat" not in t and "hub connection" not in t: + lines.append(t[:400]) + out[f"page_{lang}"] = lines + return out + + +def events_since(t_iso): + code, d = w.ctl("GET", "/api/debug/logs?level=DEBUG&lines=8000") + ents = ((d.get("data") or {}).get("entries") or []) if isinstance(d, dict) else [] + out = [] + for e in ents: + m = re.search(r"DROPPED event (\S+) \(severity (\w+)\)", e.get("message", ""), re.I) + if m and e.get("timestamp", "") >= t_iso: + out.append(f"{e['timestamp']} {m.group(1)} ({m.group(2)})") + return out + + +def box_log(t_iso, app): + code, d = w.ctl("GET", "/api/debug/logs?level=INFO&lines=5000") + ents = ((d.get("data") or {}).get("entries") or []) if isinstance(d, dict) else [] + keys = (app, "UNDO", "undo", "HELD", "hold", "restore", "Restore", "deadapp", "STOPPED by the box", + "floor", "disk", "space", "backup") + return [f"{e['timestamp']} {e.get('level')} {e.get('message', '')[:300]}" for e in ents + if e.get("timestamp", "") >= t_iso and any(k in e.get("message", "") for k in keys)][-80:] + + +def readback(app, s): + if app not in FX or not w.stack(app).get("deployed"): + return None + A = (s.get("apps", {}).get(app) or {}).get("A") + if app == "nextcloud": + S = json.load(open(f"{w.SC}/a1-secret-state.json")) + A = S["A"] + w.wait_app(SUB[app], "/status.php", want=("200",), tries=90, delay=4) + names = sorted(S["files"]) + return {"A": bool(FX[app].verify(w, SUB[app], A, w.say)), + "files": nf.verify_files(w, SUB[app], A, {names[0]: S["files"][names[0]], names[2]: S["files"][names[2]]}, w.say)} + if not A: + return {"A": None, "note": "no seed recorded"} + w.wait_app(SUB[app], "/", tries=90, delay=4) + return {"A": bool(FX[app].verify(w, SUB[app], A, w.say))} + + +def steady(app, t0, cap=1500, want_deployed=True, until=None): + last = None + while time.time() - t0 < cap: + try: + st = w.stack(app) + except Exception: + st = {} + cur = (st.get("deployed"), st.get("state"), st.get("updating"), st.get("update_phase"), st.get("hold_kind")) + if cur != last: + w.say(f" +{time.time()-t0:6.1f}s {app}: deployed={cur[0]} state={cur[1]} updating={cur[2]} phase={cur[3]} hold={cur[4]}") + last = cur + if until: + if until(st): + return round(time.time() - t0, 1), st + elif st and not st.get("updating") and not st.get("deploying"): + if (want_deployed and st.get("deployed") and st.get("state") in ("running", "stopped", "unhealthy", "degraded")) \ + or (not want_deployed and not st.get("deployed")) or st.get("hold_kind"): + return round(time.time() - t0, 1), st + time.sleep(2) + return None, w.stack(app) + + +# ------------------------------------------------------------------------------------------ the drill +def drill(edit, msg): + import fcntl + with open(f"{w.SC}/drill.lock", "w") as lk: + fcntl.flock(lk, fcntl.LOCK_EX) + w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) + edit() + w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-E " + msg]) + w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) + w.say(" drill: " + msg) + + +def break_step(app): + frm, to, port = FAIL[app] + comp, fy = f"{w.DRILL}/templates/{app}/docker-compose.yml", f"{w.DRILL}/templates/{app}/.felhom.yml" + + def edit(): + c = open(comp).read() + assert f"image: {frm}" in c, (app, frm) + open(comp, "w").write(c.replace(f"image: {frm}", f"image: {to}", 1)) + f = open(fy).read() + f2 = re.sub(r"(healthcheck:\n(?:.*\n){0,8}?\s+port: )" + str(port) + r"\b", r"\g<1>8999", f, count=1) + assert f2 != f, (app, "probe port not found") + open(fy, "w").write(f2) + drill(edit, f"{app}: failing step {frm} -> {to} + probe 8999") + + +def repair_step(app): + frm, to, port = FAIL[app] + comp, fy = f"{w.DRILL}/templates/{app}/docker-compose.yml", f"{w.DRILL}/templates/{app}/.felhom.yml" + + def edit(): + c = open(comp).read() + open(comp, "w").write(c.replace(f"image: {to}", f"image: {frm}", 1)) + f = open(fy).read() + open(fy, "w").write(re.sub(r"(healthcheck:\n(?:.*\n){0,8}?\s+port: )8999\b", r"\g<1>" + str(port), f, count=1)) + drill(edit, f"{app}: back to {frm} and its real probe") + + +# ------------------------------------------------------------------------------------------ the arm +class Arm: + def __init__(self, n, kind, note): + self.n, self.kind, self.note, self.armed = n, kind, note, False + + def __call__(self, why): + if self.armed or self.kind == "none": + return + self.armed = True + self.note["armed_at"] = time.strftime("%H:%M:%S") + self.note["armed_why"] = why + w.say(f" >>> ARMED accident {self.kind} ({why})") + if self.kind == "whole_box_backup": + code, d = w.ctl("POST", "/api/backup/run") + self.note["accident"] = f"POST /api/backup/run -> {code} {str(d)[:160]} at {time.strftime('%H:%M:%S')}" + w.say(" >>> " + self.note["accident"]) + return + json.dump({"round": self.n, "kind": self.kind, "at": time.time(), "why": why}, + open(os.path.join(EVD, f"arm-{self.n:02d}.json"), "w")) + + def wait_done(self, cap=900): + if self.kind in ("none", "whole_box_backup") or not self.armed: + return + p = os.path.join(EVD, f"done-{self.n:02d}.json") + t0 = time.time() + while time.time() - t0 < cap and not os.path.exists(p): + time.sleep(2) + self.note["accident"] = json.load(open(p)) if os.path.exists(p) else "NOT FIRED (no done file)" + w.say(f" >>> accident done: {self.note['accident']}") + + +def wait_controller(cap=900): + t0 = time.time() + while time.time() - t0 < cap: + try: + w.login() + code, _ = w.ctl("GET", "/api/stacks") + if code == "200": + return round(time.time() - t0, 1) + except SystemExit: + pass + time.sleep(5) + return None + + +# ------------------------------------------------------------------------------------------ actions +def a_unhealthy_install(app, note, arm, t0): + """crash_loop / oom_app: install the drill app through the product; the box must stop it by itself.""" + vals = w.deploy_values(app, app) + code, d = w.ctl("POST", f"/api/stacks/{app}/deploy", {"values": vals}) + note["act"] = f"deploy -> {code}" + threading.Timer(20, arm, args=("20 s after the install",)).start() + return steady(app, t0, cap=1500, until=lambda st: st.get("hold_kind") == "unhealthy_stop") + + +def a_install(app, note, arm, t0, s): + """n8n at the ladder's FIRST from, so round 10 has a step to climb.""" + sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts") + import ladder + comp = f"{w.DRILL}/templates/{app}/docker-compose.yml" + E, _, _ = ladder.parse(open(f"{w.DRILL}/templates/{app}/.felhom.yml").read()) + head = open(comp).read() + s.setdefault("heads", {})[app] = head + save(s) + + def edit(): + c = head + for svc, ref in ladder.images_in(head).items(): + if E[0]["from"].get(svc) and E[0]["from"][svc] != ref: + c = c.replace(f"image: {ref}", f"image: {E[0]['from'][svc]}", 1) + open(comp, "w").write(c) + drill(edit, f"{app} at the ladder's first from {E[0]['from']}") + w.sync_rescan() + time.sleep(3) + w.sync_rescan() + vals = w.deploy_values(app, SUB[app]) + threading.Timer(20, arm, args=("20 s after the install press",)).start() + code, d = w.ctl("POST", f"/api/stacks/{app}/deploy", {"values": vals}) + note["act"] = f"deploy -> {code} {str(d)[:100]}" + secs, st = steady(app, t0, until=lambda st: st.get("deployed") and (st.get("app_config") or {}).get("pinned_images") + and not st.get("deploying")) + if st.get("deployed"): + w.wait_app(SUB[app], "/", tries=90, delay=4) + A = FX[app].seed(w, SUB[app], w.say) + s.setdefault("apps", {}).setdefault(app, {})["A"] = A + save(s) + + def back(): + open(comp, "w").write(head) + drill(back, f"{app} back to the head (2 steps offered)") + w.sync_rescan() + return secs, st + + +def a_update(app, note, arm, t0, on_verify=None): + code, d = w.ctl("POST", f"/api/stacks/{app}/update") + note["act"] = f"Update -> {code} {str(d)[:140]}" + w.say(" " + note["act"]) + phases, seen = [], None + while time.time() - t0 < 1800: + try: + st = w.stack(app) + except Exception: + st = {} + ph = st.get("update_phase") + if ph != seen and ph: + seen = ph + phases.append((round(time.time() - t0, 1), ph)) + w.say(f" +{phases[-1][0]:6.1f}s phase={ph}") + if ph == "verifying": + if on_verify: + on_verify() + arm("at verifying") + if ph == "starting": + arm("at starting") + if st and not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - t0 > 3: + break + if not st and time.time() - t0 > 3: + time.sleep(5) + time.sleep(0.5) + note["phases"] = phases + return steady(app, t0) + + +def cut_newest_copy(app, note): + out = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={app} | sort | tail -1); echo "copy: $c" +docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c | head -3'""") + note["cutoff"] = out.strip() + w.say(" >>> finished-marker removed from the NEWEST undo copy:\n" + out) + + +def a_cutoff_file_app(app, note, arm, t0, s): + break_step(app) + note["badge_catchup_s"] = w.sync_rescan(expect_app=app, expect_ref=FAIL[app][1], tries=60, delay=5) + secs, st = a_update(app, note, arm, t0, on_verify=lambda: cut_newest_copy(app, note)) + repair_step(app) + w.sync_rescan() + note["held"] = {k: st.get(k) for k in ("update_phase", "hold_kind", "hold_no_whole_copy", "hold_reason")} + note["held_page"] = household(app) + if st.get("hold_kind") == "update_failed" and not st.get("hold_no_whole_copy"): + sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip() + csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip() + r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST", + "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={app}", f"{w.BASE}/backup/tier2/unit-restore"], timeout=120) + note["whole_restore_press"] = r.stdout + w.say(f" whole restore (the second drive) -> {r.stdout}") + last = None + for _ in range(600): + c2, dd = w.ctl("GET", "/api/backup/restore-status") + d2 = (dd.get("data") or {}) if isinstance(dd, dict) else {} + if not d2.get("running") and d2.get("last"): + last = d2.get("last") + break + time.sleep(2) + note["whole_restore"] = last + w.say(f" whole restore: {str(last)[:300]}") + return steady(app, t0) + return secs, st + + +def a_failing_step(app, note, arm, t0, s): + break_step(app) + note["badge_catchup_s"] = w.sync_rescan(expect_app=app, expect_ref=FAIL[app][1], tries=60, delay=5) + r = a_update(app, note, arm, t0) + repair_step(app) + w.sync_rescan() + return r + + +def a_remove_held(app, note, arm, t0): + st = w.stack(app) + note["held_before"] = {k: st.get(k) for k in ("deployed", "hold_kind", "hold_reason")} + threading.Timer(2, arm, args=("2 s after the remove press",)).start() + code, d = w.ctl("POST", f"/api/stacks/{app}/remove", {"remove_hdd_data": True, "remove_backups": True}) + note["act"] = f"remove (data + backups) -> {code} {str(d)[:200]}" + w.say(" " + note["act"]) + return steady(app, t0, want_deployed=False, cap=600) + + +def a_ladder_step(app, note, arm, t0, s): + w.sync_rescan() + st = w.stack(app) + note["steps_left_before"] = st.get("ladder_steps_left") + note["pinned_before"] = (st.get("app_config") or {}).get("pinned_images") + r = a_update(app, note, arm, t0) + w.sync_rescan() + st = w.stack(app) + note["steps_left_after"] = st.get("ladder_steps_left") + note["pinned_after"] = (st.get("app_config") or {}).get("pinned_images") + return r + + +def a_start_after_unhealthy(app, note, arm, t0): + st = w.stack(app) + note["before"] = {k: st.get(k) for k in ("state", "hold_kind", "hold_reason")} + code, d = w.ctl("POST", f"/api/stacks/{app}/start") + note["act"] = f"Start -> {code} {str(d)[:140]}" + w.say(" " + note["act"]) + threading.Timer(5, arm, args=("5 s after Start",)).start() + time.sleep(3) + note["after_start"] = {k: w.stack(app).get(k) for k in ("state", "hold_kind")} + return steady(app, t0, cap=1200, until=lambda st: st.get("hold_kind") == "unhealthy_stop") + + +# ------------------------------------------------------------------------------------------ stages +def setup(): + s = load() + w.login() + for app in ("wishlist", "navidrome"): + w.wait_app(SUB[app], "/", tries=60, delay=3) + A = FX[app].seed(w, SUB[app], w.say) + s.setdefault("apps", {}).setdefault(app, {})["A"] = A + w.say(f" seeded {app}: {bool(A)}") + save(s) + s["setup_readback"] = {a: readback(a, s) for a in ("wishlist", "navidrome", "nextcloud")} + w.say("setup readback: " + json.dumps(s["setup_readback"])) + save(s) + open(os.path.join(EVD, "00-setup.log"), "w").write("\n".join(w.LOG) + "\n") + + +def run_round(n): + s = load() + sched = {r["round"]: r for r in schedule()}[n] + app = TARGET[n] + w.login() + t_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + w.say(f"==== ROUND {n}: {sched['action']} on {app} x {sched['accident']} ({t_iso})") + note = {"round": n, **sched, "app": app, "started": t_iso, "before": household(app)} + arm = Arm(n, sched["accident"], note) + t0 = time.time() + act = sched["action"] + try: + if act in ("oom_app", "crash_loop"): + secs, st = a_unhealthy_install(app, note, arm, t0) + elif act == "install": + secs, st = a_install(app, note, arm, t0, s) + elif act == "cutoff_undo_file_app": + secs, st = a_cutoff_file_app(app, note, arm, t0, s) + elif act == "failing_step": + secs, st = a_failing_step(app, note, arm, t0, s) + elif act == "remove_held": + secs, st = a_remove_held(app, note, arm, t0) + elif act == "ladder_step": + secs, st = a_ladder_step(app, note, arm, t0, s) + elif act == "start_after_unhealthy": + secs, st = a_start_after_unhealthy(app, note, arm, t0) + else: + raise SystemExit("unknown action " + act) + except Exception as e: + note["act_exception"] = repr(e) + w.say(" ACT EXCEPTION " + repr(e)) + secs, st = None, {} + arm.wait_done() + if sched["accident"] in ("power_cut", "docker_restart", "controller_killed"): + note["controller_back_s"] = wait_controller() + if act in ("oom_app", "crash_loop", "start_after_unhealthy"): + secs2, st = steady(app, t0, cap=1800, until=lambda st: st.get("hold_kind") == "unhealthy_stop") + else: + secs2, st = steady(app, t0, want_deployed=(act != "remove_held")) + secs = secs2 or secs + note["time_to_steady_s"] = round(time.time() - t0, 1) if secs is None else secs + note["after"] = household(app) + if not w.stack(app).get("deployed"): + note["after"]["leftovers"] = w.guest(f"docker ps -a --filter label=com.docker.compose.project={app} --format '{{{{.Names}}}}'; docker volume ls -q | grep '^{app}_' || true; ls -d /opt/docker/stacks/{app}/applied-meta 2>/dev/null || true").strip() + note["readback"] = readback(app, s) + note["events"] = events_since(t_iso) + note["box_log"] = box_log(t_iso, app) + note["undo_copies_left"] = w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip() + open(os.path.join(EVD, f"round-{n:02d}-controller.log"), "w").write( + w.guest(f"docker logs --since {t_iso} felhom-controller 2>&1 | grep -vE 'auth: valid session|router.go:81|Status refresh' | tail -3000")) + json.dump(note, open(os.path.join(EVD, f"round-{n:02d}.json"), "w"), indent=2, ensure_ascii=False, default=str) + open(os.path.join(EVD, f"round-{n:02d}.log"), "w").write("\n".join(w.LOG) + "\n") + w.say(f"==== ROUND {n} END: steady in {note['time_to_steady_s']}s; readback={note['readback']}; events={note['events']}") + + +if __name__ == "__main__": + if sys.argv[1] == "setup": + setup() + elif sys.argv[1] == "round": + run_round(int(sys.argv[2]))