admin
3208cb2083
gates / gates (push) Successful in 24s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
451 lines
20 KiB
Python
451 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""chaos.py — Part E, the chaos hour on tonight's work. Guest 9202 (scratch), controller v0.269.1, drill catalog.
|
|
|
|
python3 chaos.py setup # seed wishlist + navidrome through their front doors; note nextcloud's seeds
|
|
python3 chaos.py round <N> # one round of chaos_schedule.py (seed 20260924), evidence written as it happens
|
|
|
|
EVIDENCE, NOT PRODUCT. Every ACT is the endpoint the UI invokes. The ACCIDENT is NOT fired by this script:
|
|
at the moment the round is armed it writes `E/arm-NN.json` and waits for `E/done-NN.json`, and the session
|
|
fires the accident as its own visible command (power cut = pct stop/start 9202; controller kill = kill -9 of
|
|
the controller's PID; docker restart; disk fill). The one exception is the backup accident, which is a
|
|
product endpoint (`POST /api/backup/run`) and is pressed here.
|
|
|
|
CHANGED BEFORE ROUND 1 (said in the findings doc): the "whole-box backup" accident is the box's own full
|
|
backup run, not a vzdump — demo-hp's backup storage (`local`) had ~4 GB free and the host root was 90 %
|
|
full, so a vzdump of 9202 would have filled the HOST.
|
|
|
|
The five things per round: the household's page in both languages, what the box did (its own log), time to
|
|
steady, which events fired (9202 has no hub: R-620's DROPPED lines are the witness), and — judged in the
|
|
findings doc against `08` — what should have fired and did not. Plus the data read back.
|
|
"""
|
|
import html as H, json, os, re, sys, threading, time
|
|
sys.path.insert(0, ".")
|
|
import walk as w
|
|
import fixtures as fx
|
|
import ncfiles as nf
|
|
from chaos_schedule import schedule
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
EVD = os.path.join(HERE, "..", "E")
|
|
os.makedirs(EVD, exist_ok=True)
|
|
STATE = os.path.join(EVD, "chaos-state.json")
|
|
|
|
TARGET = {1: "chaosoom", 2: "n8n", 3: "nextcloud", 4: "wishlist", 5: "gokapi", 6: "navidrome", 7: "chaoscrash",
|
|
8: "chaosoomb", 9: "chaoscrash", 10: "n8n", 11: "nextcloud", 12: "chaosoomb"}
|
|
FAIL = { # failing steps: a real image move + the drill probe on a port the app does not answer
|
|
"wishlist": ("ghcr.io/cmintey/wishlist:v0.67.1", "ghcr.io/cmintey/wishlist:latest", 3000),
|
|
"navidrome": ("deluan/navidrome:0.64.1", "deluan/navidrome:latest", 4533),
|
|
"nextcloud": ("redis:7-alpine", "redis:7.4.0-alpine", 80),
|
|
}
|
|
FX = {"wishlist": fx.FIXTURES["wishlist"], "navidrome": fx.FIXTURES["navidrome"], "n8n": fx.FIXTURES["n8n"],
|
|
"nextcloud": fx.Nextcloud()}
|
|
SUB = {"wishlist": "c-wish", "navidrome": "c-navi", "n8n": "e-n8n", "nextcloud": "cloud-a1"}
|
|
|
|
|
|
def load():
|
|
return json.load(open(STATE)) if os.path.exists(STATE) else {}
|
|
|
|
|
|
def save(s):
|
|
json.dump(s, open(STATE, "w"), indent=2, ensure_ascii=False)
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------ observing
|
|
def household(app):
|
|
st = w.stack(app)
|
|
out = {"state": {k: st.get(k) for k in ("deployed", "state", "update_phase", "hold_reason", "hold_kind",
|
|
"update_error", "ladder_steps_left")}}
|
|
if not st.get("deployed"):
|
|
return out
|
|
out["badges"] = w.badges(app)
|
|
for lang in ("hu", "en"):
|
|
h = H.unescape(w.page(f"/apps/{app}?lang={lang}"))
|
|
lines = []
|
|
for m in re.finditer(r'<div class="alert[^"]*"([^>]*)>(.*?)</div>', h, re.S):
|
|
t = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", m.group(2))).strip()
|
|
if t and "' +" not in t and "nem visszavonható" not in t and "cannot undo" not in t \
|
|
and "Hub kapcsolat" not in t and "hub connection" not in t:
|
|
lines.append(t[:400])
|
|
out[f"page_{lang}"] = lines
|
|
return out
|
|
|
|
|
|
def events_since(t_iso):
|
|
code, d = w.ctl("GET", "/api/debug/logs?level=DEBUG&lines=8000")
|
|
ents = ((d.get("data") or {}).get("entries") or []) if isinstance(d, dict) else []
|
|
out = []
|
|
for e in ents:
|
|
m = re.search(r"DROPPED event (\S+) \(severity (\w+)\)", e.get("message", ""), re.I)
|
|
if m and e.get("timestamp", "") >= t_iso:
|
|
out.append(f"{e['timestamp']} {m.group(1)} ({m.group(2)})")
|
|
return out
|
|
|
|
|
|
def box_log(t_iso, app):
|
|
code, d = w.ctl("GET", "/api/debug/logs?level=INFO&lines=5000")
|
|
ents = ((d.get("data") or {}).get("entries") or []) if isinstance(d, dict) else []
|
|
keys = (app, "UNDO", "undo", "HELD", "hold", "restore", "Restore", "deadapp", "STOPPED by the box",
|
|
"floor", "disk", "space", "backup")
|
|
return [f"{e['timestamp']} {e.get('level')} {e.get('message', '')[:300]}" for e in ents
|
|
if e.get("timestamp", "") >= t_iso and any(k in e.get("message", "") for k in keys)][-80:]
|
|
|
|
|
|
def readback(app, s):
|
|
if app not in FX or not w.stack(app).get("deployed"):
|
|
return None
|
|
A = (s.get("apps", {}).get(app) or {}).get("A")
|
|
if app == "nextcloud":
|
|
S = json.load(open(f"{w.SC}/a1-secret-state.json"))
|
|
A = S["A"]
|
|
w.wait_app(SUB[app], "/status.php", want=("200",), tries=90, delay=4)
|
|
names = sorted(S["files"])
|
|
return {"A": bool(FX[app].verify(w, SUB[app], A, w.say)),
|
|
"files": nf.verify_files(w, SUB[app], A, {names[0]: S["files"][names[0]], names[2]: S["files"][names[2]]}, w.say)}
|
|
if not A:
|
|
return {"A": None, "note": "no seed recorded"}
|
|
w.wait_app(SUB[app], "/", tries=90, delay=4)
|
|
return {"A": bool(FX[app].verify(w, SUB[app], A, w.say))}
|
|
|
|
|
|
def steady(app, t0, cap=1500, want_deployed=True, until=None):
|
|
last = None
|
|
while time.time() - t0 < cap:
|
|
try:
|
|
st = w.stack(app)
|
|
except Exception:
|
|
st = {}
|
|
cur = (st.get("deployed"), st.get("state"), st.get("updating"), st.get("update_phase"), st.get("hold_kind"))
|
|
if cur != last:
|
|
w.say(f" +{time.time()-t0:6.1f}s {app}: deployed={cur[0]} state={cur[1]} updating={cur[2]} phase={cur[3]} hold={cur[4]}")
|
|
last = cur
|
|
if until:
|
|
if until(st):
|
|
return round(time.time() - t0, 1), st
|
|
elif st and not st.get("updating") and not st.get("deploying"):
|
|
if (want_deployed and st.get("deployed") and st.get("state") in ("running", "stopped", "unhealthy", "degraded")) \
|
|
or (not want_deployed and not st.get("deployed")) or st.get("hold_kind"):
|
|
return round(time.time() - t0, 1), st
|
|
time.sleep(2)
|
|
return None, w.stack(app)
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------ the drill
|
|
def drill(edit, msg):
|
|
import fcntl
|
|
with open(f"{w.SC}/drill.lock", "w") as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
|
edit()
|
|
w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-E " + msg])
|
|
w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120)
|
|
w.say(" drill: " + msg)
|
|
|
|
|
|
def break_step(app):
|
|
frm, to, port = FAIL[app]
|
|
comp, fy = f"{w.DRILL}/templates/{app}/docker-compose.yml", f"{w.DRILL}/templates/{app}/.felhom.yml"
|
|
|
|
def edit():
|
|
c = open(comp).read()
|
|
assert f"image: {frm}" in c, (app, frm)
|
|
open(comp, "w").write(c.replace(f"image: {frm}", f"image: {to}", 1))
|
|
f = open(fy).read()
|
|
f2 = re.sub(r"(healthcheck:\n(?:.*\n){0,8}?\s+port: )" + str(port) + r"\b", r"\g<1>8999", f, count=1)
|
|
assert f2 != f, (app, "probe port not found")
|
|
open(fy, "w").write(f2)
|
|
drill(edit, f"{app}: failing step {frm} -> {to} + probe 8999")
|
|
|
|
|
|
def repair_step(app):
|
|
frm, to, port = FAIL[app]
|
|
comp, fy = f"{w.DRILL}/templates/{app}/docker-compose.yml", f"{w.DRILL}/templates/{app}/.felhom.yml"
|
|
|
|
def edit():
|
|
c = open(comp).read()
|
|
open(comp, "w").write(c.replace(f"image: {to}", f"image: {frm}", 1))
|
|
f = open(fy).read()
|
|
open(fy, "w").write(re.sub(r"(healthcheck:\n(?:.*\n){0,8}?\s+port: )8999\b", r"\g<1>" + str(port), f, count=1))
|
|
drill(edit, f"{app}: back to {frm} and its real probe")
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------ the arm
|
|
class Arm:
|
|
def __init__(self, n, kind, note):
|
|
self.n, self.kind, self.note, self.armed = n, kind, note, False
|
|
|
|
def __call__(self, why):
|
|
if self.armed or self.kind == "none":
|
|
return
|
|
self.armed = True
|
|
self.note["armed_at"] = time.strftime("%H:%M:%S")
|
|
self.note["armed_why"] = why
|
|
w.say(f" >>> ARMED accident {self.kind} ({why})")
|
|
if self.kind == "whole_box_backup":
|
|
code, d = w.ctl("POST", "/api/backup/run")
|
|
self.note["accident"] = f"POST /api/backup/run -> {code} {str(d)[:160]} at {time.strftime('%H:%M:%S')}"
|
|
w.say(" >>> " + self.note["accident"])
|
|
return
|
|
json.dump({"round": self.n, "kind": self.kind, "at": time.time(), "why": why},
|
|
open(os.path.join(EVD, f"arm-{self.n:02d}.json"), "w"))
|
|
|
|
def wait_done(self, cap=900):
|
|
if self.kind in ("none", "whole_box_backup") or not self.armed:
|
|
return
|
|
p = os.path.join(EVD, f"done-{self.n:02d}.json")
|
|
t0 = time.time()
|
|
while time.time() - t0 < cap and not os.path.exists(p):
|
|
time.sleep(2)
|
|
self.note["accident"] = json.load(open(p)) if os.path.exists(p) else "NOT FIRED (no done file)"
|
|
w.say(f" >>> accident done: {self.note['accident']}")
|
|
|
|
|
|
def wait_controller(cap=900):
|
|
t0 = time.time()
|
|
while time.time() - t0 < cap:
|
|
try:
|
|
w.login()
|
|
code, _ = w.ctl("GET", "/api/stacks")
|
|
if code == "200":
|
|
return round(time.time() - t0, 1)
|
|
except SystemExit:
|
|
pass
|
|
time.sleep(5)
|
|
return None
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------ actions
|
|
def a_unhealthy_install(app, note, arm, t0):
|
|
"""crash_loop / oom_app: install the drill app through the product; the box must stop it by itself."""
|
|
vals = w.deploy_values(app, app)
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/deploy", {"values": vals})
|
|
note["act"] = f"deploy -> {code}"
|
|
threading.Timer(20, arm, args=("20 s after the install",)).start()
|
|
return steady(app, t0, cap=1500, until=lambda st: st.get("hold_kind") == "unhealthy_stop")
|
|
|
|
|
|
def a_install(app, note, arm, t0, s):
|
|
"""n8n at the ladder's FIRST from, so round 10 has a step to climb."""
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
import ladder
|
|
comp = f"{w.DRILL}/templates/{app}/docker-compose.yml"
|
|
E, _, _ = ladder.parse(open(f"{w.DRILL}/templates/{app}/.felhom.yml").read())
|
|
head = open(comp).read()
|
|
s.setdefault("heads", {})[app] = head
|
|
save(s)
|
|
|
|
def edit():
|
|
c = head
|
|
for svc, ref in ladder.images_in(head).items():
|
|
if E[0]["from"].get(svc) and E[0]["from"][svc] != ref:
|
|
c = c.replace(f"image: {ref}", f"image: {E[0]['from'][svc]}", 1)
|
|
open(comp, "w").write(c)
|
|
drill(edit, f"{app} at the ladder's first from {E[0]['from']}")
|
|
w.sync_rescan()
|
|
time.sleep(3)
|
|
w.sync_rescan()
|
|
vals = w.deploy_values(app, SUB[app])
|
|
threading.Timer(20, arm, args=("20 s after the install press",)).start()
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/deploy", {"values": vals})
|
|
note["act"] = f"deploy -> {code} {str(d)[:100]}"
|
|
secs, st = steady(app, t0, until=lambda st: st.get("deployed") and (st.get("app_config") or {}).get("pinned_images")
|
|
and not st.get("deploying"))
|
|
if st.get("deployed"):
|
|
w.wait_app(SUB[app], "/", tries=90, delay=4)
|
|
A = FX[app].seed(w, SUB[app], w.say)
|
|
s.setdefault("apps", {}).setdefault(app, {})["A"] = A
|
|
save(s)
|
|
|
|
def back():
|
|
open(comp, "w").write(head)
|
|
drill(back, f"{app} back to the head (2 steps offered)")
|
|
w.sync_rescan()
|
|
return secs, st
|
|
|
|
|
|
def a_update(app, note, arm, t0, on_verify=None):
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/update")
|
|
note["act"] = f"Update -> {code} {str(d)[:140]}"
|
|
w.say(" " + note["act"])
|
|
phases, seen = [], None
|
|
while time.time() - t0 < 1800:
|
|
try:
|
|
st = w.stack(app)
|
|
except Exception:
|
|
st = {}
|
|
ph = st.get("update_phase")
|
|
if ph != seen and ph:
|
|
seen = ph
|
|
phases.append((round(time.time() - t0, 1), ph))
|
|
w.say(f" +{phases[-1][0]:6.1f}s phase={ph}")
|
|
if ph == "verifying":
|
|
if on_verify:
|
|
on_verify()
|
|
arm("at verifying")
|
|
if ph == "starting":
|
|
arm("at starting")
|
|
if st and not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - t0 > 3:
|
|
break
|
|
if not st and time.time() - t0 > 3:
|
|
time.sleep(5)
|
|
time.sleep(0.5)
|
|
note["phases"] = phases
|
|
return steady(app, t0)
|
|
|
|
|
|
def cut_newest_copy(app, note):
|
|
out = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={app} | sort | tail -1); echo "copy: $c"
|
|
docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c | head -3'""")
|
|
note["cutoff"] = out.strip()
|
|
w.say(" >>> finished-marker removed from the NEWEST undo copy:\n" + out)
|
|
|
|
|
|
def a_cutoff_file_app(app, note, arm, t0, s):
|
|
break_step(app)
|
|
note["badge_catchup_s"] = w.sync_rescan(expect_app=app, expect_ref=FAIL[app][1], tries=60, delay=5)
|
|
secs, st = a_update(app, note, arm, t0, on_verify=lambda: cut_newest_copy(app, note))
|
|
repair_step(app)
|
|
w.sync_rescan()
|
|
note["held"] = {k: st.get(k) for k in ("update_phase", "hold_kind", "hold_no_whole_copy", "hold_reason")}
|
|
note["held_page"] = household(app)
|
|
if st.get("hold_kind") == "update_failed" and not st.get("hold_no_whole_copy"):
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = w.sh(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={app}", f"{w.BASE}/backup/tier2/unit-restore"], timeout=120)
|
|
note["whole_restore_press"] = r.stdout
|
|
w.say(f" whole restore (the second drive) -> {r.stdout}")
|
|
last = None
|
|
for _ in range(600):
|
|
c2, dd = w.ctl("GET", "/api/backup/restore-status")
|
|
d2 = (dd.get("data") or {}) if isinstance(dd, dict) else {}
|
|
if not d2.get("running") and d2.get("last"):
|
|
last = d2.get("last")
|
|
break
|
|
time.sleep(2)
|
|
note["whole_restore"] = last
|
|
w.say(f" whole restore: {str(last)[:300]}")
|
|
return steady(app, t0)
|
|
return secs, st
|
|
|
|
|
|
def a_failing_step(app, note, arm, t0, s):
|
|
break_step(app)
|
|
note["badge_catchup_s"] = w.sync_rescan(expect_app=app, expect_ref=FAIL[app][1], tries=60, delay=5)
|
|
r = a_update(app, note, arm, t0)
|
|
repair_step(app)
|
|
w.sync_rescan()
|
|
return r
|
|
|
|
|
|
def a_remove_held(app, note, arm, t0):
|
|
st = w.stack(app)
|
|
note["held_before"] = {k: st.get(k) for k in ("deployed", "hold_kind", "hold_reason")}
|
|
threading.Timer(2, arm, args=("2 s after the remove press",)).start()
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/remove", {"remove_hdd_data": True, "remove_backups": True})
|
|
note["act"] = f"remove (data + backups) -> {code} {str(d)[:200]}"
|
|
w.say(" " + note["act"])
|
|
return steady(app, t0, want_deployed=False, cap=600)
|
|
|
|
|
|
def a_ladder_step(app, note, arm, t0, s):
|
|
w.sync_rescan()
|
|
st = w.stack(app)
|
|
note["steps_left_before"] = st.get("ladder_steps_left")
|
|
note["pinned_before"] = (st.get("app_config") or {}).get("pinned_images")
|
|
r = a_update(app, note, arm, t0)
|
|
w.sync_rescan()
|
|
st = w.stack(app)
|
|
note["steps_left_after"] = st.get("ladder_steps_left")
|
|
note["pinned_after"] = (st.get("app_config") or {}).get("pinned_images")
|
|
return r
|
|
|
|
|
|
def a_start_after_unhealthy(app, note, arm, t0):
|
|
st = w.stack(app)
|
|
note["before"] = {k: st.get(k) for k in ("state", "hold_kind", "hold_reason")}
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/start")
|
|
note["act"] = f"Start -> {code} {str(d)[:140]}"
|
|
w.say(" " + note["act"])
|
|
threading.Timer(5, arm, args=("5 s after Start",)).start()
|
|
time.sleep(3)
|
|
note["after_start"] = {k: w.stack(app).get(k) for k in ("state", "hold_kind")}
|
|
return steady(app, t0, cap=1200, until=lambda st: st.get("hold_kind") == "unhealthy_stop")
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------ stages
|
|
def setup():
|
|
s = load()
|
|
w.login()
|
|
for app in ("wishlist", "navidrome"):
|
|
w.wait_app(SUB[app], "/", tries=60, delay=3)
|
|
A = FX[app].seed(w, SUB[app], w.say)
|
|
s.setdefault("apps", {}).setdefault(app, {})["A"] = A
|
|
w.say(f" seeded {app}: {bool(A)}")
|
|
save(s)
|
|
s["setup_readback"] = {a: readback(a, s) for a in ("wishlist", "navidrome", "nextcloud")}
|
|
w.say("setup readback: " + json.dumps(s["setup_readback"]))
|
|
save(s)
|
|
open(os.path.join(EVD, "00-setup.log"), "w").write("\n".join(w.LOG) + "\n")
|
|
|
|
|
|
def run_round(n):
|
|
s = load()
|
|
sched = {r["round"]: r for r in schedule()}[n]
|
|
app = TARGET[n]
|
|
w.login()
|
|
t_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
w.say(f"==== ROUND {n}: {sched['action']} on {app} x {sched['accident']} ({t_iso})")
|
|
note = {"round": n, **sched, "app": app, "started": t_iso, "before": household(app)}
|
|
arm = Arm(n, sched["accident"], note)
|
|
t0 = time.time()
|
|
act = sched["action"]
|
|
try:
|
|
if act in ("oom_app", "crash_loop"):
|
|
secs, st = a_unhealthy_install(app, note, arm, t0)
|
|
elif act == "install":
|
|
secs, st = a_install(app, note, arm, t0, s)
|
|
elif act == "cutoff_undo_file_app":
|
|
secs, st = a_cutoff_file_app(app, note, arm, t0, s)
|
|
elif act == "failing_step":
|
|
secs, st = a_failing_step(app, note, arm, t0, s)
|
|
elif act == "remove_held":
|
|
secs, st = a_remove_held(app, note, arm, t0)
|
|
elif act == "ladder_step":
|
|
secs, st = a_ladder_step(app, note, arm, t0, s)
|
|
elif act == "start_after_unhealthy":
|
|
secs, st = a_start_after_unhealthy(app, note, arm, t0)
|
|
else:
|
|
raise SystemExit("unknown action " + act)
|
|
except Exception as e:
|
|
note["act_exception"] = repr(e)
|
|
w.say(" ACT EXCEPTION " + repr(e))
|
|
secs, st = None, {}
|
|
arm.wait_done()
|
|
if sched["accident"] in ("power_cut", "docker_restart", "controller_killed"):
|
|
note["controller_back_s"] = wait_controller()
|
|
if act in ("oom_app", "crash_loop", "start_after_unhealthy"):
|
|
secs2, st = steady(app, t0, cap=1800, until=lambda st: st.get("hold_kind") == "unhealthy_stop")
|
|
else:
|
|
secs2, st = steady(app, t0, want_deployed=(act != "remove_held"))
|
|
secs = secs2 or secs
|
|
note["time_to_steady_s"] = round(time.time() - t0, 1) if secs is None else secs
|
|
note["after"] = household(app)
|
|
if not w.stack(app).get("deployed"):
|
|
note["after"]["leftovers"] = w.guest(f"docker ps -a --filter label=com.docker.compose.project={app} --format '{{{{.Names}}}}'; docker volume ls -q | grep '^{app}_' || true; ls -d /opt/docker/stacks/{app}/applied-meta 2>/dev/null || true").strip()
|
|
note["readback"] = readback(app, s)
|
|
note["events"] = events_since(t_iso)
|
|
note["box_log"] = box_log(t_iso, app)
|
|
note["undo_copies_left"] = w.guest(f"docker volume ls -q --filter label=felhom.undo-copy-of={app} | wc -l").strip()
|
|
open(os.path.join(EVD, f"round-{n:02d}-controller.log"), "w").write(
|
|
w.guest(f"docker logs --since {t_iso} felhom-controller 2>&1 | grep -vE 'auth: valid session|router.go:81|Status refresh' | tail -3000"))
|
|
json.dump(note, open(os.path.join(EVD, f"round-{n:02d}.json"), "w"), indent=2, ensure_ascii=False, default=str)
|
|
open(os.path.join(EVD, f"round-{n:02d}.log"), "w").write("\n".join(w.LOG) + "\n")
|
|
w.say(f"==== ROUND {n} END: steady in {note['time_to_steady_s']}s; readback={note['readback']}; events={note['events']}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1] == "setup":
|
|
setup()
|
|
elif sys.argv[1] == "round":
|
|
run_round(int(sys.argv[2]))
|