50 lines
2.8 KiB
Python
50 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""walk_crash_t1.py — claims fail.app-crash, fault.customer-email, fault.operator-email on Tester 1, front doors only.
|
|
|
|
A household's app starts crashing. A throwaway Radicale is installed through the product; its main process is killed with
|
|
kill -9, 15 s apart (never `docker kill`: the restart policy would mask it). Want: the box stops it (crash loop), holds it, the hub
|
|
hears, and a mail reaches the mailbox (read through the mailbox, a different channel). Then the household's Start, then the
|
|
product's Remove; nothing left."""
|
|
import json, os, sys, time
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
os.environ.setdefault("TARGET", "tester-1")
|
|
import box_walk as w
|
|
|
|
APP, SUB = "radicale", "radicale"
|
|
EV = os.environ["EV"]
|
|
t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
w.login()
|
|
assert not w.stack(APP).get("deployed"), "radicale is already on Tester 1 — not a throwaway"
|
|
assert w.deploy(APP, SUB), "install failed"
|
|
w.wait_app(SUB, "/", tries=40)
|
|
w.say(f" installed {w.stack(APP).get('state')}; t0 {t0}")
|
|
start = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
for i in range(12):
|
|
pid = w.guest(f"docker inspect -f '{{{{.State.Pid}}}}' {APP} 2>/dev/null").strip()
|
|
if pid and pid != "0":
|
|
w.guest(f"kill -9 {pid}")
|
|
st = w.stack(APP)
|
|
w.say(f" kill -9 #{i + 1} pid {pid}: state {st.get('state')} hold {st.get('hold_reason')!r}")
|
|
if st.get("hold_reason"):
|
|
break
|
|
time.sleep(15)
|
|
time.sleep(40)
|
|
st = w.stack(APP)
|
|
w.say(f" after the kills {time.strftime('%H:%M:%S', time.gmtime())}: state={st.get('state')} hold={st.get('hold_reason')!r}")
|
|
w.say(" controller log:\n" + w.guest(f"docker logs --since {start} felhom-controller 2>&1 | grep -E 'crash_loop|STOPPING|unhealthy|app_held|app_crash|PushEvent|Event sent' | grep -v DEBUG | cut -c1-240 | tail -14"))
|
|
open(os.path.join(EV, "crash-walk-part1.log"), "w").write("\n".join(w.LOG) + "\n")
|
|
print("WAITING-FOR-MAIL", flush=True)
|
|
# the mail is read by the operator's session from the mailbox; the app stays held until then
|
|
time.sleep(int(os.environ.get("HOLD_S", "240")))
|
|
c, d = w.ctl("POST", f"/api/stacks/{APP}/start")
|
|
w.say(f" the household's Start -> {c} {str(d)[:140]}")
|
|
for _ in range(24):
|
|
time.sleep(5)
|
|
st = w.stack(APP)
|
|
if st.get("state") == "running" and not st.get("hold_reason"):
|
|
break
|
|
w.say(f" after Start: state={st.get('state')} hold={st.get('hold_reason')!r}")
|
|
w.remove(APP)
|
|
w.say(f" removed: deployed={w.stack(APP).get('deployed')}; leftovers: " + w.guest(f"docker ps -a --format '{{{{.Names}}}}' | grep -c '^{APP}' ; docker volume ls --format '{{{{.Name}}}}' | grep -c '^{APP}_'; ls /opt/docker/stacks/{APP}/app.yaml 2>&1 | tail -1").replace("\n", " "))
|
|
open(os.path.join(EV, "crash-walk.log"), "w").write("\n".join(w.LOG) + "\n")
|