#!/usr/bin/env python3 """walk_crash_t1.py — claims fail.app-crash, fault.customer-email, fault.operator-email on Tester 1, front doors only. A household's app starts crashing. A throwaway Radicale is installed through the product; its main process is killed with kill -9, 15 s apart (never `docker kill`: the restart policy would mask it). Want: the box stops it (crash loop), holds it, the hub hears, and a mail reaches the mailbox (read through the mailbox, a different channel). Then the household's Start, then the product's Remove; nothing left.""" import json, os, sys, time sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts") os.environ.setdefault("TARGET", "tester-1") import box_walk as w APP, SUB = "radicale", "radicale" EV = os.environ["EV"] t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) w.login() assert not w.stack(APP).get("deployed"), "radicale is already on Tester 1 — not a throwaway" assert w.deploy(APP, SUB), "install failed" w.wait_app(SUB, "/", tries=40) w.say(f" installed {w.stack(APP).get('state')}; t0 {t0}") start = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) for i in range(12): pid = w.guest(f"docker inspect -f '{{{{.State.Pid}}}}' {APP} 2>/dev/null").strip() if pid and pid != "0": w.guest(f"kill -9 {pid}") st = w.stack(APP) w.say(f" kill -9 #{i + 1} pid {pid}: state {st.get('state')} hold {st.get('hold_reason')!r}") if st.get("hold_reason"): break time.sleep(15) time.sleep(40) st = w.stack(APP) w.say(f" after the kills {time.strftime('%H:%M:%S', time.gmtime())}: state={st.get('state')} hold={st.get('hold_reason')!r}") w.say(" controller log:\n" + w.guest(f"docker logs --since {start} felhom-controller 2>&1 | grep -E 'crash_loop|STOPPING|unhealthy|app_held|app_crash|PushEvent|Event sent' | grep -v DEBUG | cut -c1-240 | tail -14")) open(os.path.join(EV, "crash-walk-part1.log"), "w").write("\n".join(w.LOG) + "\n") print("WAITING-FOR-MAIL", flush=True) # the mail is read by the operator's session from the mailbox; the app stays held until then time.sleep(int(os.environ.get("HOLD_S", "240"))) c, d = w.ctl("POST", f"/api/stacks/{APP}/start") w.say(f" the household's Start -> {c} {str(d)[:140]}") for _ in range(24): time.sleep(5) st = w.stack(APP) if st.get("state") == "running" and not st.get("hold_reason"): break w.say(f" after Start: state={st.get('state')} hold={st.get('hold_reason')!r}") w.remove(APP) w.say(f" removed: deployed={w.stack(APP).get('deployed')}; leftovers: " + w.guest(f"docker ps -a --format '{{{{.Names}}}}' | grep -c '^{APP}' ; docker volume ls --format '{{{{.Name}}}}' | grep -c '^{APP}_'; ls /opt/docker/stacks/{APP}/app.yaml 2>&1 | tail -1").replace("\n", " ")) open(os.path.join(EV, "crash-walk.log"), "w").write("\n".join(w.LOG) + "\n")