"""Night 3 on 9202 — accident: the controller is killed (kill -9 of its PID) BETWEEN two apps of the leg, right after navidrome's step ends. The controller log is saved when the accident is ARMED (R-683), before the kill.""" import sys, time, subprocess, datetime, json, os import walk as w w.login() EVD = w.EV + "/C/night3-kill" os.makedirs(EVD, exist_ok=True) start = datetime.datetime.now() + datetime.timedelta(minutes=2) W = (start - datetime.timedelta(minutes=105)).strftime("%H:%M") w.say(f"night3: W={W}, leg at {start.strftime('%H:%M')}") subprocess.run(["python3", "movewin.py", W, EVD + "/window-moved.txt"], capture_output=True, text=True) since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) t0 = time.time() while time.time() - t0 < 1800: out = w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -F '[update-leg] navidrome: step ended' | tail -1") if out.strip(): break time.sleep(1) w.say("ARMED: " + out.strip()) open(EVD + "/controller-before-kill.log", "w").write(w.guest(f"docker logs --since {since} felhom-controller 2>&1")) k = w.guest("P=$(docker inspect -f '{{.State.Pid}}' felhom-controller); echo pid=$P; kill -9 $P; echo kill_rc=$?; date -u +%FT%TZ") w.say("KILL: " + k.strip().replace("\n", " ")) t1 = time.time() while time.time() - t1 < 600: time.sleep(5) st = w.guest("docker inspect -f '{{.State.Status}} {{.State.Health.Status}} {{.RestartCount}} {{.State.StartedAt}}' felhom-controller 2>&1").strip() if "running healthy" in st: break w.say(f"controller after kill: {st} ({round(time.time()-t1)} s)") time.sleep(150) # give a resumed leg every chance to show itself after = w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -E 'update-leg|\\[stacks\\] update (vikunja|navidrome)|Running job: offbox|resum|Resum' ") open(EVD + "/controller-after-kill.log", "w").write(after) w.say("after-kill leg lines:\n" + after[-1500:]) w.login() subprocess.run(["python3", "c1_state.py", EVD + "/pages-after.json"]) subprocess.run(["python3", "applines.py", EVD + "/app-lines.txt"]) subprocess.run(["python3", "seedread.py", "read", "night3-kill (after the kill)"]) open(EVD + "/runner.log", "w").write("\n".join(w.LOG) + "\n")