admin
a7971c4822
gates / gates (push) Successful in 25s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
39 lines
2.2 KiB
Python
39 lines
2.2 KiB
Python
"""Night 3 on 9202 — accident: the controller is killed (kill -9 of its PID) BETWEEN two apps of the leg, right after
|
|
navidrome's step ends. The controller log is saved when the accident is ARMED (R-683), before the kill."""
|
|
import sys, time, subprocess, datetime, json, os
|
|
import walk as w
|
|
w.login()
|
|
EVD = w.EV + "/C/night3-kill"
|
|
os.makedirs(EVD, exist_ok=True)
|
|
start = datetime.datetime.now() + datetime.timedelta(minutes=2)
|
|
W = (start - datetime.timedelta(minutes=105)).strftime("%H:%M")
|
|
w.say(f"night3: W={W}, leg at {start.strftime('%H:%M')}")
|
|
subprocess.run(["python3", "movewin.py", W, EVD + "/window-moved.txt"], capture_output=True, text=True)
|
|
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
t0 = time.time()
|
|
while time.time() - t0 < 1800:
|
|
out = w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -F '[update-leg] navidrome: step ended' | tail -1")
|
|
if out.strip():
|
|
break
|
|
time.sleep(1)
|
|
w.say("ARMED: " + out.strip())
|
|
open(EVD + "/controller-before-kill.log", "w").write(w.guest(f"docker logs --since {since} felhom-controller 2>&1"))
|
|
k = w.guest("P=$(docker inspect -f '{{.State.Pid}}' felhom-controller); echo pid=$P; kill -9 $P; echo kill_rc=$?; date -u +%FT%TZ")
|
|
w.say("KILL: " + k.strip().replace("\n", " "))
|
|
t1 = time.time()
|
|
while time.time() - t1 < 600:
|
|
time.sleep(5)
|
|
st = w.guest("docker inspect -f '{{.State.Status}} {{.State.Health.Status}} {{.RestartCount}} {{.State.StartedAt}}' felhom-controller 2>&1").strip()
|
|
if "running healthy" in st:
|
|
break
|
|
w.say(f"controller after kill: {st} ({round(time.time()-t1)} s)")
|
|
time.sleep(150) # give a resumed leg every chance to show itself
|
|
after = w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -E 'update-leg|\\[stacks\\] update (vikunja|navidrome)|Running job: offbox|resum|Resum' ")
|
|
open(EVD + "/controller-after-kill.log", "w").write(after)
|
|
w.say("after-kill leg lines:\n" + after[-1500:])
|
|
w.login()
|
|
subprocess.run(["python3", "c1_state.py", EVD + "/pages-after.json"])
|
|
subprocess.run(["python3", "applines.py", EVD + "/app-lines.txt"])
|
|
subprocess.run(["python3", "seedread.py", "read", "night3-kill (after the kill)"])
|
|
open(EVD + "/runner.log", "w").write("\n".join(w.LOG) + "\n")
|