118 lines
6.4 KiB
Python
118 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""r1350_live.py <phase> — R-1350 + D65 live proof on Tester 1 (controller 0.349.0-rc.1, agent 0.170.0, hub 0.177.0), 2026-10-29 evening.
|
|
notice the household dashboard after the operator reboot: the line („...ujrainditotta a dobozt").
|
|
hold a throwaway Gokapi; its compose made to exit at once (entrypoint `sh -c 'exit 1'`) and the household's Restart
|
|
→ Docker's restarts → the box's crash-loop stop and its unhealthy_stop hold. Then the hub's held-apps list.
|
|
clear the hub's „Clear hold" (the button's endpoint, CC_HOSTS token) → the result; the hold gone on the box (memory
|
|
through the stack API, the file) and on the hub; then wait for the controller's NEXT settings save and look
|
|
again (R-1343's order).
|
|
teardown the entrypoint line removed, the app removed through the product."""
|
|
import json, os, re, subprocess, sys, time
|
|
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad"
|
|
os.environ.setdefault("SC", f"{SCR}/rt-tester1")
|
|
os.environ.setdefault("TARGET", "tester-1")
|
|
import box_walk as w
|
|
|
|
EV = f"{SCR}/d65/live/ev"
|
|
APP, SUB = "gokapi", "gokapi"
|
|
COMPOSE = f"/opt/docker/stacks/{APP}/docker-compose.yml"
|
|
SETTINGS = "/opt/docker/felhom-controller/data/settings.json"
|
|
HOST = "tester-1-d70be4"
|
|
|
|
|
|
def now():
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
def hold_in_file():
|
|
return w.guest(f"docker exec felhom-controller python3 -c \"import json;print(sorted((json.load(open('{SETTINGS}')).get('restore_holds') or {{}}).keys()))\" 2>/dev/null || "
|
|
f"python3 -c \"import json;print(sorted((json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json')).get('restore_holds') or {{}}).keys()))\"").strip()
|
|
|
|
|
|
def look(tag):
|
|
st = w.stack(APP)
|
|
w.say(f" [{tag}] api: state={st.get('state')} hold={st.get('hold_reason')!r} | file holds: {hold_in_file()}")
|
|
return st
|
|
|
|
|
|
def hub_page(pat):
|
|
r = subprocess.run(["sec", "run", "CC_HOSTS", "--", "bash", f"{SCR}/d63/hubget.sh", "CC_HOSTS", f"/hosts/{HOST}", pat],
|
|
capture_output=True, text=True, timeout=60)
|
|
return r.stdout.strip()
|
|
|
|
|
|
w.login()
|
|
phase = sys.argv[1]
|
|
t0 = now()
|
|
w.say(f"=== phase {phase} at {t0}")
|
|
w.say("controller: " + w.guest("docker ps --filter name=^/felhom-controller$ --format '{{.Image}} {{.Status}}'").strip())
|
|
|
|
if phase == "notice":
|
|
body = w.page("/dashboard")
|
|
m = re.search(r'data-operator-reboot>([^<]*)<', body or "")
|
|
w.say(" dashboard line: " + (m.group(1) if m else "NONE"))
|
|
w.say(" ASCII control: 'jraind' in page = %s; 'Vez' (the page title) in page = %s" % ("jraind" in (body or ""), "Vez" in (body or "")))
|
|
|
|
elif phase == "hold":
|
|
assert w.deploy(APP, SUB), "deploy failed"
|
|
# the template's own entrypoint ends `exec /app/run.sh`: make it `exit 1` (the first try added a second entrypoint key)
|
|
w.guest(f"sed -i '/^ entrypoint: \\[\"sh\", \"-c\", \"exit 1\"\\]$/d; s#^ exec /app/run.sh$# exit 1#' {COMPOSE}")
|
|
w.say(" compose: " + w.guest(f"grep -nE 'image:|entrypoint:' {COMPOSE}").strip())
|
|
c, d = w.ctl("POST", f"/api/stacks/{APP}/restart")
|
|
w.say(f" Restart → {c} {str(d)[:120]}")
|
|
for _ in range(80):
|
|
time.sleep(10)
|
|
st = w.stack(APP)
|
|
if st.get("hold_reason"):
|
|
break
|
|
look("held")
|
|
w.say(" box log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'gokapi.*(crash|hold|stop)|unhealthy' | tail -6"))
|
|
for _ in range(12):
|
|
time.sleep(10)
|
|
page = hub_page("gokapi|Clear hold|No app is held")
|
|
if "gokapi" in page:
|
|
break
|
|
w.say(" hub page: " + page.replace("\n", " | ")[:600])
|
|
|
|
elif phase == "clear":
|
|
look("before")
|
|
r = subprocess.run(["sec", "run", "CC_HOSTS", "--", "bash", f"{SCR}/p1/press.sh", "CC_HOSTS", HOST, "clear_hold", APP],
|
|
capture_output=True, text=True, timeout=60)
|
|
w.say(" press: " + r.stdout.strip().replace("\n", " | "))
|
|
res = ""
|
|
for _ in range(60):
|
|
time.sleep(5)
|
|
res = subprocess.run(["sudo", "kubectl", "-n", "felhom-system", "logs", "deploy/hub", "--since", "10m"], capture_output=True, text=True).stdout
|
|
m = [l for l in res.splitlines() if "operator action #" in l and "clear_hold gokapi" in l and "(pressed by" in l and ("done" in l or "refused" in l or "failed" in l)]
|
|
if m:
|
|
w.say(" hub result: " + m[-1][:400])
|
|
break
|
|
w.say(" box log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'hold CLEARED|clear_hold|operator action' | tail -6"))
|
|
look("after clear")
|
|
t1 = now()
|
|
for _ in range(80):
|
|
time.sleep(15)
|
|
if "Settings saved" in w.guest(f"docker logs --since {t1} felhom-controller 2>&1 | grep -E 'Settings saved' | tail -1"):
|
|
break
|
|
w.say(" next settings save: " + w.guest(f"docker logs --since {t1} felhom-controller 2>&1 | grep -E 'Settings saved' | head -1").strip())
|
|
look("after the next save")
|
|
time.sleep(70)
|
|
w.say(" hub page: " + hub_page("gokapi|No app is held").replace("\n", " | ")[:400])
|
|
w.say(" hub held-apps log: " + subprocess.run(["sudo", "kubectl", "-n", "felhom-system", "logs", "deploy/hub", "--since", "15m"],
|
|
capture_output=True, text=True).stdout.count("tester-1/gokapi no longer held").__str__() + " 'no longer held' line(s)")
|
|
w.say(" R-1350 record: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'recorded as wanted|desired state for gokapi' | tail -3"))
|
|
w.say(" api after: " + json.dumps({k: w.stack(APP).get(k) for k in ("state", "hold_reason")}) + " desired=" + w.guest(f"grep -m1 desired_state /opt/docker/stacks/{APP}/app.yaml").strip())
|
|
time.sleep(240)
|
|
w.say(" app_start_failed since the clear (R-1350; want none): [" + w.guest(f"docker logs --since {t1} felhom-controller 2>&1 | grep -iE 'app_start_failed|nem fut: Gokapi|gokapi.*(not running|dead)' | tail -4").strip() + "]")
|
|
w.say(" positive control — the dead-app scan ran meanwhile: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'deadapp\] check alive' | tail -1").strip())
|
|
|
|
elif phase == "teardown":
|
|
w.guest(f"sed -i 's#^ exit 1$# exec /app/run.sh#' {COMPOSE}")
|
|
look("before remove")
|
|
w.remove(APP)
|
|
look("after remove")
|
|
|
|
open(os.path.join(EV, f"p3-{phase}.log"), "a").write("\n".join(w.LOG) + "\n")
|