109 lines
5.9 KiB
Python
109 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""p4_live.py <phase> — Tester 1 on the release build (controller 0.347.0, agent 0.168.0), 2026-10-29, attended.
|
|
deploy a throwaway Gokapi (catalog cap 128 MB).
|
|
backup the household's „Mentés most" (whole box; the only backup press there is, R-648) — makes Gokapi's unit.
|
|
restore one LOCAL restore of Gokapi from its unit through the restore page's form; the app must run after.
|
|
oom a memory hog inside Gokapi (`tail /dev/zero`, a child): the host counter rises; the controller logs the OOM
|
|
line ONCE with the app and the source, and pushes app_oom once.
|
|
teardown the app removed through the product."""
|
|
import json, os, sys, time
|
|
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad"
|
|
os.environ.setdefault("SC", f"{SCR}/rt-tester1")
|
|
os.environ.setdefault("TARGET", "tester-1")
|
|
import box_walk as w
|
|
|
|
EV = f"{SCR}/d63/ev"
|
|
APP, SUB = "gokapi", "gokapi"
|
|
|
|
|
|
def now():
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
def look():
|
|
st = w.stack(APP)
|
|
w.say(f" app: state={st.get('state')} hold={st.get('hold_reason')!r} deployed={st.get('deployed')}")
|
|
w.say(" container: " + w.guest(f"docker ps -a --filter name=^/{APP}$ --format '{{{{.Image}}}} {{{{.Status}}}}'").strip())
|
|
return st
|
|
|
|
|
|
w.login()
|
|
phase = sys.argv[1]
|
|
t0 = now()
|
|
w.say(f"=== phase {phase} at {t0}")
|
|
w.say("controller: " + w.guest("docker ps --filter name=^/felhom-controller$ --format '{{.Image}} {{.Status}}'").strip())
|
|
|
|
if phase == "deploy":
|
|
assert w.deploy(APP, SUB), "deploy failed"
|
|
w.say(" cap: " + w.guest(f"docker inspect -f '{{{{.HostConfig.Memory}}}}' {APP}; docker exec {APP} sh -c 'command -v tail; echo ok'").strip())
|
|
look()
|
|
|
|
elif phase == "backup":
|
|
c, d = w.ctl("POST", "/api/backup/run")
|
|
w.say(f" backup press → {c} {str(d)[:160]}")
|
|
for _ in range(240):
|
|
time.sleep(10)
|
|
lg = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|Recovery unit (captured|NOT captured) for {APP}' | tail -3")
|
|
if "App-data backup completed" in lg:
|
|
break
|
|
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|{APP}' | tail -8"))
|
|
w.say(f" restore points: {json.dumps(w.snapshots(APP))[:300]}")
|
|
|
|
elif phase == "restore":
|
|
look()
|
|
res = w.restore(APP)
|
|
w.say(f" result: {json.dumps(res)[:600]}")
|
|
code, d = w.ctl("GET", "/api/backup/restore-status")
|
|
w.say(f" restore record: {json.dumps(d)[:600]}")
|
|
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E '{APP}|Restore-from-unit|R-1270|restore copy' | tail -15"))
|
|
look()
|
|
w.say(" front door: " + str(w.app_curl(SUB, "/")[1]))
|
|
|
|
elif phase == "oom":
|
|
cid = w.guest(f"docker inspect -f '{{{{.Id}}}}' {APP}").strip()
|
|
hostf = f"/sys/fs/cgroup/lxc/9201/ns/system.slice/docker-{cid}.scope/memory.events"
|
|
w.say(" host counter before: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip())
|
|
w.say(" hog: " + w.guest(f"docker exec {APP} sh -c 'tail /dev/zero' ; echo rc=$?", timeout=300).strip())
|
|
w.say(" after: " + w.guest(f"docker inspect -f 'running={{{{.State.Running}}}} oomkilled={{{{.State.OOMKilled}}}} restarts={{{{.RestartCount}}}}' {APP}"))
|
|
w.say(" host counter after: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip())
|
|
time.sleep(75)
|
|
w.say(" OOM lines (first 2 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10"))
|
|
time.sleep(65)
|
|
w.say(" app_oom events pushed: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -cE 'Event pushed: app_oom'"))
|
|
w.say(" OOM lines (after 4 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10"))
|
|
|
|
elif phase == "d62":
|
|
# D62 end to end on the release build and hub 0.175.0: restore, the copy listed, deleted through the form at once
|
|
import re, subprocess
|
|
res = w.restore(APP)
|
|
w.say(f" restore ok={res.get('state_after')} hold={res.get('hold_after')}")
|
|
page = w.page("/backups/restore")
|
|
rows = re.findall(r'<details class="kept-delete" data-restore-copy="([^"]+)">', page)
|
|
w.say(f" rows: {rows}")
|
|
m = re.search(r'data-restore-copies="true">(.*?)</div>\s*<script>', page, re.S)
|
|
if m:
|
|
w.say(" list text: " + re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", m.group(1))).strip()[:400])
|
|
if rows:
|
|
app, stamp = rows[0].split("/", 1)
|
|
names = re.findall(r'<span class="app-row-name">([^<]+)</span>', page[:page.find('data-restore-copy="' + rows[0])])
|
|
typed = names[-1].strip() if names else app
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = subprocess.run(["curl", "-sk", "-w", "\n%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}",
|
|
"-H", "Accept: application/json", "-X", "POST", "--data-urlencode", f"_csrf={csrf}",
|
|
"--data-urlencode", f"app={app}", "--data-urlencode", f"stamp={stamp}", "--data-urlencode", f"confirm={typed}",
|
|
f"{w.BASE}/backup/restore-copy/delete"], capture_output=True, text=True, timeout=60)
|
|
w.say(f" delete ({typed!r}) → {r.stdout.strip()[-300:]}")
|
|
time.sleep(8)
|
|
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'restore copy DELETED|restore_copy_deleted|Event push' | tail -4"))
|
|
w.say(" copies left: " + (w.guest("find /mnt/sys_drive/felhom-data/backups/restore-undo -mindepth 2 -maxdepth 2 2>/dev/null").strip() or "(none)"))
|
|
|
|
elif phase == "teardown":
|
|
look()
|
|
w.remove(APP)
|
|
look()
|
|
|
|
open(os.path.join(EV, f"p4-{phase}.log"), "a").write("\n".join(w.LOG) + "\n")
|