admin
67053335d1
gates / gates (push) Successful in 13m17s
Every catalog app installed, used, backed up, removed keeping data, reinstalled, restored and removed with data on a fresh box from the public ISO (demo-hp VM 9360, Tester4), endpoint-level. Worst: a reinstall's capture rewrites the kept unit so its restore breaks the app (R-1269, P2), the restore still reports success (R-1270, P2), and „use my kept data" loaded another box's future-dated database (R-1271, P2). Drill catalog fixes for the morning list (n8n usage data; OpenGist 256M, proven by a cgroup probe). VM 9360 and LXC 9361 destroyed; hub host Tester4-aa9466 left for the operator. REPORT-catalog-sweep-2026-10-24.md.
363 lines
18 KiB
Python
363 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""sweep.py — the catalog sweep of 2026-10-24: every app, one at a time, the household's cycle, endpoint-level.
|
|
|
|
The same requests the dashboard's pages send (POST /api/stacks/<app>/deploy, /stop, /remove, /api/backup/run,
|
|
POST /backup/restore), and the app's own front door through traefik (Host: <sub>.tester4.test) for the seed and the
|
|
read-back (`upgrade_fixtures_box.py`, the R-156 rule: nothing is planted by hand). No docker exec changes state; the
|
|
docker commands here only READ (ps, stats, inspect, logs, volume ls).
|
|
|
|
usage: SC=<0700 dir with .ctlpw> SWEEP_EV=<evidence dir> DRIVE_ROOT=<data drive> python3 sweep.py <app> [--restore]
|
|
|
|
Per app it writes <SWEEP_EV>/<app>/record.json and walk.log. A step that fails is recorded and the cycle goes on
|
|
where it can; the app is always removed (with its data) at the end, so the next app starts on an empty box.
|
|
"""
|
|
import json, os, re, sys, threading, time
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
os.environ.setdefault("TARGET", "sweep")
|
|
import box_walk as w # noqa: E402 (the sweep copy: a `sweep` target row, DRIVE_ROOT from env)
|
|
import upgrade_fixtures_box as fxm # noqa: E402
|
|
from sweep_fixtures import SWEEP_FIXTURES # noqa: E402 (fixtures written for this sweep; never override the catalog's)
|
|
for _k, _v in SWEEP_FIXTURES.items():
|
|
fxm.FIXTURES.setdefault(_k, _v)
|
|
|
|
EV = os.environ["SWEEP_EV"]
|
|
CATALOG = os.environ.get("CATALOG", "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu")
|
|
SECRET_RE = re.compile(r"(?i)(pass(?:word)?|secret|token|api[_-]?key|authorization|cookie)(\s*[:=]\s*)\S+")
|
|
|
|
|
|
def scrub(s):
|
|
return SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + "<redacted>", s)
|
|
|
|
|
|
def g(script, timeout=300):
|
|
return w.guest(script, timeout=timeout)
|
|
|
|
|
|
def containers(app):
|
|
out = g(f"docker ps -a --filter label=com.docker.compose.project={app} --format '{{{{.Names}}}}'")
|
|
return [c for c in out.split() if c]
|
|
|
|
|
|
class MemWatch(threading.Thread):
|
|
"""docker stats every 15 s for the app's containers; peak per container (docker stats excludes the file cache)."""
|
|
|
|
def __init__(self, app):
|
|
super().__init__(daemon=True)
|
|
self.app, self.peak, self.anon, self.stop_ev = app, {}, {}, threading.Event()
|
|
|
|
def run(self):
|
|
while not self.stop_ev.is_set():
|
|
out = g(f"c=$(docker ps --filter label=com.docker.compose.project={self.app} -q); "
|
|
"[ -n \"$c\" ] && docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' $c", timeout=60)
|
|
# anon memory from the cgroup (the file cache excluded — the project's measure, memory note
|
|
# harness-memory-peak-includes-page-cache); docker stats above can include active file cache
|
|
an = g(f"for c in $(docker ps --filter label=com.docker.compose.project={self.app} -q); do "
|
|
"n=$(docker inspect -f '{{.Name}}' $c); id=$(docker inspect -f '{{.Id}}' $c); "
|
|
"echo $n $(grep -E '^anon ' /sys/fs/cgroup/system.slice/docker-$id.scope/memory.stat | cut -d' ' -f2); done", timeout=60)
|
|
for line in an.splitlines():
|
|
p = line.split()
|
|
if len(p) == 2 and p[1].isdigit():
|
|
k = p[0].lstrip("/")
|
|
self.anon[k] = max(self.anon.get(k, 0), round(int(p[1]) / 1048576, 1))
|
|
for line in out.splitlines():
|
|
m = re.match(r"(\S+) ([\d.]+)([KMG]i?B)", line)
|
|
if m:
|
|
v = float(m.group(2)) * {"KiB": 1 / 1024, "KB": 1 / 1024, "MiB": 1, "MB": 1, "GiB": 1024, "GB": 1024}.get(m.group(3), 1)
|
|
self.peak[m.group(1)] = max(self.peak.get(m.group(1), 0), round(v, 1))
|
|
self.stop_ev.wait(15)
|
|
|
|
|
|
def limits(app):
|
|
out = g(f"for c in $(docker ps -a --filter label=com.docker.compose.project={app} -q); do "
|
|
"docker inspect -f '{{.Name}} {{.HostConfig.Memory}} {{.RestartCount}} {{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' $c; done")
|
|
res = {}
|
|
for line in out.splitlines():
|
|
p = line.split()
|
|
if len(p) >= 4:
|
|
res[p[0].lstrip("/")] = {"limit_mib": round(int(p[1]) / 1048576) if p[1].isdigit() else None,
|
|
"restarts": int(p[2]) if p[2].isdigit() else None, "status": p[3],
|
|
"health": p[4] if len(p) > 4 else ""}
|
|
return res
|
|
|
|
|
|
def error_lines(app, since):
|
|
out = g(f"for c in $(docker ps -a --filter label=com.docker.compose.project={app} --format '{{{{.Names}}}}'); do "
|
|
f"docker logs --since {int(since)} $c 2>&1 | grep -iE '(^|[^a-z_])(error|fatal|panic|exception|traceback)([^a-z_]|$)' "
|
|
"| grep -viE 'error_reporting|errorlog|level=info|no error|0 errors|errors: 0|\"level\":\"info\"' | sed \"s/^/$c: /\"; done", timeout=180)
|
|
lines = [l for l in out.splitlines() if l.strip()]
|
|
return {"count": len(lines), "sample": [scrub(l)[:200] for l in lines[:4]]}
|
|
|
|
|
|
def page_text(path, lang="hu"):
|
|
"""The page as the household reads it (?lang= is the page's own language switch), menu and footer cut off."""
|
|
import html as _h
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
sep = "&" if "?" in path else "?"
|
|
r = w.sh(["curl", "-sk", "-L", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", f"{w.BASE}{path}{sep}lang={lang}"])
|
|
t = re.sub(r"(?s)<script.*?</script>|<style.*?</style>|<nav.*?</nav>", " ", r.stdout or "")
|
|
t = _h.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", t))).strip()
|
|
m = re.search(r"\d+\.\d+\.\d+ Magyar English", t)
|
|
return t[m.end():].strip() if m else t
|
|
|
|
|
|
EN_IN_HU = re.compile(r"(?i)\b(the|and|your|you|please|click|with|this|password|settings|open|first|login)\b")
|
|
HU_IN_EN = re.compile(r"[őűŐŰáéíóöúüÁÉÍÓÖÚÜ]")
|
|
|
|
|
|
def slug_of(app):
|
|
try:
|
|
m = re.search(r'(?m)^slug:\s*"?([a-z0-9-]+)', open(f"{CATALOG}/templates/{app}/.felhom.yml").read())
|
|
return m.group(1) if m else app
|
|
except OSError:
|
|
return app
|
|
|
|
|
|
def texts(app, tab, d):
|
|
"""A tab of the app page in both languages, saved whole; English words in the Hungarian text and Hungarian letters
|
|
in the English text are counted (a first filter for a reader, not a verdict)."""
|
|
slug = slug_of(app)
|
|
path = f"/apps/{slug}" + (f"/{tab}" if tab else "")
|
|
hu, en = page_text(path, "hu"), page_text(path, "en")
|
|
open(os.path.join(d, f"page-{tab or 'overview'}-hu.txt"), "w").write(hu)
|
|
open(os.path.join(d, f"page-{tab or 'overview'}-en.txt"), "w").write(en)
|
|
return {"path": path, "hu_len": len(hu), "en_len": len(en),
|
|
"english_words_in_hu": sorted(set(m.lower() for m in EN_IN_HU.findall(hu)))[:15],
|
|
"accented_in_en": len(HU_IN_EN.findall(en)), "same_text": hu == en}
|
|
|
|
|
|
def shots(app, sub, phase, d):
|
|
"""Real Chromium: the install page before the install; after it the guide (phone, HU) and the app's own first
|
|
screen as the household opens it (computer, HU), the setup gate passed by the dashboard session."""
|
|
import subprocess
|
|
slug = slug_of(app)
|
|
out = f"/root/shots/{app}-{phase}"
|
|
if phase == "install":
|
|
steps = [{"goto": f"https://felhom.tester4.test/apps/{slug}/install"}, {"wait": 1500}, {"shot": "install"}]
|
|
spec = [("phone", "hu", steps), ("desktop", "en", steps)]
|
|
else:
|
|
spec = [("phone", "hu", [{"goto": f"https://felhom.tester4.test/apps/{slug}/guide"}, {"wait": 1500}, {"shot": "guide"}]),
|
|
("desktop", "hu", [{"goto": f"https://{sub}.tester4.test/", "until": "load"}, {"wait": 5000}, {"shot": "front"}])]
|
|
res = []
|
|
for size, lang, st in spec:
|
|
env = dict(os.environ, UI_STEPS=json.dumps({"size": size, "lang": lang, "state": "/root/work/state-dash.json",
|
|
"out": out, "steps": st}))
|
|
r = subprocess.run(["bash", os.path.join(os.path.dirname(os.path.abspath(__file__)), "brun.sh"), "--env", "UI_STEPS", "--",
|
|
"/opt/pw/bin/python", "/root/work/ui.py", "-"], capture_output=True, text=True, timeout=240, env=env)
|
|
res.append(" | ".join(l[:160] for l in (r.stdout or r.stderr).splitlines() if l.startswith(("goto", "shot", "STEP")))[:400])
|
|
pull = subprocess.run(["bash", os.path.join(os.path.dirname(os.path.abspath(__file__)), "brpull.sh"), os.path.join(d, "screens")],
|
|
capture_output=True, text=True, timeout=120)
|
|
return res
|
|
|
|
|
|
def front(sub, scheme="https", cookie=None):
|
|
a = ["curl", "-sk", "--max-time", "20", "-o", "/dev/null", "-w", "%{http_code} %{redirect_url}",
|
|
"-H", "Accept: text/html", "-H", f"Host: {sub}.{w.DOMAIN}"]
|
|
if cookie:
|
|
a += ["-H", f"Cookie: {cookie}"]
|
|
base = w.BASE if scheme == "https" else w.BASE.replace("https://", "http://")
|
|
r = w.sh(a + [f"{base}/"])
|
|
return (r.stdout or "").strip()[:200]
|
|
|
|
|
|
def backup_now(rec, say):
|
|
t0 = time.time()
|
|
c, d = w.ctl("POST", "/api/backup/run")
|
|
say(f" [4] Mentés most -> {c} {str(d)[:140]}")
|
|
last = None
|
|
for _ in range(240):
|
|
time.sleep(5)
|
|
c2, st = w.ctl("GET", "/api/backup/status")
|
|
s = (st.get("data") or {}) if isinstance(st, dict) else {}
|
|
last = s
|
|
if not s.get("running") and time.time() - t0 > 10:
|
|
break
|
|
rec["backup"] = {"code": c, "seconds": round(time.time() - t0), "status": json.dumps(last, ensure_ascii=False)[:900]}
|
|
|
|
|
|
def snapshots(app):
|
|
c, d = w.ctl("GET", f"/api/backup/snapshots?stack={app}")
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
if isinstance(data, dict):
|
|
for k in ("snapshots", "items", "restore_points"):
|
|
if isinstance(data.get(k), list):
|
|
return data[k]
|
|
return data if isinstance(data, list) else []
|
|
|
|
|
|
def restore_unit(app, rec, say):
|
|
pts = snapshots(app)
|
|
rec["restore"] = {"points": len(pts)}
|
|
if not pts:
|
|
say(" [6] no restore point offered")
|
|
return
|
|
sid = pts[0].get("short_id") or pts[0].get("id")
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={app}",
|
|
"--data-urlencode", f"snapshot_id={sid}", f"{w.BASE}/backup/restore"])
|
|
head = (r.stdout or "").splitlines()
|
|
loc = [l for l in head if l.lower().startswith("location")]
|
|
say(f" [6] POST /backup/restore -> {head[0] if head else '?'} {loc[:1]}")
|
|
t0, last = time.time(), None
|
|
while time.time() - t0 < 1800:
|
|
time.sleep(5)
|
|
c, d = w.ctl("GET", "/api/backup/restore-status")
|
|
dd = (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
last = dd
|
|
if not dd.get("running") and time.time() - t0 > 10:
|
|
break
|
|
rec["restore"].update({"location": loc[:1], "seconds": round(time.time() - t0),
|
|
"status": json.dumps(last, ensure_ascii=False)[:600], "state": w.stack(app).get("state")})
|
|
|
|
|
|
def leftovers(app, sub):
|
|
out = g(f"echo containers: $(docker ps -a --filter label=com.docker.compose.project={app} --format '{{{{.Names}}}}'); "
|
|
f"echo volumes: $(docker volume ls -q --filter label=com.docker.compose.project={app}); "
|
|
f"echo appyaml: $(ls /opt/docker/stacks/{app}/app.yaml 2>/dev/null); "
|
|
f"echo appdata: $(ls -d /mnt/felhom-drives/*/appdata/{app} 2>/dev/null); "
|
|
f"echo kept: $(ls -d /mnt/felhom-drives/*/kept/{app} 2>/dev/null)")
|
|
res = {k.strip(): v.strip() for k, _, v in (l.partition(":") for l in out.splitlines()) if k}
|
|
res["subdomain"] = front(sub)
|
|
return res
|
|
|
|
|
|
def remove(app, keep, say):
|
|
c1, _ = w.ctl("POST", f"/api/stacks/{app}/stop")
|
|
for _ in range(36):
|
|
time.sleep(5)
|
|
if w.stack(app).get("state") not in ("running", "unhealthy", "degraded", "starting"):
|
|
break
|
|
body = {"remove_hdd_data": not keep, "remove_backups": not keep}
|
|
c, d = w.ctl("POST", f"/api/stacks/{app}/remove", body)
|
|
say(f" [{'5' if keep else '7'}] stop {c1}, remove {'KEEPING data' if keep else 'with data'} -> {c} {str(d)[:200]}")
|
|
for _ in range(24):
|
|
time.sleep(5)
|
|
if not w.stack(app).get("deployed"):
|
|
break
|
|
return c, d
|
|
|
|
|
|
def fresh_verify(fx, sub, tok, say):
|
|
"""Ask the gate again before every read-back: once the box opens an app's SETUP gate (the first admin exists),
|
|
a family-gated app sits behind the FAMILY gate, which answers an old setup-gate cookie with 401 (measured on
|
|
Grimmory 2026-10-24, „setup gate OPENED by probe" 20:23:32)."""
|
|
w.GATE.pop(sub, None)
|
|
return fx.verify(w, sub, tok, say)
|
|
|
|
|
|
def run(app, do_restore=False):
|
|
d = os.path.join(EV, app)
|
|
os.makedirs(d, exist_ok=True)
|
|
log = open(os.path.join(d, "walk.log"), "a")
|
|
|
|
def say(*a):
|
|
line = time.strftime("%H:%M:%SZ", time.gmtime()) + " " + " ".join(str(x) for x in a)
|
|
print(line, flush=True)
|
|
log.write(scrub(line) + "\n")
|
|
log.flush()
|
|
|
|
w.say = lambda *a: say(*a) # the helper's own lines into this app's log
|
|
fx = fxm.FIXTURES.get(app)
|
|
sub = getattr(fx, "sub", app) if fx else app
|
|
rec = {"app": app, "sub": sub, "fixture": type(fx).__name__ if fx else None, "started": time.strftime("%FT%TZ", time.gmtime())}
|
|
w.login()
|
|
t0 = time.time()
|
|
mw = MemWatch(app)
|
|
try:
|
|
# 1. install
|
|
df = w.ctl("GET", f"/api/stacks/{app}/deploy-fields")[1]
|
|
meta = ((df.get("data") or {}).get("metadata") or {}) if isinstance(df, dict) else {}
|
|
rec["deploy_fields"] = [(f.get("env_var"), f.get("type"), bool(f.get("required"))) for f in meta.get("deploy_fields") or []]
|
|
rec["install_page"] = texts(app, "install", d)
|
|
rec["shots_install"] = shots(app, sub, "install", d)
|
|
mw.start()
|
|
t0 = time.time()
|
|
ok = w.deploy(app, sub)
|
|
rec["install"] = {"ok": ok, "seconds_to_deployed": round(time.time() - t0)}
|
|
if not ok:
|
|
rec["verdict_hint"] = "install failed"
|
|
return rec
|
|
t1 = time.time()
|
|
rec["install"]["front_door"] = w.wait_app(sub, "/", want=("200", "301", "302", "303", "307", "308", "401", "403"), tries=90, delay=5)
|
|
rec["install"]["seconds_to_front_door"] = round(time.time() - t0)
|
|
say(f" [1] installed: deployed {rec['install']['seconds_to_deployed']} s, front door {rec['install']['seconds_to_front_door']} s")
|
|
# 2. first login / gate, the household and a stranger; https only
|
|
rec["stranger"] = front(sub)
|
|
rec["http_port_80"] = front(sub, scheme="http")
|
|
rec["guide"] = texts(app, "guide", d)
|
|
rec["overview"] = texts(app, "", d)
|
|
# 3. use: one item through the front door
|
|
tok = None
|
|
if fx:
|
|
try:
|
|
tok = fx.seed(w, sub, say)
|
|
except Exception as e: # a fixture crash is a recorded result
|
|
say(f" [3] seed raised {type(e).__name__}: {str(e)[:200]}")
|
|
rec["seed"] = bool(tok)
|
|
if tok:
|
|
rec["seed_read_now"] = bool(fresh_verify(fx, sub, tok, say))
|
|
else:
|
|
rec["seed"] = None
|
|
say(" [3] no fixture for this app — the front door answering is the smallest honest check")
|
|
rec["shots_after"] = shots(app, sub, "after", d) # after the seed: a visit must not close a sign-up first
|
|
# 4. Mentés most
|
|
backup_now(rec, say)
|
|
rec["snapshots_after_backup"] = len(snapshots(app))
|
|
# 6. restore from the recovery unit (every fifth app)
|
|
if do_restore:
|
|
restore_unit(app, rec, say)
|
|
if tok:
|
|
rec["seed_read_after_restore"] = bool(fresh_verify(fx, sub, tok, say))
|
|
# 5. remove keeping data, install again
|
|
remove(app, keep=True, say=say)
|
|
rec["after_keep_remove"] = leftovers(app, sub)
|
|
os.environ.pop("KEPT", None)
|
|
again = w.ctl("POST", f"/api/stacks/{app}/deploy",
|
|
{"values": dict(w.deploy_values(app, sub, dict(w.GENERATED.get(app) or {})))})
|
|
rec["reinstall_first_answer"] = {"code": again[0], "body": scrub(json.dumps(again[1], ensure_ascii=False))[:700]}
|
|
if again[0] == "409":
|
|
os.environ["KEPT"] = "use"
|
|
w.DEPLOYED_THIS_RUN.discard(app)
|
|
ok2 = w.deploy(app, sub, dict(w.GENERATED.get(app) or {}))
|
|
os.environ.pop("KEPT", None)
|
|
rec["kept"] = {"offered": True, "use_ok": ok2}
|
|
else:
|
|
rec["kept"] = {"offered": False, "plain_install_code": again[0]}
|
|
if again[0] == "202":
|
|
for _ in range(90):
|
|
time.sleep(5)
|
|
st = w.stack(app)
|
|
if st.get("deployed") and (st.get("app_config") or {}).get("pinned_images"):
|
|
break
|
|
w.wait_app(sub, "/", want=("200", "301", "302", "303", "307", "308", "401", "403"), tries=90, delay=5)
|
|
if tok:
|
|
rec["seed_read_after_reinstall"] = bool(fresh_verify(fx, sub, tok, say))
|
|
if not rec["seed_read_after_reinstall"] and not rec["kept"]["offered"]:
|
|
# the household's other route back: the recovery unit kept by „keep my data"
|
|
restore_unit(app, rec, say)
|
|
rec["seed_read_after_unit_restore"] = bool(fresh_verify(fx, sub, tok, say))
|
|
# 8. measured while it ran
|
|
rec["containers"] = limits(app)
|
|
rec["errors"] = error_lines(app, t0 - 5)
|
|
except Exception as e:
|
|
say(f" !! the cycle raised {type(e).__name__}: {str(e)[:300]}")
|
|
rec["exception"] = f"{type(e).__name__}: {str(e)[:300]}"
|
|
finally:
|
|
mw.stop_ev.set()
|
|
rec["mem_peak_mib"] = mw.peak
|
|
rec["mem_anon_peak_mib"] = mw.anon
|
|
# 7. remove with data; nothing left
|
|
if w.stack(app).get("deployed"):
|
|
remove(app, keep=False, say=say)
|
|
rec["after_final_remove"] = leftovers(app, sub)
|
|
rec["total_seconds"] = round(time.time() - t0)
|
|
json.dump(rec, open(os.path.join(d, "record.json"), "w"), ensure_ascii=False, indent=1)
|
|
say(f" == {app} done in {rec['total_seconds']} s")
|
|
return rec
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(sys.argv[1], "--restore" in sys.argv)
|