admin
67053335d1
gates / gates (push) Successful in 13m17s
Every catalog app installed, used, backed up, removed keeping data, reinstalled, restored and removed with data on a fresh box from the public ISO (demo-hp VM 9360, Tester4), endpoint-level. Worst: a reinstall's capture rewrites the kept unit so its restore breaks the app (R-1269, P2), the restore still reports success (R-1270, P2), and „use my kept data" loaded another box's future-dated database (R-1271, P2). Drill catalog fixes for the morning list (n8n usage data; OpenGist 256M, proven by a cgroup probe). VM 9360 and LXC 9361 destroyed; hub host Tester4-aa9466 left for the operator. REPORT-catalog-sweep-2026-10-24.md.
919 lines
48 KiB
Python
919 lines
48 KiB
Python
#!/usr/bin/env python3
|
|
"""box_walk.py — the BOX venue's client (moved into the catalog 2026-09-30 from the audits' walk.py, so the monthly
|
|
re-test `retest-floating.py` has a stable home). ONE app's walk on scratch guest 9202, through the product's own
|
|
endpoints. Configuration by environment: SC (a 0600 scratch dir holding `.ctlpw`, the 9202 dashboard password —
|
|
never committed), EV (where evidence goes), GUEST/BASE/DOMAIN as before.
|
|
|
|
EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses:
|
|
POST /api/stacks/<n>/deploy · POST /api/sync · POST /api/stacks/rescan
|
|
POST /api/stacks/<n>/update · POST /api/stacks/<n>/remove
|
|
and reads GET /api/stacks/<n>. No controller code exists for it.
|
|
|
|
The walk, per `09` §6.4 and the update-night brief §4:
|
|
1 deploy from the DRILL catalog at the LIVE pin
|
|
2 seed through the app's OWN front door (R-156: never a volume, never SQL)
|
|
3 read the seed back <- control C1; a fixture that cannot prove itself proves nothing
|
|
4 „Mentés most"
|
|
5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages
|
|
6 press the guarded Update, record every phase with timestamps
|
|
7 read the seed back through the front door
|
|
8 the four version observables side by side
|
|
9 write the verdict record in `09`'s JSON shape
|
|
|
|
`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`.
|
|
"""
|
|
import argparse, json, os, re, subprocess, sys, time
|
|
from datetime import datetime, timezone
|
|
|
|
SC = os.environ.get('SC', os.path.expanduser('~/.felhom-retest'))
|
|
EV = os.environ.get('EV', os.path.join(SC, 'evidence'))
|
|
DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
|
|
# THE TARGETS (R-892, 2026-10-06): one row per box the walk may drive. `hp` is the box's NAME for the guards
|
|
# (upgrade_fixtures_box.BOX_ADMIN_SEED_GUESTS keys on (hp, guest)); `ssh` is how its Proxmox host is reached, with an
|
|
# optional `jump` (never an edit of DooPlex's ~/.ssh/config); `guest` the customer LXC; `base` and `domain` its dashboard.
|
|
# TARGET=9202 (default) scratch guest 9202 on demo-hp
|
|
# TARGET=9201 demo-hp's hub-enabled guest (the mail proof) — household-shaped, never seeded through an admin
|
|
# TARGET=tester-1 the Tester 1 box: VM 341 on the HP box (`09` §3 decision 158), its node `felhom` at
|
|
# 192.168.0.154, guest 9201 (DHCP — read at start) for felhom.enkicsifelhom.hu (identity matched
|
|
# 2026-10-06: the agent's report `host.node=felhom` = the VM's certificate; the guest answers
|
|
# that domain, the other one 404 — `felhom.eu/documentation/audits/readback-2026-10-07/`)
|
|
# GUEST=<vmid> still selects a demo-hp row (the old switch), unless TARGET is set.
|
|
TARGETS = {
|
|
# 9202 takes its address by DHCP (.114 until 2026-10-19, then .121), so its row names none: it is read from the
|
|
# guest at start (resolve_base below); BASE= still wins.
|
|
"9202": {"hp": "demo-hp", "ssh": "demo-hp", "jump": None, "guest": "9202",
|
|
"base": None, "domain": "enkisfelhom.hu"},
|
|
"9201": {"hp": "demo-hp", "ssh": "demo-hp", "jump": None, "guest": "9201",
|
|
"base": "https://192.168.0.155", "domain": "enkisfelhom.hu"},
|
|
# catalog sweep 2026-10-24: the fresh Tester4 box (nested VM 9360 on demo-hp, host 192.168.0.138, DHCP) — SWEEP COPY
|
|
"sweep": {"hp": "sweep", "ssh": "root@192.168.0.138", "jump": None, "guest": "9201",
|
|
"base": None, "domain": "tester4.test"},
|
|
"tester-1": {"hp": "tester-1", "ssh": "root@192.168.0.154", "jump": "demo-hp", "guest": "9201",
|
|
"base": None, "domain": "enkicsifelhom.hu"}, # DHCP: .101 until 2026-10-23, then .148 — read
|
|
}
|
|
TARGET = os.environ.get("TARGET") or os.environ.get("GUEST", "9202")
|
|
if TARGET not in TARGETS:
|
|
raise SystemExit(f"box_walk: unknown TARGET {TARGET!r} — one of {sorted(TARGETS)}")
|
|
_T = TARGETS[TARGET]
|
|
GUEST = _T["guest"]
|
|
|
|
|
|
def ssh_args():
|
|
"""The ssh argv prefix that reaches the target's Proxmox host (a jump when the row names one)."""
|
|
a = ["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new"]
|
|
if _T["jump"]:
|
|
a += ["-J", _T["jump"]]
|
|
return a + [_T["ssh"]]
|
|
|
|
LOG = []
|
|
|
|
|
|
def resolve_base():
|
|
"""A DHCP guest's dashboard address, read from the guest itself over the host's pct (no guessing)."""
|
|
r = subprocess.run(ssh_args() + ["pct", "exec", _T["guest"], "--", "ip", "-4", "-o",
|
|
"addr", "show", "eth0"], capture_output=True, text=True, timeout=60)
|
|
m = re.search(r"inet (\d+\.\d+\.\d+\.\d+)/", r.stdout)
|
|
if not m:
|
|
raise SystemExit("box_walk: could not read guest %s's address (%s) — set BASE=https://<ip>"
|
|
% (_T["guest"], (r.stderr or r.stdout).strip()[-120:]))
|
|
return "https://" + m.group(1)
|
|
|
|
|
|
BASE = os.environ.get("BASE") or _T["base"] or resolve_base()
|
|
DOMAIN = os.environ.get("DOMAIN") or _T["domain"]
|
|
HOSTHDR = f"Host: felhom.{DOMAIN}"
|
|
HP = _T["hp"]
|
|
|
|
|
|
def say(*a):
|
|
line = " ".join(str(x) for x in a)
|
|
ts = datetime.now().strftime("%H:%M:%S")
|
|
print(f"{ts} {line}", flush=True)
|
|
LOG.append(f"{ts} {line}")
|
|
|
|
|
|
def sh(args, timeout=300, inp=None):
|
|
try:
|
|
return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp)
|
|
except (subprocess.TimeoutExpired, OSError) as e:
|
|
return subprocess.CompletedProcess(args, 124, "", f"{e}")
|
|
|
|
|
|
def guest(script, timeout=600):
|
|
"""Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting)."""
|
|
# ONE TEMP FILE PER CALL (night 2026-09-23): the shared /tmp/w<guest>.sh swapped scripts under
|
|
# two concurrent walks (memory: guest-helper-shares-one-tmp-file).
|
|
import secrets as _s
|
|
t = f"/tmp/w{GUEST}-{os.getpid()}-{_s.token_hex(4)}.sh"
|
|
r = sh(ssh_args() + [
|
|
f"export LC_ALL=C; cat > {t}; pct push {GUEST} {t} {t} >/dev/null 2>&1; "
|
|
f"pct exec {GUEST} -- bash {t}; pct exec {GUEST} -- rm -f {t}; rm -f {t}"],
|
|
timeout=timeout, inp=script)
|
|
return r.stdout or ""
|
|
|
|
|
|
def login():
|
|
pw = open(f"{SC}/.ctlpw").read().strip()
|
|
sh(["curl", "-sk", "-D", f"{SC}/hdr{os.getpid()}.txt", "-o", "/dev/null", "-H", HOSTHDR,
|
|
"-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"])
|
|
h = open(f"{SC}/hdr{os.getpid()}.txt").read()
|
|
m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I)
|
|
if not m:
|
|
sys.exit("login failed: no session cookie")
|
|
open(f"{SC}/sess{os.getpid()}.txt", "w").write(m.group(0))
|
|
r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"])
|
|
c = re.search(r'<meta name="csrf-token" content="([^"]+)"', r.stdout or "")
|
|
if not c:
|
|
sys.exit("login failed: no csrf token")
|
|
open(f"{SC}/csrf{os.getpid()}.txt", "w").write(c.group(1))
|
|
|
|
|
|
def ctl(method, path, data=None, raw=False, tries=2):
|
|
"""One controller API call. Re-logs in once on a 302/401 — the controller's session store is
|
|
in memory, so any controller restart during the night invalidates it silently."""
|
|
for attempt in range(tries):
|
|
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip()
|
|
args = ["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", "-w", "\n%{http_code}"]
|
|
if method != "GET":
|
|
args += ["-H", f"X-CSRF-Token: {csrf}", "-H", "Content-Type: application/json",
|
|
"-X", method]
|
|
if data is not None:
|
|
args += ["--data", json.dumps(data)]
|
|
args.append(f"{BASE}{path}")
|
|
r = sh(args)
|
|
body, _, code = (r.stdout or "").rpartition("\n")
|
|
if code.strip() in ("302", "401") and attempt + 1 < tries:
|
|
login()
|
|
continue
|
|
if raw:
|
|
return code.strip(), body
|
|
try:
|
|
return code.strip(), json.loads(body)
|
|
except Exception:
|
|
return code.strip(), {"_raw": body[:600]}
|
|
return code.strip(), {"_raw": body[:600]}
|
|
|
|
|
|
def page(path):
|
|
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
|
r = sh(["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", f"{BASE}{path}"])
|
|
return r.stdout or ""
|
|
|
|
|
|
GATE = {} # sub -> the felhom_gate cookie the household's browser would hold (setup gate, v0.280.0)
|
|
|
|
|
|
def _loc(out):
|
|
m = re.search(r"(?im)^location:\s*(\S+)", out or "")
|
|
return m.group(1) if m else ""
|
|
|
|
|
|
def gate_cookie(sub):
|
|
"""Pass the setup gate (`09` decision 46) the way the HOUSEHOLD does: the app host redirects to the
|
|
dashboard's /__gate/start, which (with the dashboard session) redirects back to the app host's
|
|
/__felhom_gate/cb, which sets `felhom_gate`. Never printed."""
|
|
import urllib.parse
|
|
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {sub}.{DOMAIN}", f"{BASE}/"])
|
|
loc = _loc(r.stdout)
|
|
st = re.match(r"HTTP/\S+ (\d+)", r.stdout or "")
|
|
if not loc and (not st or st.group(1) in ("404", "502", "503", "504")):
|
|
# the app is not routed at this moment (a restart, an update's stop): NOT the same as "not gated" — a cached ""
|
|
# here sent every later call past nothing and read the gate's 401 as the app's (2026-10-02, grim-step)
|
|
say(f" gate: {sub} not routed right now ({st.group(1) if st else 'no answer'}) — not cached, asked again next call")
|
|
return ""
|
|
if "/__family/start" in loc: # v0.287.0: the FAMILY gate — the household's dashboard session vouches, as for
|
|
return family_cookie(sub, loc, sess) # the setup gate (decisions 63/64)
|
|
if "/__gate/start" not in loc:
|
|
GATE[sub] = ""
|
|
return "" # not gated (open, or no gate for this app)
|
|
u = urllib.parse.urlsplit(loc)
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", "-H", f"Cookie: {sess}",
|
|
f"{BASE}{u.path}?{u.query}"])
|
|
loc = _loc(r.stdout)
|
|
u = urllib.parse.urlsplit(loc)
|
|
if "/__felhom_gate/cb" not in u.path:
|
|
say(f" gate: the dashboard did not hand back a callback for {sub} ({loc[:80]})")
|
|
return ""
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", f"{BASE}{u.path}?{u.query}"])
|
|
m = re.search(r"(?im)^set-cookie:\s*(felhom_gate=[^;\r\n]+)", r.stdout or "")
|
|
GATE[sub] = m.group(1) if m else ""
|
|
say(f" gate: {sub} is gated — passed as the household (cookie {'set' if GATE[sub] else 'NOT set'})")
|
|
return GATE[sub]
|
|
|
|
|
|
def family_cookie(sub, loc, sess):
|
|
"""Pass the FAMILY gate (controller >= 0.287.0) as the household: /__family/start on the dashboard host with the
|
|
dashboard session → the app host's /__felhom_gate/fcb → `felhom_famgate`. Never printed."""
|
|
import urllib.parse
|
|
u = urllib.parse.urlsplit(loc)
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", "-H", f"Cookie: {sess}",
|
|
f"{BASE}{u.path}?{u.query}"])
|
|
u = urllib.parse.urlsplit(_loc(r.stdout))
|
|
if "/__felhom_gate/fcb" not in u.path:
|
|
say(f" family gate: the dashboard did not hand back a callback for {sub}")
|
|
GATE[sub] = ""
|
|
return ""
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", f"{BASE}{u.path}?{u.query}"])
|
|
m = re.search(r"(?im)^set-cookie:\s*(felhom_famgate=[^;\r\n]+)", r.stdout or "")
|
|
GATE[sub] = m.group(1) if m else ""
|
|
say(f" family gate: {sub} is family-gated — passed as the household (cookie {'set' if GATE[sub] else 'NOT set'})")
|
|
return GATE[sub]
|
|
|
|
|
|
def app_curl(sub, path, *extra, method=None, data=None, timeout=45, _retry=True):
|
|
"""A call to the APP's own front door on 9202 — the household's route, not ours. Carries the setup
|
|
gate's cookie when the app is gated, merged into a fixture's own Cookie header (never a second one)."""
|
|
raw = list(extra)
|
|
gc = GATE[sub] if sub in GATE else gate_cookie(sub)
|
|
ext = list(raw)
|
|
if gc:
|
|
merged = False
|
|
for i, a in enumerate(ext):
|
|
if isinstance(a, str) and a.lower().startswith("cookie:") and i > 0 and ext[i - 1] == "-H":
|
|
ext[i] = a + "; " + gc
|
|
merged = True
|
|
if not merged:
|
|
ext = ["-H", f"Cookie: {gc}"] + ext
|
|
args = ["curl", "-sSk", "--max-time", str(timeout), "-H", f"Host: {sub}.{DOMAIN}",
|
|
"-w", "\n%{http_code} %{redirect_url}"]
|
|
if method:
|
|
args += ["-X", method]
|
|
if data is not None:
|
|
args += ["--data-binary", "@-"]
|
|
args += ext + [f"{BASE}{path}"]
|
|
r = sh(args, timeout=timeout + 30, inp=data)
|
|
body, _, tail = (r.stdout or "").rpartition("\n")
|
|
code, _, redir = tail.strip().partition(" ")
|
|
if _retry and ("/__gate/start" in redir or "/__family/start" in redir):
|
|
GATE.pop(sub, None) # the gate cookie expired or was never taken — log in as the household again
|
|
return app_curl(sub, path, *raw, method=method, data=data, timeout=timeout, _retry=False)
|
|
return r.returncode, code.strip(), body
|
|
|
|
|
|
def stack(name):
|
|
_, d = ctl("GET", f"/api/stacks/{name}")
|
|
return (d.get("data") or {}) if isinstance(d, dict) else {}
|
|
|
|
|
|
def wait_app(sub, path="/", want=("200", "302", "303", "401", "403"), tries=60, delay=5):
|
|
"""Settling says the container runs; this says the APP answers. Not the same thing."""
|
|
last = None
|
|
for _ in range(tries):
|
|
rc, code, _ = app_curl(sub, path, timeout=15)
|
|
last = (rc, code)
|
|
if rc == 0 and code in want:
|
|
return True
|
|
time.sleep(delay)
|
|
say(f" app never answered on {sub}{path} (last rc={last[0]} code={last[1]})")
|
|
return False
|
|
|
|
|
|
# ------------------------------------------------------------------ the walk
|
|
|
|
|
|
DRIVE_ROOT = os.environ.get("DRIVE_ROOT", "/mnt/felhom-drives/scratch_hdd") # SWEEP COPY: env override
|
|
DRIVE = DRIVE_ROOT + "/userdata"
|
|
|
|
# What THIS run generated for a deploy, per app. Deploy secrets are ENCRYPTED AT REST in
|
|
# `app.yaml` (`ENC:…`), which is right and which means a fixture cannot read an app's admin
|
|
# password back off the box — the household sees it once. So the value the harness itself
|
|
# generated is kept here for the life of the run, and nowhere else.
|
|
GENERATED = {}
|
|
# The apps THIS run installed (deploy() adds one on its 202). The test-box admin seed refuses an app not in it: an app
|
|
# the walk merely reused may be a household's (upgrade_fixtures_box.box_admin_seed_allowed, R-890).
|
|
DEPLOYED_THIS_RUN = set()
|
|
|
|
|
|
def deploy_values(name, sub, given=None):
|
|
"""Fill EVERY required deploy field the way the wizard would, by asking the box what this app
|
|
asks for — `GET /api/stacks/<n>/deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN.
|
|
|
|
Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because
|
|
a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password
|
|
(grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only
|
|
reason this was safe to discover by running it (live-probes rule).
|
|
|
|
A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on
|
|
the scratch drive first — the same act the drive browser performs for a household.
|
|
"""
|
|
code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields")
|
|
fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
|
|
values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub}
|
|
values.update(given or {}) # a caller's own value wins, and no drive folder is made for it (2026-10-16: a passed
|
|
made = [] # HDD_PATH still got DRIVE/<app> created, an empty folder nobody asked for)
|
|
for f in fields:
|
|
ev, ty = f.get("env_var"), f.get("type")
|
|
if ev in values:
|
|
continue
|
|
# `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses
|
|
# when the caller sends none, deliberately ("the user needs to know their password"),
|
|
# while `.felhom.yml` declares `required: false` and the API serves that verbatim. A
|
|
# caller that trusts the contract gets a 400. Measured tonight on grafana; filed.
|
|
if not f.get("required") and ty != "password":
|
|
continue # the controller generates the optional secrets itself
|
|
if ty == "path" and ev == "HDD_PATH":
|
|
# R-839 (controller): the deploy API refuses an HDD_PATH that is a folder INSIDE a registered drive —
|
|
# the app's data place is the drive itself. Measured 2026-10-24: DRIVE/<app> came back 409.
|
|
values[ev] = DRIVE_ROOT
|
|
elif ty == "path":
|
|
p = f"{DRIVE}/{name}"
|
|
values[ev] = p
|
|
made.append(p)
|
|
elif ty in ("secret", "password"):
|
|
import secrets as _s
|
|
values[ev] = "Drill-" + _s.token_hex(12)
|
|
GENERATED.setdefault(name, {})[ev] = values[ev]
|
|
elif f.get("default"):
|
|
values[ev] = f["default"]
|
|
else:
|
|
values[ev] = f"drill-{name}"
|
|
if made:
|
|
guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made))
|
|
say(f" [1] made the drive paths this app requires: {made}")
|
|
extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")]
|
|
if extra:
|
|
say(f" [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}")
|
|
return values
|
|
|
|
|
|
def deploy(name, sub, extra_values=None):
|
|
st = stack(name)
|
|
if st.get("deployed"):
|
|
say(f" [1] {name} already deployed — reusing")
|
|
return True
|
|
values = deploy_values(name, sub, extra_values)
|
|
body = {"values": values}
|
|
if os.environ.get("KEPT"): # decision 36: the household's answer when the drive holds old data ("fresh" moves it aside, deletes nothing)
|
|
body["kept_data"] = os.environ["KEPT"]
|
|
code, d = ctl("POST", f"/api/stacks/{name}/deploy", body)
|
|
say(f" [1] deploy -> {code} {str(d)[:120]}")
|
|
if code != "202":
|
|
return False
|
|
DEPLOYED_THIS_RUN.add(name)
|
|
# WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the
|
|
# container `healthy` while the controller's own state read `unhealthy` — a gate on `running`
|
|
# alone therefore times out on an app that is up. The state is RECORDED rather than required;
|
|
# the real gate is the fixture's own `wait_app`, which asks whether the APP answers.
|
|
seen = None
|
|
for _ in range(90):
|
|
time.sleep(5)
|
|
st = stack(name)
|
|
seen = st.get("state")
|
|
# `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21:
|
|
# tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm
|
|
# read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the
|
|
# deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark
|
|
# (`runComposeDeploy` writes it), so that is what to wait for.
|
|
pins = (st.get("app_config") or {}).get("pinned_images")
|
|
if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"):
|
|
say(f" [1] deployed, controller state={seen}, "
|
|
f"pinned={(st.get('app_config') or {}).get('pinned_images')}")
|
|
if seen != "running":
|
|
say(f" [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, "
|
|
f"not treated as a failure; the fixture's front-door wait is the real gate")
|
|
return True
|
|
say(f" [1] never became deployed (last controller state={seen!r})")
|
|
return False
|
|
|
|
|
|
def backup_now(name):
|
|
"""R-648 (2026-09-23): NO whole-box „Mentés most" from a drill, ever.
|
|
|
|
`POST /api/backup/run` is the only backup endpoint and it is WHOLE-BOX: on 9201 it stopped and
|
|
restarted 9 of 10 standing apps twice, and on 9202 it broke a deploy in flight (R-634). The product
|
|
has NO per-app backup endpoint (router.go: /backup/run, /backup/tier2 only); the per-app backup
|
|
exists only inside the guarded update, whose `backing-up` phase calls RunAppBackupNow for the one
|
|
app. So this presses nothing: the update takes the throwaway app's own backup, and says so in its
|
|
phase list. A seed written "after the backup" is therefore written before the update's own backup
|
|
— the undo's last-second copy is still the one that must bring it back."""
|
|
say(f" [4] backup press SKIPPED for {name} (R-648: whole-box only; the update's backing-up phase backs up {name} alone)")
|
|
return None
|
|
|
|
def drill_bump(app, frm, to, service_hint=None):
|
|
"""Serialised across concurrent walks: one git working tree, one lock."""
|
|
import fcntl
|
|
with open(f"{SC}/drill.lock", "w") as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
sh(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
|
return _drill_bump(app, frm, to, service_hint)
|
|
|
|
|
|
def _drill_bump(app, frm, to, service_hint=None):
|
|
"""Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates).
|
|
|
|
`frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in
|
|
two images (adventurelog's backend and frontend) moves both in one edge, while its engine
|
|
sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move
|
|
every service that carries the app's own version and no others.
|
|
"""
|
|
comp = f"{DRILL}/templates/{app}/docker-compose.yml"
|
|
fy = f"{DRILL}/templates/{app}/.felhom.yml"
|
|
s = open(comp).read()
|
|
froms = [x.strip() for x in frm.split(",") if x.strip()]
|
|
tos = [x.strip() for x in to.split(",") if x.strip()]
|
|
if len(froms) != len(tos):
|
|
say(f" [5] from/to lists differ in length: {froms} vs {tos}")
|
|
return None
|
|
for f1, t1 in zip(froms, tos):
|
|
if f"image: {f1}" not in s:
|
|
say(f" [5] FROM ref not found in compose: {f1}")
|
|
return None
|
|
s = s.replace(f"image: {f1}", f"image: {t1}")
|
|
open(comp, "w").write(s)
|
|
f = open(fy).read()
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M)
|
|
open(fy, "w").write(f)
|
|
sh(["git", "-C", DRILL, "add", "-A"])
|
|
sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"])
|
|
r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120)
|
|
h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip()
|
|
say(f" [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})")
|
|
refresh_box_mirror()
|
|
return h
|
|
|
|
|
|
BOX_VOL = "/var/lib/docker/volumes/felhom-controller-data/_data" # the controller's data volume, guest path
|
|
BOX_CVOL = "/opt/docker/felhom-controller" # the same, inside the controller container
|
|
MIRROR_URL = f"file://{BOX_CVOL}/drill-catalog.git"
|
|
|
|
|
|
def refresh_box_mirror():
|
|
"""A box pointed at the drill through its LOCAL MIRROR (R-992: no catalog credential on the box — the mirror method of
|
|
felhom.eu audits/burndown-3-2026-10-17/tools/repoint.py) sees a drill commit only after the mirror is loaded again.
|
|
Without this a drill commit reaches Gitea and never the box, and a walk then presses Update on nothing. A box on any
|
|
other catalog URL is left alone (it fetches the drill itself)."""
|
|
conf = guest(f"grep -E '^\\s+repo_url:' {BOX_VOL}/controller.yaml")
|
|
if MIRROR_URL not in conf:
|
|
return False
|
|
import secrets as _s
|
|
b = f"/tmp/drill-walk-{os.getpid()}-{_s.token_hex(4)}.bundle"
|
|
sh(["git", "-C", DRILL, "bundle", "create", b, "main"])
|
|
sh(["scp", "-q", "-o", "StrictHostKeyChecking=accept-new"] + (["-J", _T["jump"]] if _T["jump"] else [])
|
|
+ [b, f"{_T['ssh']}:{b}"], timeout=120)
|
|
sh(ssh_args() + [f"pct push {GUEST} {b} {BOX_VOL}/{os.path.basename(b)} && rm -f {b}"], timeout=120)
|
|
out = guest(f"""docker exec felhom-controller sh -c 'git -C {BOX_CVOL}/drill-catalog.git fetch -q {BOX_CVOL}/{os.path.basename(b)} +main:main && rm -f {BOX_CVOL}/{os.path.basename(b)}; git -C {BOX_CVOL}/drill-catalog.git log -1 --format="%h" main'""")
|
|
os.remove(b)
|
|
say(f" [5] the box's drill mirror now at {out.strip()}")
|
|
return True
|
|
|
|
|
|
def sync_rescan(expect_app=None, expect_ref=None, tries=12, delay=5):
|
|
"""Sync, rescan, and — when told what to expect — WAIT FOR THE BADGE TO CATCH UP.
|
|
|
|
R-607: `POST /api/sync` answers "nincs valtozas" while the catalog HAS moved, and
|
|
`catalog_images` stays stale until a separate rescan. Tonight showed the rescan alone is not
|
|
enough either: mealie's badge read "Naprakesz" seconds after its bump was pushed, and the
|
|
Update that followed moved nothing and still reported "Frissitve". So when the caller knows
|
|
which reference should appear, this polls for it and SAYS HOW LONG IT TOOK — which is the
|
|
NUMBER R-607 asks for and has never had.
|
|
"""
|
|
t0 = time.time()
|
|
ctl("POST", "/api/sync")
|
|
time.sleep(2)
|
|
ctl("POST", "/api/stacks/rescan")
|
|
time.sleep(2)
|
|
if not expect_app or not expect_ref:
|
|
return None
|
|
for i in range(tries):
|
|
cat = stack(expect_app).get("catalog_images") or {}
|
|
if expect_ref in cat.values():
|
|
waited = round(time.time() - t0, 1)
|
|
if i:
|
|
say(f" [sync] the badge needed {waited}s and {i+1} sync+rescan rounds to catch up "
|
|
f"to {expect_ref} — R-607's window, measured")
|
|
return waited
|
|
time.sleep(delay)
|
|
ctl("POST", "/api/sync")
|
|
time.sleep(1)
|
|
ctl("POST", "/api/stacks/rescan")
|
|
say(f" [sync] the badge NEVER caught up to {expect_ref} in {round(time.time()-t0,1)}s — "
|
|
f"catalog_images = {stack(expect_app).get('catalog_images')}")
|
|
return None
|
|
|
|
|
|
def badges(name):
|
|
out = {}
|
|
for lang, suffix in (("hu", ""), ("en", "?lang=en")):
|
|
h = page(f"/apps/{name}{suffix}")
|
|
m = re.findall(r'<span class="tag tag-[^"]*"[^>]*title="([^"]*)"[^>]*>([^<]*)<', h)
|
|
out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3]
|
|
return out
|
|
|
|
|
|
def press_update(name, poll=1.0, cap_s=1800):
|
|
code, d = ctl("POST", f"/api/stacks/{name}/update")
|
|
say(f" [6] Update -> {code} {str(d)[:220]}")
|
|
if code not in ("202", "200"):
|
|
return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0}
|
|
phases, seen, t0 = [], None, time.time()
|
|
while time.time() - t0 < cap_s:
|
|
st = stack(name)
|
|
ph = st.get("update_phase")
|
|
if ph != seen:
|
|
seen = ph
|
|
rec = {"t": round(time.time() - t0, 1), "phase": ph,
|
|
"label": st.get("update_phase_label"), "updating": st.get("updating"),
|
|
"error": st.get("update_error"), "hold": st.get("hold_reason")}
|
|
phases.append(rec)
|
|
say(f" +{rec['t']:>6.1f}s phase={ph} label={rec['label']} "
|
|
f"err={rec['error']} hold={rec['hold']}")
|
|
if not st.get("updating") and ph in ("done", "failed", "undone", None) and time.time() - t0 > 3:
|
|
break
|
|
time.sleep(poll)
|
|
st = stack(name)
|
|
return {"accepted": True, "http": code, "phases": phases,
|
|
"duration_s": round(time.time() - t0, 1),
|
|
"final_phase": st.get("update_phase"), "update_error": st.get("update_error"),
|
|
"hold_reason": st.get("hold_reason"), "state": st.get("state")}
|
|
|
|
|
|
def observables(name):
|
|
st = stack(name)
|
|
ac = st.get("app_config") or {}
|
|
live = guest(f"""
|
|
grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//'
|
|
echo '---inspect---'
|
|
for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do
|
|
echo -n "$c "; docker inspect "$c" --format '{{{{.Config.Image}}}} running={{{{.State.Running}}}} restarts={{{{.RestartCount}}}}'
|
|
done
|
|
""")
|
|
a, _, b = live.partition("---inspect---")
|
|
return {
|
|
"pinned_images": ac.get("pinned_images"),
|
|
"installed_images": {k: (v.get("ref") if isinstance(v, dict) else v)
|
|
for k, v in (ac.get("installed_images") or {}).items()},
|
|
"catalog_images": st.get("catalog_images"),
|
|
"live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()],
|
|
"docker_inspect": [x for x in b.strip().splitlines() if x.strip()],
|
|
}
|
|
|
|
|
|
def app_logs(name, lines=400):
|
|
"""The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is
|
|
one string with escaped newlines — a scan over the envelope sees a single enormous line and
|
|
finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96
|
|
rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines."""
|
|
code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}")
|
|
if isinstance(d, dict):
|
|
data = d.get("data")
|
|
if isinstance(data, dict) and isinstance(data.get("logs"), str):
|
|
return data["logs"]
|
|
if isinstance(d.get("_raw"), str):
|
|
return d["_raw"]
|
|
return str(d)
|
|
|
|
|
|
def write_verdict(rec, appdir):
|
|
os.makedirs(appdir, exist_ok=True)
|
|
p = os.path.join(appdir, "verdict.json")
|
|
json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False)
|
|
say(f" [9] verdict {rec['verdict']} -> {p}")
|
|
|
|
|
|
def remove(name):
|
|
"""Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint
|
|
refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up."""
|
|
c1, d1 = ctl("POST", f"/api/stacks/{name}/stop")
|
|
say(f" [X] stop -> {c1} {str(d1)[:100]}")
|
|
for _ in range(24):
|
|
time.sleep(5)
|
|
if stack(name).get("state") != "running":
|
|
break
|
|
code, d = ctl("POST", f"/api/stacks/{name}/remove",
|
|
{"remove_hdd_data": True, "remove_backups": True})
|
|
say(f" [X] remove (with drive data) -> {code} {str(d)[:160]}")
|
|
if code == "409":
|
|
# R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive
|
|
# path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202
|
|
# `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits
|
|
# this. The household's other choice — remove the app, KEEP the data — is accepted, and the
|
|
# harness takes it, then tidies its own directory by name at teardown.
|
|
say(" [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)"
|
|
" — removing the app and KEEPING the drive data instead")
|
|
code, d = ctl("POST", f"/api/stacks/{name}/remove",
|
|
{"remove_hdd_data": False, "remove_backups": True})
|
|
say(f" [X] remove (keeping drive data) -> {code} {str(d)[:160]}")
|
|
time.sleep(5)
|
|
st = stack(name)
|
|
# The stack folder itself stays: it is the catalog's template copy (every app has one). An install leaves its record,
|
|
# app.yaml — that is the leftover to look for (2026-10-24: the folder alone read as a leftover after a clean remove).
|
|
left = guest(f"ls -d /opt/docker/stacks/{name}/app.yaml 2>/dev/null; "
|
|
f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'")
|
|
say(f" [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}")
|
|
return code
|
|
|
|
|
|
def app_env(name, key):
|
|
"""Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the
|
|
app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller
|
|
shows them the same value. Data still goes in through the app's own front door."""
|
|
out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1")
|
|
if ":" in out:
|
|
return out.split(":", 1)[1].strip().strip('"').strip("'")
|
|
return ""
|
|
|
|
|
|
def snapshots(name):
|
|
"""The restorable copies the backups page offers for this app."""
|
|
code, d = ctl("GET", f"/api/backup/snapshots?stack={name}")
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
if isinstance(data, dict):
|
|
for k in ("snapshots", "items", "restore_points"):
|
|
if isinstance(data.get(k), list):
|
|
return data[k]
|
|
return data if isinstance(data, list) else []
|
|
|
|
|
|
def restore(name, snapshot_id=None, wait_s=1200):
|
|
"""The household's own way out: the „Visszaállítás a mentésből" button on the backups page.
|
|
|
|
A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`,
|
|
`snapshot_id` — because that is the button the sentence tells them to press.
|
|
"""
|
|
snaps = snapshots(name)
|
|
if snapshot_id is None:
|
|
if not snaps:
|
|
say(f" [R] no restorable copy offered for {name}")
|
|
return {"ok": False, "why": "no snapshot offered", "snapshots": snaps}
|
|
first = snaps[0]
|
|
snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id")
|
|
say(f" [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)")
|
|
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}",
|
|
"-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}",
|
|
"--data-urlencode", f"stack_name={name}",
|
|
"--data-urlencode", f"snapshot_id={snapshot_id}",
|
|
f"{BASE}/backup/restore"], timeout=180)
|
|
head = (r.stdout or "").split("\n")[0].strip()
|
|
loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")]
|
|
say(f" [R] POST /backup/restore -> {head} {loc[:1]}")
|
|
t0 = time.time()
|
|
last = None
|
|
while time.time() - t0 < wait_s:
|
|
code, d = ctl("GET", "/api/backup/restore-status")
|
|
dd = d.get("data") or {}
|
|
cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message"))
|
|
if cur != last:
|
|
say(f" +{round(time.time()-t0,1):>6.1f}s restore {cur}")
|
|
last = cur
|
|
if not dd.get("running", False) and time.time() - t0 > 5:
|
|
break
|
|
time.sleep(2)
|
|
st = stack(name)
|
|
say(f" [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} "
|
|
f"phase={st.get('update_phase')}")
|
|
return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps,
|
|
"http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1),
|
|
"state_after": st.get("state"), "hold_after": st.get("hold_reason"),
|
|
"observables_after": observables(name)}
|
|
|
|
|
|
# ------------------------------------------------------------------ the security-fix walk (D39 = B)
|
|
|
|
SECURITY_WALK_VERSION = 1
|
|
SOAK_DEFAULT_S = 600
|
|
|
|
|
|
def soak(name, seconds, interval=10):
|
|
"""The box's memory soak (`09` §3 decision 255): every `interval` s for `seconds`, per container of the app, the
|
|
app's own memory (cgroup `anon`, decision 22 — the cgroup peak counts the file cache) against its limit, the
|
|
cgroup peak, the OOM-kill counter and the restart count. One guest call; the guest prints one JSON line a sample."""
|
|
script = f"""
|
|
set -u
|
|
ids=$(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.ID}}}}')
|
|
end=$(( $(date +%s) + {int(seconds)} ))
|
|
while :; do
|
|
for id in $ids; do
|
|
full=$(docker inspect -f '{{{{.Id}}}}' $id); nm=$(docker inspect -f '{{{{.Name}}}}' $id | tr -d /)
|
|
rc=$(docker inspect -f '{{{{.RestartCount}}}}' $id); run=$(docker inspect -f '{{{{.State.Running}}}}' $id)
|
|
start=$(docker inspect -f '{{{{.State.StartedAt}}}}' $id); oomk=$(docker inspect -f '{{{{.State.OOMKilled}}}}' $id)
|
|
cg=""; for c in /sys/fs/cgroup/system.slice/docker-$full.scope /sys/fs/cgroup/docker/$full; do [ -d "$c" ] && cg=$c; done
|
|
anon=0; mx=max; pk=0; ok=0
|
|
if [ -n "$cg" ]; then
|
|
anon=$(awk '$1=="anon"{{print $2}}' $cg/memory.stat); mx=$(cat $cg/memory.max)
|
|
pk=$(cat $cg/memory.peak 2>/dev/null || echo 0); ok=$(awk '$1=="oom_kill"{{print $2}}' $cg/memory.events)
|
|
fi
|
|
printf '{{"t":%s,"c":"%s","anon":%s,"max":"%s","peak":%s,"oom_kill":%s,"restarts":%s,"running":"%s","started":"%s","oomkilled":"%s","cg":"%s"}}\\n' \\
|
|
"$(date +%s)" "$nm" "${{anon:-0}}" "$mx" "${{pk:-0}}" "${{ok:-0}}" "$rc" "$run" "$start" "$oomk" "$cg"
|
|
done
|
|
[ "$(date +%s)" -ge "$end" ] && break
|
|
sleep {int(interval)}
|
|
done
|
|
"""
|
|
out = guest(script, timeout=int(seconds) + 300)
|
|
samples = []
|
|
for line in out.splitlines():
|
|
try:
|
|
samples.append(json.loads(line))
|
|
except ValueError:
|
|
continue
|
|
per = {}
|
|
for s in samples:
|
|
c = per.setdefault(s["c"], {"samples": 0, "anon_peak": 0, "cg_peak": 0, "limit": None, "oom_kill": [],
|
|
"restarts": [], "running": [], "started": set(), "oomkilled": False, "cgroup": s["cg"]})
|
|
c["samples"] += 1
|
|
c["anon_peak"] = max(c["anon_peak"], int(s["anon"]))
|
|
c["cg_peak"] = max(c["cg_peak"], int(s["peak"]))
|
|
c["limit"] = int(s["max"]) if str(s["max"]).isdigit() else None
|
|
c["oom_kill"].append(int(s["oom_kill"]))
|
|
c["restarts"].append(int(s["restarts"]))
|
|
c["running"].append(s["running"] == "true")
|
|
c["started"].add(s["started"])
|
|
c["oomkilled"] = c["oomkilled"] or s["oomkilled"] == "true"
|
|
res = {"soak_s": int(seconds), "interval_s": interval, "samples": len(samples), "containers": {}}
|
|
kills = restarts = 0
|
|
peaks = []
|
|
for nm, c in per.items():
|
|
k = (max(c["oom_kill"]) - min(c["oom_kill"])) if c["oom_kill"] else 0
|
|
r = (max(c["restarts"]) - min(c["restarts"])) + (len(c["started"]) - 1)
|
|
kills += k + (1 if c["oomkilled"] else 0)
|
|
restarts += r
|
|
pct = round(100.0 * c["anon_peak"] / c["limit"], 1) if c["limit"] else None
|
|
cgp = round(100.0 * c["cg_peak"] / c["limit"], 1) if c["limit"] else None
|
|
if pct is not None:
|
|
peaks.append(pct)
|
|
res["containers"][nm] = {"samples": c["samples"], "anon_peak_bytes": c["anon_peak"], "limit_bytes": c["limit"],
|
|
"anon_peak_pct": pct, "cgroup_peak_pct": cgp, "oom_kill_delta": k,
|
|
"oom_killed_flag": c["oomkilled"], "restarts_delta": r,
|
|
"running_all_samples": all(c["running"]), "cgroup": c["cgroup"]}
|
|
res["oom_kills"], res["restarts"] = kills, restarts
|
|
res["peak_pct"] = max(peaks) if peaks else None
|
|
return res
|
|
|
|
|
|
def setup_gate_probe(sub):
|
|
"""How the setup gate (decision 46) answers this app right now, as the household's browser and as a script: status and
|
|
redirect target only (the cookie is never printed)."""
|
|
out = {}
|
|
for label, hdr in (("browser", ["-H", "Accept: text/html"]), ("script", [])):
|
|
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", f"Host: {sub}.{DOMAIN}"] + hdr + [f"{BASE}/identity"])
|
|
st = re.match(r"HTTP/\S+ (\d+)", r.stdout or "")
|
|
loc = _loc(r.stdout)
|
|
out[label] = {"status": st.group(1) if st else None, "redirect": re.sub(r"\?.*", "?…", loc) if loc else None}
|
|
return out
|
|
|
|
|
|
def security_walk(app, frm, to, evdir, soak_s=SOAK_DEFAULT_S, values=None):
|
|
"""`box_walk.py --security <app> --from <svc=ref,…> --to <svc=ref,…> --ev <dir> [--soak N] [--value K=V …]` — the BOX venue of a
|
|
security-fix step (`09` §3 decision 255, D39 = B), on scratch guest 9202 pointed at the DRILL catalog, through the
|
|
product's own endpoints only:
|
|
1. the drill's template for <app> must stand at FROM (the caller puts it there; this walk never guesses);
|
|
2. <app> is installed fresh; the four version observables must read FROM;
|
|
3. seed + read-back when a box fixture has the app, else `seed: "none"` — written, never hidden;
|
|
4. a DRILL-only commit moves the compose FROM -> TO; sync + rescan until the box's catalog_images carry TO;
|
|
5. the product's guarded Update, every phase with its time;
|
|
6. after: final phase `done`, no undo, no hold, the app running, the four observables equal TO, the seed read back;
|
|
7. the memory soak (default 600 s): a kill or a restart = failed;
|
|
8. <evdir>/box-verdict-<app>.json: `proven` only if all of that holds; `inconclusive` stays `inconclusive`.
|
|
The app is removed through the product at the end unless KEEP=1."""
|
|
import upgrade_fixtures_box as fixtures
|
|
import upgrade_fixtures_box28 as _f28
|
|
global LOG
|
|
LOG = []
|
|
FX = dict(_f28.FIXTURES28)
|
|
FX.update(fixtures.FIXTURES)
|
|
os.makedirs(evdir, exist_ok=True)
|
|
rec = {"app": app, "walk": "security", "box_walk_version": SECURITY_WALK_VERSION, "verdict": "inconclusive",
|
|
"venue": f"box {GUEST} (drill catalog), the guarded Update", "from": frm, "to": to}
|
|
|
|
def finish(verdict, why):
|
|
rec["verdict"], rec["why"] = verdict, why
|
|
rec["log"] = LOG
|
|
write_verdict(rec, evdir)
|
|
os.replace(os.path.join(evdir, "verdict.json"), os.path.join(evdir, f"box-verdict-{app}.json"))
|
|
say(f"RESULT {verdict} — {why}")
|
|
if not os.environ.get("KEEP") and stack(app).get("deployed"):
|
|
remove(app)
|
|
return 0 if verdict == "proven" else 1
|
|
|
|
login()
|
|
conf = guest("grep -A3 '^git:' /var/lib/docker/volumes/felhom-controller-data/_data/controller.yaml")
|
|
if "app-catalog-drill" not in conf and MIRROR_URL not in conf:
|
|
return finish("inconclusive", f"guest {GUEST} is not on the drill catalog (nor its local mirror) — nothing done")
|
|
sh(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
|
drill_imgs = images_of(open(f"{DRILL}/templates/{app}/docker-compose.yml").read())
|
|
if drill_imgs != frm:
|
|
return finish("inconclusive", f"the drill's {app} is at {drill_imgs}, not at FROM {frm} — put it there first")
|
|
fx = FX.get(app)
|
|
sub = getattr(fx, "sub", app) if fx else app
|
|
if stack(app).get("deployed"):
|
|
say(f" [1] removing the existing {app} first (scratch box)")
|
|
remove(app)
|
|
sync_rescan()
|
|
if not deploy(app, sub, values):
|
|
return finish("inconclusive", "the install at FROM did not complete — nothing was tested")
|
|
wait_app(sub, "/identity" if app == "plex" else "/", tries=60)
|
|
rec["observables_before"] = observables(app)
|
|
if rec["observables_before"]["pinned_images"] != frm:
|
|
return finish("inconclusive", f"installed at {rec['observables_before']['pinned_images']}, not at FROM")
|
|
rec["setup_gate_before"] = setup_gate_probe(sub)
|
|
say(f" [2] setup gate as installed: {rec['setup_gate_before']}")
|
|
tok = None
|
|
if fx is None or fx.__class__.__name__ == "NoRoute":
|
|
rec["seed"] = "none"
|
|
rec["seed_why"] = getattr(fx, "tried", "no box fixture for this app")
|
|
say(f" [3] seed: none — {rec['seed_why']}")
|
|
else:
|
|
tok = fx.seed(w_self(), sub, say)
|
|
if tok is None or not fx.verify(w_self(), sub, tok, say):
|
|
return finish("inconclusive", "C1: the fixture could not seed and read back before the update")
|
|
rec["seed"] = "read_back"
|
|
froms = ",".join(frm[s] for s in sorted(frm) if frm[s] != to.get(s))
|
|
tos = ",".join(to[s] for s in sorted(frm) if frm[s] != to.get(s))
|
|
rec["drill_commit"] = drill_bump(app, froms, tos)
|
|
if not rec["drill_commit"]:
|
|
return finish("inconclusive", "the drill could not be moved FROM -> TO")
|
|
moved = [s for s in sorted(to) if frm.get(s) != to[s]]
|
|
rec["sync_waited_s"] = sync_rescan(app, to[moved[0]], tries=40, delay=10)
|
|
if rec["sync_waited_s"] is None:
|
|
return finish("inconclusive", f"the box never read TO {to[moved[0]]} from the drill catalog — nothing pressed")
|
|
rec["badge_before"] = badges(app)
|
|
rec["measured_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
rec["update"] = press_update(app, poll=1.0, cap_s=1800)
|
|
time.sleep(15)
|
|
st = stack(app)
|
|
rec["observables_after"] = obs = observables(app)
|
|
rec["badge_after"] = badges(app)
|
|
rec["state_after"], rec["hold_after"] = st.get("state"), st.get("hold_reason")
|
|
ii = (st.get("app_config") or {}).get("installed_images") or {}
|
|
rec["to_digests"] = {s: (ii.get(s) or {}).get("digest") if isinstance(ii.get(s), dict) else None for s in to}
|
|
u = rec["update"]
|
|
if not u.get("accepted"):
|
|
return finish("failed", f"the Update was refused: {u.get('http')} {str(u.get('refusal'))[:200]}")
|
|
if u.get("final_phase") != "done" or u.get("update_error") or rec["hold_after"]:
|
|
return finish("failed", f"the update ended {u.get('final_phase')!r} (error {u.get('update_error')!r}, hold {rec['hold_after']!r})")
|
|
live = sorted({l.split("image:", 1)[1].strip() for l in obs.get("live_compose_image_lines") or [] if "image:" in l})
|
|
four = {"pinned": obs.get("pinned_images"), "installed": obs.get("installed_images"), "catalog": obs.get("catalog_images"),
|
|
"live_compose": live}
|
|
rec["four_observables"] = four
|
|
running = {l.split()[1] for l in obs.get("docker_inspect") or [] if "running=true" in l}
|
|
bad = {k: v for k, v in four.items() if k != "live_compose" and v != to}
|
|
if bad or set(live) != set(to.values()) or not set(to.values()) <= running:
|
|
return finish("failed", f"the version observables are not TO: {bad or four['live_compose']}, running {sorted(running)}")
|
|
if rec["state_after"] != "running":
|
|
return finish("failed", f"the app is {rec['state_after']!r} after the update")
|
|
if tok is not None:
|
|
rec["seed_read_after"] = fx.verify(w_self(), sub, tok, say)
|
|
if not rec["seed_read_after"]:
|
|
return finish("failed", "the seed did not read back after the update")
|
|
if any(not d for d in rec["to_digests"].values()):
|
|
return finish("inconclusive", f"the box recorded no digest for a TO service: {rec['to_digests']}")
|
|
rec["setup_gate_after"] = setup_gate_probe(sub)
|
|
say(f" [7] memory soak {soak_s} s …")
|
|
rec["memory"] = m = soak(app, soak_s)
|
|
say(f" [7] soak: peak {m.get('peak_pct')}% of the limit (anon), kills {m.get('oom_kills')}, restarts {m.get('restarts')}")
|
|
if m.get("samples", 0) == 0 or m.get("peak_pct") is None:
|
|
return finish("inconclusive", "the soak read no sample — not a pass")
|
|
if m["oom_kills"] or m["restarts"] or not all(c["running_all_samples"] for c in m["containers"].values()):
|
|
return finish("failed", f"the soak saw {m['oom_kills']} OOM kill(s) and {m['restarts']} restart(s)")
|
|
return finish("proven", "the security-fix step ran on the box through the guarded Update; healthy, TO on all four "
|
|
"observables, no undo, no hold, a clean soak" + ("" if tok is None else ", seed read back"))
|
|
|
|
|
|
def images_of(compose_text):
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import ladder
|
|
return ladder.images_in(compose_text)
|
|
|
|
|
|
def w_self():
|
|
return sys.modules[__name__]
|
|
|
|
|
|
def _refs(spec):
|
|
"""`svc=ref,svc2=ref2` -> {svc: ref}."""
|
|
out = {}
|
|
for part in spec.split(","):
|
|
k, _, v = part.strip().partition("=")
|
|
if k and v:
|
|
out[k] = v
|
|
return out
|
|
|
|
|
|
def main(argv):
|
|
ap = argparse.ArgumentParser(prog="box_walk.py")
|
|
ap.add_argument("--security", metavar="APP", required=True,
|
|
help="the security-fix walk (09 decision 255) for this app")
|
|
ap.add_argument("--from", dest="frm", required=True, help="svc=ref,… the drill must stand at")
|
|
ap.add_argument("--to", required=True, help="svc=ref,… the step moves to")
|
|
ap.add_argument("--ev", required=True, help="evidence directory")
|
|
ap.add_argument("--soak", type=int, default=SOAK_DEFAULT_S)
|
|
ap.add_argument("--value", action="append", default=[], metavar="KEY=VAL",
|
|
help="a deploy field the wizard would get from the household (e.g. HDD_PATH=<the drive>)")
|
|
a = ap.parse_args(argv)
|
|
frm, to = _refs(a.frm), _refs(a.to)
|
|
if set(frm) != set(to):
|
|
print("box_walk: --from and --to must name the same services")
|
|
return 2
|
|
vals = dict(v.split("=", 1) for v in a.value if "=" in v)
|
|
return security_walk(a.security, frm, to, a.ev, a.soak, vals or None)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|