#!/usr/bin/env python3 """check-volume-persistence.py — catalog gate: the folder a template preserves must be the folder the app actually writes to. The third catalog gate, and the only RUNTIME one. Its two siblings are static: `check-image-pins.py` proves a template names a concrete tag, `check-image-resolvable.py` proves that tag still exists. Neither can see where an app puts its data, and **neither can any static check** — that was measured, not assumed: over all 53 templates a static audit of the compose files (declared volumes attached, no anonymous mounts, no stray host binds) reports the catalog completely clean, and it reports **papra** clean too. papra's compose is well-formed. It mounts `papra_data` at `/app/data`. The application writes its database to `/app/app-data/db/db.sqlite`, in the container's writable layer, and cannot write to `/app/data` at all. The consequence (R-156, Campaign 10): the app runs, the healthcheck is green, the tier-1/tier-2 backup completes and verifies — and it contains an empty directory. `DumpAppVolumes` (felhom-controller `internal/backup/backup.go:543`) tars the volume, and the volume holds nothing. A backup that fails loudly gets fixed; a backup that succeeds while holding nothing is discovered when someone needs it. python3 scripts/check-volume-persistence.py # every AVAILABLE app python3 scripts/check-volume-persistence.py --all # include hidden/abandoned apps python3 scripts/check-volume-persistence.py papra … # only these app dirs Exit codes: 0 every app in scope CLEAN · 1 at least one BROKEN (the gate REFUSES) · 2 nothing could be decided / the prober failed its own self-test. Requires Docker, network, and several minutes per app, so it is a PERIODIC gate like `check-image-resolvable.py` — run it when a template's `volumes:` block or image tag changes, and at the start of every catalog campaign. `classify()` is pure and unit-tested with no Docker (`scripts/test_check_volume_persistence.py`). WHY THE PROBER SELF-TESTS ON EVERY RUN. A detector that flags nothing is worse than no detector: it converts an unexamined catalog into a documented-clean one. So before this gate is allowed to report anything, it runs two synthetic canary templates through the *same* prober — one built to the exact R-156 signature, one built to write correctly into its volume — and refuses to issue a verdict unless it calls the first BROKEN and the second CLEAN. A green run therefore always carries a live proof that the instrument discriminates. NO `docker exec` ANYWHERE. Campaign 7 §1.1 recorded that `docker exec` writes its OCI error to STDOUT, so a missing binary read as present and the whole healthcheck audit reported every app honest. Every observation here comes from `docker inspect`, `docker diff`, `/proc` and the host filesystem, so a distroless or shell-less image is observed exactly like any other. """ import hashlib import json import os import re import secrets import shutil import stat import subprocess import sys import tempfile import time from pathlib import Path CLEAN, BROKEN, UNDETERMINED = "CLEAN", "BROKEN", "UNDETERMINED" LIFECYCLE_RE = re.compile(r"""^lifecycle:\s*["']?([a-z]+)""", re.MULTILINE) VAR_RE = re.compile(r"\$\{([A-Z0-9_]+)\}") DIFF_RE = re.compile(r"^([ACD])\s+(.*)$") PORT_RE = re.compile(r"loadbalancer\.server\.port=(\d+)") # Where the controller resolves the felhom path variables to at deploy time # (felhom-controller `internal/stacks/deploy.go:567-582`). Any host path here is scratch. SCRATCH_HDD = "/srv/felhom-gate/hdd" SCRATCH_IMPORT = "/srv/felhom-gate/import" # ---------------------------------------------------------------------------- data vs noise # # `docker diff` is noisy. These three rules decide what counts as the customer's data, and they # are the part of this gate most likely to need judgement — they are deliberately explicit rather # than buried in a heuristic. # 1. NOISE — content that is never customer data. Losing it costs a restart, nothing more. NOISE_PREFIX = ( "/tmp/", "/var/tmp/", "/run/", "/var/run/", "/proc/", "/sys/", "/dev/", "/var/log/", "/var/cache/", "/var/lib/apt/", "/var/lib/dpkg/", "/var/spool/", "/usr/share/", "/usr/lib/", "/usr/local/lib/", "/usr/local/share/", "/lib/", "/bin/", "/sbin/", "/var/lib/nginx/", "/var/lib/php/", "/etc/ssl/", "/etc/nginx/", "/etc/apache2/", "/var/lib/misc/", "/var/backups/", "/root/.cache/", "/root/.npm/", "/root/.local/share/", "/home/node/.npm/", "/var/lib/systemd/", ) NOISE_SEGMENT = ("/__pycache__/", "/.cache/", "/node_modules/", "/.git/", "/.next/cache/", "/tmp/", "/temp/", "/.pytest_cache/") NOISE_SUFFIX = (".pid", ".sock", ".log", ".pyc", ".pyo", ".swp") NOISE_EXACT = ("/etc/hosts", "/etc/hostname", "/etc/resolv.conf", "/etc/passwd", "/etc/group", "/etc/shadow", "/etc/localtime", "/etc/timezone", "/etc/mtab", "/etc/machine-id") # 2. DB SIGNATURE — a filename that PROVES a database lives in that directory. This is the # strongest signal available and needs no path heuristics at all. # # `postgresql.conf` was in this list and has been REMOVED: it is a CONFIG file, not a data # file, and the postgres entrypoint writes one to /etc/postgresql at init. That called immich # BROKEN while its database sat correctly in its volume with 1831 files. `PG_VERSION` and # `pg_control` are the real markers of a PGDATA directory, so removing it opens no blind spot — # a genuinely misplaced PGDATA still trips both of those. DB_FILE_RE = re.compile( r"(\.sqlite3?$|\.sqlite\d*$|\.db$|\.db3$|\.db-wal$|\.db-shm$|-wal$|-shm$" r"|^PG_VERSION$|^pg_control$|^ib_logfile|^ibdata|\.frm$|\.ibd$|\.MYD$|\.MYI$" r"|^dump\.rdb$|\.aof$|\.rdb$|^data\.mdb$|^lock\.mdb$|^CURRENT$|^MANIFEST-|\.ldb$|\.sst$" r"|^data\.ms$|^index\.bleve|\.duckdb$|\.bolt$|\.badger$|\.leveldb$)", re.I) # 3. DATA TOKEN — a path that says "app state". THIS RULE DOES NOT CONVICT ON ITS OWN; it is # reported for judgement. That demotion is evidence-driven: across the sweep its true positives # were all also caught by rule 2, while it produced false positives on three separate apps — # calibre-web (`cps/static/css/images/**`), crafty-controller (`…/crafty/config/__pycache__`) # and onlyoffice, which unpacks its OWN static assets into the writable layer at first boot # (plugin icons, slide-theme `media/`, `web-apps/apps/api/documents/api.js` — 2560 added # entries) while its real data mount received data normally. Vocabulary is not evidence: a # directory called `media` holds customer photos in one app and shipped clip-art in the next. # A gate that cries wolf gets ignored, and then it protects nothing — the resolvability gate's # own recorded lesson. DATA_TOKEN_RE = re.compile( r"(^|/)(data|database|db|storage|store|upload|uploads|media|library|libraries|config|" r"appdata|app-data|documents?|photos?|images?|files?|backups?|vault|repositor(y|ies)|" r"attachments?|state|var/lib/(postgresql|mysql|mariadb|redis|mongodb|influxdb)|" r"conf|settings|sessions?|index|search|metadata|thumbnails?)(/|$)", re.I) def is_noise(path: str) -> bool: """A trailing slash is appended before the segment test ON PURPOSE. `docker diff` reports directories as well as files, so a bytecode cache appears as the bare entry `A …/routes/api/crafty/config/__pycache__` while its `.pyc` children are filtered by suffix. Matching `/__pycache__/` against the un-terminated path misses the directory itself, leaving it as the only surviving entry under `…/config` — which scored as data and called crafty-controller BROKEN four times over, on an app whose data had landed correctly. """ p = ("/" + path.lstrip("/")).lower() return (p in NOISE_EXACT or any(p.startswith(x) for x in NOISE_PREFIX) or any(x in p + "/" for x in NOISE_SEGMENT) or any(p.endswith(x) for x in NOISE_SUFFIX)) def is_noise_dir(path: str) -> bool: """`is_noise` for a MOUNT TARGET rather than a file path. The prefixes are written with a trailing slash (`/run/`), so a bare `/run` does not match one. That gap made every mount rule below blind to runtime directories: privatebin's image declares `VOLUME /run`, docker made it an anonymous volume, and 14 entries — `nginx.pid`, `php-fpm.sock`, s6 supervision fifos — read as unbacked-up customer data. Losing /run costs a restart and nothing else. """ return is_noise((path or "").rstrip("/") + "/_") def rollup_diff(entries): """Group writable-layer entries by directory and split DATA from everything else. `docker diff` reports changes to the container's writable layer and EXCLUDES every mounted path, so a file appearing here is definitionally in no volume and no bind: it does not survive a redeploy and no backup can ever contain it. `A` VERSUS `C` IS THE WHOLE DIFFICULTY, and getting it wrong in either direction is fatal: A (added) — the app CREATED this file. Unambiguous: it exists only in the writable layer. papra's `/app/app-data/db/db.sqlite` is an `A`. C (changed) — a file that SHIPS IN THE IMAGE was touched. Usually a chown/chmod sweep and completely benign: linuxserver.io entrypoints re-own the whole application tree, which made calibre-web report 1305 `C` entries including `cps/static/css/images/**` — 92 PNGs of static UI furniture. Treating those as customer data called a clean app BROKEN on the first pass of this sweep. So DATA is decided from `A` entries only, and `C` on a database-signature file is held back as SUSPECT — genuinely ambiguous, because an app writing into a DB that ships in its image produces exactly the same verb. `adjudicate_suspects()` settles those by comparing bytes. """ dirs = {} for kind, path in entries: if kind == "D": continue p = "/" + path.lstrip("/") if is_noise(p): continue d = os.path.dirname(p) or "/" base = os.path.basename(p) e = dirs.setdefault(d, {"added": [], "changed": [], "db_added": False, "db_changed": []}) if kind == "A": if len(e["added"]) < 40: e["added"].append(base) if base and DB_FILE_RE.search(base): e["db_added"] = True else: if len(e["changed"]) < 40: e["changed"].append(base) if base and DB_FILE_RE.search(base): e["db_changed"].append(p) data, token, suspect, other = [], [], [], [] for d, e in sorted(dirs.items()): rec = {"dir": d, "files": e["added"] or e["changed"], "added": e["added"], "changed_count": len(e["changed"]), "db_signature": e["db_added"]} # NOTE: there is deliberately no second "are all this directory's children noise?" filter # here. One was written and removed: `e["added"]` can only ever contain entries that # already passed `is_noise` above, so the check is always False — dead code wearing the # costume of a safeguard. The single entry-level filter is the whole mechanism, and # `test_a_bytecode_cache_DIRECTORY_is_noise` is what pins it. if e["added"] and e["db_added"]: data.append(rec) # rule 2 — CONVICTS elif e["added"] and DATA_TOKEN_RE.search(d): token.append(rec) # rule 3 — reported for judgement, never convicts elif e["db_changed"]: suspect.append({"dir": d, "paths": e["db_changed"], "db_signature": True, "files": [os.path.basename(x) for x in e["db_changed"]]}) else: other.append(rec) return data, token, suspect, other def adjudicate_suspects(cid, image, suspects): """Settle a `C` on a database-signature file by BYTES, not by guessing. A chown leaves the content identical; an app writing into a shipped database does not. So the file is copied out of the running container and out of a pristine container made from the same image, and the two are compared. Identical → benign, the app's data is not here. Different → the app IS writing into an image-layer file, which is the same defect as papra's with a different verb. Returns (confirmed, benign, unresolved) — `unresolved` is never folded into either. """ confirmed, benign, unresolved = [], [], [] if not suspects: return confirmed, benign, unresolved ref = _sh(["docker", "create", image], timeout=300) refid = ref.stdout.strip().splitlines()[-1] if ref.returncode == 0 and ref.stdout.strip() else "" tmp = tempfile.mkdtemp(prefix="volgate-adj-") try: for s in suspects: for path in s["paths"]: live = os.path.join(tmp, "live") orig = os.path.join(tmp, "orig") a = _sh(["docker", "cp", f"{cid}:{path}", live], timeout=300) b = _sh(["docker", "cp", f"{refid}:{path}", orig], timeout=300) if refid else None if a.returncode != 0 or not refid or b.returncode != 0: unresolved.append({**s, "why": "could not read both copies"}) continue try: lb, ob = open(live, "rb").read(), open(orig, "rb").read() except OSError: unresolved.append({**s, "why": "unreadable"}) continue finally: for f in (live, orig): if os.path.exists(f): os.remove(f) if lb == ob: benign.append({**s, "why": "byte-identical to the image copy — a chown/chmod " "sweep, not a write"}) else: confirmed.append({**s, "why": f"DIFFERS from the image copy " f"({len(ob)} B -> {len(lb)} B) — the app is " f"writing into an image-layer file"}) finally: if refid: _sh(["docker", "rm", "-f", refid], timeout=120) shutil.rmtree(tmp, ignore_errors=True) return confirmed, benign, unresolved # ---------------------------------------------------------------------------- the verdict (pure) def classify(probe: dict): """PURE. Turn one probe into (status, [reasons]). Unit-tested without Docker. BROKEN — positive evidence that data does not land where the template preserves it: (a) DATA in a container's writable layer — never persisted, never backed up (b) an app-data mount its own uid cannot write — R-156's second leg (c) data in an ANONYMOUS volume — survives a restart, but is absent from `ResolveDockerVolumeNames` (felhom-controller `internal/appbackup/appdata.go`, which only ever returns `_` for volumes DECLARED in the compose file), so it is never backed up, and a `down` + `up` orphans it. UNDETERMINED — the question was not answered: a container never reached running/healthy, or nothing was written anywhere so there is no data to locate. **Never folded into CLEAN.** CLEAN — something was written, all of it inside a declared named volume or a bind, nothing data-classified in any writable layer, every mount writable by its app uid. """ broken, undet, notes, structural = [], [], [], [] if probe.get("error"): return UNDETERMINED, [probe["error"]] containers = probe.get("containers") or [] if not containers: return UNDETERMINED, ["no containers were created"] wrote_anything = False # Structural check, independent of every path heuristic below. The rules that name data by # its path can only ever recognise the shapes someone thought of: gramps-web writes its actual # family tree to /root/.gramps/grampsdb// as database.txt + name.txt, which carries no # database-signature filename and no data token, so it was MISSED entirely on the first pass # while a second defect in the same app was caught. This asks a question that needs no # vocabulary — did ANY of what the app created land in ANY folder the template preserves? # # ASKED PER APP, NOT PER CONTAINER, and that is the whole difficulty. Per container it fired on # docmost, immich and claper — three CORRECT apps, all the same shape: the app container's only # volume is for user uploads and is legitimately empty on a fresh install, while every byte of # real state sits in the sibling database container's volume (1540, 1833 and 1470 files # respectively). Per app it stays silent on all three and still catches gramps-web, whose # single container had every mount empty. The per-container observation is kept as a NOTE so # nothing is silently dropped. running = [c for c in containers if c.get("status") == "running"] app_persisted = [m for c in running for m in (c.get("mounts") or []) if m["class"] != "tmpfs" and not is_noise_dir(m.get("target"))] app_outside = [(c["name"], d) for c in running for d in (c.get("diff_other_dirs") or []) if d.get("added")] for c in running: mine = [m for m in (c.get("mounts") or []) if m["class"] != "tmpfs" and not is_noise_dir(m.get("target"))] outside = [d for d in (c.get("diff_other_dirs") or []) if d.get("added")] if mine and outside and all(m.get("files", 0) == 0 for m in mine): notes.append( f"{c['name']}: this container's mounts are all empty while it created entries in " f"{[d['dir'] for d in outside][:3]} — benign when a sibling container holds the " f"state, worth a look when none does") if app_persisted and app_outside and all(m.get("files", 0) == 0 for m in app_persisted): structural.append( f"NOTHING this app wrote landed in ANY folder the template preserves: all " f"{len(app_persisted)} mount(s) across {len(running)} container(s) are empty, yet " f"entries were created in {[d['dir'] for _, d in app_outside][:4]}. Needs adjudication.") for c in containers: nm = c["name"] if c.get("status") != "running": undet.append(f"{nm}: not running (status={c.get('status')} exit={c.get('exit')} " f"restarts={c.get('restarts')})") continue if c.get("health") == "unhealthy": undet.append(f"{nm}: unhealthy") for m in c.get("mounts") or []: if m["class"] == "tmpfs" or is_noise_dir(m.get("target")): continue # /run, /tmp, /var/log … — runtime state, not customer data if m.get("files", 0) > 0: wrote_anything = True if m["class"] == "anonymous": broken.append(f"{nm}: {m['files']} file(s) in an ANONYMOUS volume at " f"{m['target']} — not in the compose `volumes:` block, so it is " f"never backed up and a redeploy orphans it") if m.get("writable_by_app") == "NO" and m["class"] in ("named-declared", "bind", "anonymous"): broken.append(f"{nm}: mount {m['target']} is NOT writable by the app's own " f"uid={c.get('uid')} gid={c.get('gid')}") if m["class"] == "named-declared" and m.get("files", 0) == 0: notes.append(f"{nm}: declared volume {m['target']} is EMPTY") for d in c.get("diff_data_dirs") or []: wrote_anything = True broken.append(f"{nm}: DATA in the writable layer at {d['dir']} " f"(db_signature={d.get('db_signature')}, e.g. {d['files'][:4]})" + (f" [{d['why']}]" if d.get("why") else "")) for d in c.get("diff_token_dirs") or []: # Rule 3 never convicts. It is surfaced so a human decides, and counted as evidence # that the app wrote SOMETHING (so an app is not called idle when it plainly was not). wrote_anything = True notes.append(f"{nm}: writable-layer writes at {d['dir']} (path suggests state, no " f"database signature — judgement needed): {d['added'][:4]}") for d in c.get("diff_benign_db_touches") or []: notes.append(f"{nm}: {d['dir']} — database file(s) touched but byte-identical to the " f"image; a chown sweep, not a write") for d in c.get("diff_unresolved") or []: # Never fold an unresolved suspect into CLEAN. undet.append(f"{nm}: could not decide whether {d['dir']} holds live data " f"({d.get('why')})") # `structural` is always shown, whatever the verdict, and on its own it is enough to withhold # a clean bill of health. if broken: return BROKEN, broken + structural + notes if undet or structural: return UNDETERMINED, undet + structural + notes if not wrote_anything: return UNDETERMINED, ["nothing was written to any mount and nothing data-classified in any " "writable layer — the app produced no data to locate. Health is not " "data: this is UNDETERMINED, not CLEAN"] + notes return CLEAN, notes # ---------------------------------------------------------------------------- the prober (Docker) def _sh(args, timeout=180): try: return subprocess.run(args, capture_output=True, text=True, timeout=timeout) except (subprocess.TimeoutExpired, OSError) as e: return subprocess.CompletedProcess(args, 124, "", f"{e}") def parse_deploy_fields(text: str): """Minimal `deploy_fields:` reader — env_var / type / default / generate. Line-based on purpose: this repo ships no requirements file and the two sibling gates parse `.felhom.yml` the same way, so the gate keeps working with a bare python3. """ fields, cur, indent = [], None, None in_block = False for line in text.splitlines(): if re.match(r"^deploy_fields:\s*$", line): in_block, cur = True, None continue if in_block: if line.strip() and not line.startswith((" ", "\t")): break # a new top-level key ends the block m = re.match(r"^(\s*)-\s+env_var:\s*[\"']?([A-Za-z0-9_]+)", line) if m: if cur: fields.append(cur) indent, cur = len(m.group(1)), {"env_var": m.group(2)} continue if cur is not None: m = re.match(r"^\s+(type|default|generate):\s*(.*?)\s*$", line) if m: cur[m.group(1)] = m.group(2).strip().strip('"').strip("'") if cur: fields.append(cur) return fields def _gen(spec, ftype): """Mirror `generateValue` in felhom-controller `internal/stacks/deploy.go:870`. It has to be the SAME value shape the controller mints, or the gate measures an app the customer never runs. `base64key` is the one that bites: the controller returns `"base64:" + b64` (deploy.go:904) because Laravel's APP_KEY is invalid without the prefix — dropping it leaves bookstack serving 500s, which reads as an app defect and is a harness bug. """ if spec: kind, _, n = spec.partition(":") n = int(n) if n.isdigit() else 32 if kind == "hex": return secrets.token_hex(n) if kind in ("password", "secret"): alph = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" return "".join(secrets.choice(alph) for _ in range(n)) if kind == "base64key": import base64 return "base64:" + base64.b64encode(secrets.token_bytes(n)).decode() if kind == "static": return spec.partition(":")[2] if ftype in ("password", "secret", "secret_input"): return "Gate" + secrets.token_hex(12) return "" def build_env(app: str, felhom_text: str, compose_text: str): """Resolve every `${VAR}` the compose uses. Generated values are never printed or written to any artifact — only the KEYS are, per the repo's no-secrets rule.""" sub = re.search(r"^subdomain:\s*[\"']?([a-z0-9-]+)", felhom_text, re.M) env = {"DOMAIN": "gate.invalid", "SUBDOMAIN": sub.group(1) if sub else app, "HDD_PATH": SCRATCH_HDD, "USERDATA_PATH": SCRATCH_HDD + "/userdata", "IMPORT_PATH": SCRATCH_IMPORT, "TZ": "Europe/Budapest"} for f in parse_deploy_fields(felhom_text): var, t = f["env_var"], f.get("type", "text") if t == "path": env[var] = SCRATCH_HDD elif f.get("default"): env[var] = f["default"] elif t == "domain": env[var] = "gate.invalid" elif t == "subdomain": env[var] = env["SUBDOMAIN"] else: env[var] = _gen(f.get("generate"), t) for var in set(VAR_RE.findall(compose_text)): # A var that resolves to "" binds a bogus root-owned dir at the container root # (felhom-controller deploy.go:571) — never leave one unset. env.setdefault(var, "g" + hashlib.sha256(var.encode()).hexdigest()[:20]) return env def _inspect(cid): r = _sh(["docker", "inspect", cid]) try: return json.loads(r.stdout)[0] except Exception: return None def _uid(info): """The uid PID 1 ACTUALLY runs as, from /proc — `Config.User` is frequently empty even when the image drops privileges inside its entrypoint, which is exactly papra's shape.""" pid = (info.get("State") or {}).get("Pid") or 0 if pid: try: uid = gid = None for line in open(f"/proc/{pid}/status"): if line.startswith("Uid:"): uid = int(line.split()[1]) elif line.startswith("Gid:"): gid = int(line.split()[1]) if uid is not None: return uid, gid except OSError: pass u = (info.get("Config") or {}).get("User") or "" a, _, b = u.partition(":") return (int(a) if a.isdigit() else None, int(b) if b.isdigit() else None) def _classify_mount(m, project, declared): t = m.get("Type") if t in ("tmpfs", "bind"): return t if t == "volume": name = m.get("Name") or "" if re.fullmatch(r"[0-9a-f]{64}", name): return "anonymous" if name.startswith(project + "_") and name[len(project) + 1:] in declared: return "named-declared" return "named-external" return t or "unknown" def _walk(src, limit=4000): if not src or not os.path.isdir(src): return -1, [] n, sample = 0, [] for dp, _, fns in os.walk(src, onerror=lambda e: None): for f in fns: n += 1 if len(sample) < 40: sample.append(os.path.relpath(os.path.join(dp, f), src)) if n > limit: return n, sample return n, sample def _writable(src, uid, gid): """Can uid/gid create a file in src? Decided from the host-side owner/mode — the same question R-156 answered with `touch` inside the container, without needing a shell there.""" if uid is None: return "unknown-uid" if not src or not os.path.isdir(src): return "no-source" if uid == 0: return "yes" try: st = os.stat(src) except OSError: return "stat-error" if st.st_uid == uid and st.st_mode & stat.S_IWUSR: return "yes" if gid is not None and st.st_gid == gid and st.st_mode & stat.S_IWGRP: return "yes" return "yes" if st.st_mode & stat.S_IWOTH else "NO" PATHS_FIRST = ("/",) PATHS_DEEP = ("/", "/login", "/setup", "/signup", "/register", "/install", "/admin", "/api/health", "/health", "/healthz", "/status", "/web", "/index.php", "/dashboard") def _exercise(cids, ports, deep=False): """Minimum exercise: an HTTP request the app's OWN router answers. A container that has only started may have written nothing, and health-check-passing is not data-writing — conflating the two is precisely what let papra look fine. Any status code (including 3xx/4xx/5xx) proves the request reached application code; `000` does not. """ hits = [] for cid in cids: info = _inspect(cid) or {} for net in ((info.get("NetworkSettings") or {}).get("Networks") or {}).values(): ip = net.get("IPAddress") if not ip: continue for port in ports: for path in (PATHS_DEEP if deep else PATHS_FIRST): a = ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "20"] if deep: a += ["-L", "--max-redirs", "5"] code = _sh(a + [f"http://{ip}:{port}{path}"], timeout=40).stdout.strip() if code and code != "000": hits.append(f"{ip}:{port}{path} -> {code}") if not deep: break return hits def docker_prober(app: str, app_dir: Path, settle: int = 45, wait: int = 300) -> dict: """Deploy the template, exercise it, and report WHERE the data landed. The Docker seam. Everything Docker-touching lives here so `classify()` stays pure and testable. """ compose_src = app_dir / "docker-compose.yml" felhom_src = app_dir / ".felhom.yml" if not compose_src.is_file(): return {"app": app, "error": "no docker-compose.yml"} compose_text = compose_src.read_text(encoding="utf-8") felhom_text = felhom_src.read_text(encoding="utf-8") if felhom_src.is_file() else "" work = Path(tempfile.mkdtemp(prefix=f"volgate-{app}-")) project = "volgate-" + re.sub(r"[^a-z0-9]+", "", app.lower()) cf = work / "docker-compose.yml" shutil.copy(compose_src, cf) env = build_env(app, felhom_text, compose_text) (work / ".env").write_text("".join(f"{k}={v}\n" for k, v in sorted(env.items())), encoding="utf-8") os.makedirs(SCRATCH_HDD + "/userdata", exist_ok=True) os.makedirs(SCRATCH_IMPORT, exist_ok=True) _sh(["docker", "network", "create", "traefik-public"], timeout=60) # templates expect it base = ["docker", "compose", "-p", project, "--project-directory", str(work), "-f", str(cf)] try: cfg = _sh(base + ["config", "--format", "json"], timeout=180) try: resolved = json.loads(cfg.stdout) except Exception: return {"app": app, "error": f"compose config failed: " f"{(cfg.stderr or cfg.stdout)[:300]}"} declared = set((resolved.get("volumes") or {}).keys()) ports = sorted({int(m.group(1)) for svc in (resolved.get("services") or {}).values() for lbl in ((svc.get("labels") or {}).values() if isinstance(svc.get("labels"), dict) else (svc.get("labels") or [])) for m in [PORT_RE.search(str(lbl))] if m}) up = _sh(base + ["up", "-d"], timeout=1800) cids = [c for c in _sh(base + ["ps", "-aq"], timeout=120).stdout.split() if c] if not cids: return {"app": app, "error": f"no containers created (compose up rc={up.returncode}: " f"{(up.stderr or '')[-300:]})"} deadline = time.time() + wait while time.time() < deadline: pend = False for cid in cids: st = (_inspect(cid) or {}).get("State") or {} if st.get("Status") in ("created", "restarting") or \ (st.get("Health") or {}).get("Status") == "starting": pend = True if not pend: break time.sleep(10) running = [c for c in cids if ((_inspect(c) or {}).get("State") or {}).get("Status") == "running"] hits = _exercise(running, ports) if (running and ports) else [] time.sleep(settle) def observe(): out = [] for cid in cids: info = _inspect(cid) if not info: continue uid, gid = _uid(info) mounts = [] for m in (info.get("Mounts") or []): n, sample = _walk(m.get("Source")) mounts.append({"target": m.get("Destination"), "class": _classify_mount(m, project, declared), "name": m.get("Name"), "source": m.get("Source"), "files": n, "sample": sample, "writable_by_app": _writable(m.get("Source"), uid, gid)}) entries = [(g.group(1), g.group(2)) for line in _sh(["docker", "diff", cid], timeout=180).stdout.splitlines() for g in [DIFF_RE.match(line)] if g] data, token, suspect, other = rollup_diff(entries) image = (info.get("Config") or {}).get("Image") conf, benign, unres = adjudicate_suspects(cid, image, suspect) st = info.get("State") or {} out.append({"name": (info.get("Name") or cid).lstrip("/"), "image": image, "status": st.get("Status"), "health": (st.get("Health") or {}).get("Status"), "exit": st.get("ExitCode"), "restarts": st.get("RestartCount"), "uid": uid, "gid": gid, "mounts": mounts, "diff_total": len(entries), "diff_added": sum(1 for k, _ in entries if k == "A"), "diff_data_dirs": data + conf, "diff_token_dirs": token, "diff_benign_db_touches": benign, "diff_unresolved": unres, "diff_other_dirs": other}) return out def nothing_written(cs): return not any(c["diff_data_dirs"] or c["diff_token_dirs"] or any(m["class"] != "tmpfs" and m["files"] > 0 for m in c["mounts"]) for c in cs) containers = observe() # Second chance before declaring the question unanswerable: walk a wider path list # following redirects, so a first-run wizard is actually reached. if nothing_written(containers) and running and ports: hits += _exercise(running, ports, deep=True) time.sleep(90) containers = observe() return {"app": app, "ports": ports, "exercise": hits, "declared_volumes": sorted(declared), "containers": containers, "env_keys": sorted(env)} finally: # `compose down -v` removes THIS project's volumes and nothing else. Deliberately NOT # `docker volume prune -f`: that is a GLOBAL sweep of every unused volume on the host, # which on any box also running real stacks would delete data this gate never created. # The workspace CLAUDE.md bans exactly this class of global Docker cleanup. _sh(base + ["down", "-v", "--remove-orphans"], timeout=900) shutil.rmtree(work, ignore_errors=True) # ---------------------------------------------------------------------------- canary self-test # A purpose-built canary image reproducing papra's exact shape: `/app/data` exists and is # root-owned, `/app/app-data` exists and belongs to the app's own non-root uid. A fresh named # volume inherits the ownership of whatever directory it is mounted over, so mounting at # /app/data yields a volume the app cannot write, and mounting at /app/app-data yields one it can. # That single difference is the whole of R-156, which makes the canary pair a live demonstration # of the defect AND of its fix on every run. CANARY_IMAGE = "felhom-volgate-canary:1" CANARY_DOCKERFILE = """FROM alpine:3.22 RUN mkdir -p /app/data /app/app-data \ && adduser -D -u 4242 appuser \ && chown 4242:4242 /app/app-data """ _CANARY_CMD = ('["sh", "-c", "mkdir -p /app/app-data/db && ' 'echo canary > /app/app-data/db/db.sqlite && sleep 900"]') # BROKEN: the volume is mounted where the app does NOT write, and cannot write. CANARY_BROKEN = f"""services: canary: image: {CANARY_IMAGE} user: "4242:4242" command: {_CANARY_CMD} volumes: - canary_data:/app/data volumes: canary_data: """ # CLEAN: the same app, same uid, volume mounted where it actually writes. CANARY_CLEAN = f"""services: canary: image: {CANARY_IMAGE} user: "4242:4242" command: {_CANARY_CMD} volumes: - canary_data:/app/app-data volumes: canary_data: """ def ensure_canary_image() -> bool: if _sh(["docker", "image", "inspect", CANARY_IMAGE], timeout=120).returncode == 0: return True d = tempfile.mkdtemp(prefix="volgate-canary-build-") try: with open(os.path.join(d, "Dockerfile"), "w") as fh: fh.write(CANARY_DOCKERFILE) return _sh(["docker", "build", "-q", "-t", CANARY_IMAGE, d], timeout=900).returncode == 0 finally: shutil.rmtree(d, ignore_errors=True) def self_test(prober) -> tuple[bool, str]: """Prove the prober discriminates, on THIS run, in BOTH directions. A detector that flags everything and a detector that flags nothing are both useless, and the second is actively dangerous: it turns an unexamined catalog into a documented-clean one. So the gate refuses to report a verdict at all unless it has just called a known-broken template BROKEN and a known-good one CLEAN. """ if prober is docker_prober and not ensure_canary_image(): return False, f"could not build {CANARY_IMAGE}" for name, body, want in (("canary-broken", CANARY_BROKEN, BROKEN), ("canary-clean", CANARY_CLEAN, CLEAN)): d = Path(tempfile.mkdtemp(prefix=f"volgate-{name}-")) try: (d / "docker-compose.yml").write_text(body, encoding="utf-8") got, why = classify(prober(name, d, settle=10, wait=90)) if got != want: return False, f"{name}: expected {want}, got {got} ({'; '.join(why)[:200]})" finally: shutil.rmtree(d, ignore_errors=True) return True, "" # ---------------------------------------------------------------------------- driver def app_lifecycle(app_dir: Path) -> str: f = app_dir / ".felhom.yml" if not f.is_file(): return "available" m = LIFECYCLE_RE.search(f.read_text(encoding="utf-8")) v = m.group(1) if m else "available" return v if v in ("available", "hidden", "abandoned") else "available" def collect_apps(root: Path, only=None, include_unavailable=False): apps, skipped = [], [] for f in sorted(root.glob("templates/*/docker-compose.yml")): app = f.parent.name if only and app not in only: continue lc = app_lifecycle(f.parent) if lc != "available" and not include_unavailable: skipped.append(f"{app} ({lc})") continue apps.append((app, f.parent)) return apps, skipped def check(root: Path, only=None, prober=docker_prober, include_unavailable=False, evidence: Path | None = None, skip_self_test=False) -> int: apps, skipped = collect_apps(root, only, include_unavailable) if skipped: print(f"skipping {len(skipped)} app(s) not offered for new installs: {', '.join(skipped)}") if not apps: if skipped: print("nothing to check — every app in scope is out of circulation") return 0 print(f"ERROR: no templates found under {root}/templates/", file=sys.stderr) return 2 if not skip_self_test: print("self-testing the prober (both directions)…") ok, why = self_test(prober) if not ok: print(f"ERROR: the prober failed its own canary — {why}\n" " refusing to report a verdict: a broken detector reporting CLEAN is worse " "than no detector at all", file=sys.stderr) return 2 print(" prober flags the R-156 signature and clears a correct template — trustworthy") results = [] for app, d in apps: probe = prober(app, d) status, why = classify(probe) results.append((app, status, why)) print(f"{app:<20} {status:<13} {'; '.join(why)[:150]}", flush=True) if evidence: out = evidence / app out.mkdir(parents=True, exist_ok=True) (out / "probe.json").write_text( json.dumps({"verdict": status, "reasons": why, "probe": probe}, indent=2, sort_keys=True), encoding="utf-8") broken = [a for a, s, _ in results if s == BROKEN] undet = [(a, w) for a, s, w in results if s == UNDETERMINED] clean = [a for a, s, _ in results if s == CLEAN] if broken: print("\nBROKEN — the app's data does NOT land where the template preserves it:") for a, s, w in results: if s == BROKEN: print(f" {a}") for line in w: print(f" {line}") if undet: print("\nUNDETERMINED — NOT a pass. The question was not answered for these:") for a, w in undet: print(f" {a}: {'; '.join(w)[:200]}") print(f"\n{len(clean)} clean · {len(broken)} broken · {len(undet)} undetermined " f"(of {len(apps)} in scope)") if broken: print("volume-persistence gate REFUSED") return 1 if undet: print("INCOMPLETE — no broken template among those decided, but " f"{len(undet)} were never decided. This is not a clean bill of health.") return 2 print("volume-persistence gate OK") return 0 if __name__ == "__main__": argv = sys.argv[1:] all_apps = "--all" in argv no_self = "--no-self-test" in argv ev = None for i, a in enumerate(argv): if a == "--evidence" and i + 1 < len(argv): ev = Path(argv[i + 1]) argv = [a for i, a in enumerate(argv) if a not in ("--all", "--no-self-test", "--evidence") and not (i > 0 and argv[i - 1] == "--evidence")] sys.exit(check(Path(__file__).resolve().parent.parent, only=argv or None, include_unavailable=all_apps, evidence=ev, skip_self_test=no_self))