From 6c690a1947c65a832e6485476f2e183d11aa00ec Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 22 Sep 2026 10:51:34 +0200 Subject: [PATCH] gates: refuse a health probe the app does not answer (R-618) check-probe-matches-compose.py, a --fast gate so it bites in the hook and in CI. The oracle was already in every template: the probed service's own compose healthcheck dials the app on 127.0.0.1. The gate compares the .felhom.yml probe against it, statically. Port mismatch REFUSES for every check type. Path mismatch REFUSES only where the probe can fail on it (type api WITH expect) and WARNS otherwise, because probeHTTP calls any response healthy otherwise - measured, not assumed. Six WARNs on the current catalog, each named in the CHANGELOG; paperless-ngx is the loud one: no container matches the stack name, so no probe ever runs. Four red-proofs and five decoys, suite now 51 cases. --root lets the suite judge its own clone. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 37 ++++ scripts/catalog_gates.py | 9 + scripts/check-probe-matches-compose.py | 253 +++++++++++++++++++++++++ scripts/test_gate_decoys.py | 101 ++++++++++ 4 files changed, 400 insertions(+) create mode 100644 scripts/check-probe-matches-compose.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2919a7..68a086f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +## A gate that refuses a probe the app does not answer (2026-09-22, R-618) + +`scripts/check-probe-matches-compose.py`, wired into `catalog_gates.py` as a **`--fast`** gate, so +it runs in the pre-push hook and in CI. + +**The oracle was already in the file.** Every template's probed service carries a compose +`healthcheck.test` that dials the app on loopback — `wget http://127.0.0.1:80/accounts/login/`. It +is written by whoever added the template and exercised by docker on every start. The gate compares +the `.felhom.yml` probe against it, statically, with no network and no container. The probed service +is resolved exactly as `findProbeContainer` resolves it (container name equal to the stack name, +else prefix). + +**Two verdicts, and the narrower one is a measurement, not a compromise.** A wrong PORT is always +fatal — nothing listens, every dial is refused — so it REFUSES, for every check type. A wrong PATH +is fatal only when the probe can fail on it: `probeHTTP` calls ANY response healthy for type `http`, +and for type `api` with no `expect` block. So a path mismatch REFUSES only with `api` + `expect`, +and WARNS otherwise. Over all 53 apps that is the difference between convicting `zipline` and merely +warning about `home-assistant`, whose badge is right today and would break the day someone adds an +`expect`. + +**Six warnings on the current catalog**, none a conviction and none a pass: + +| app | why no verdict | +|---|---| +| `paperless-ngx` | **no container_name equals or begins with the stack name** — `findProbeContainer` returns nothing and the stack is skipped, so no probe ever runs and the badge can never go red | +| `vikunja` | the probed service has no compose healthcheck — no oracle | +| `crafty-controller`, `mealie`, `uptime-kuma` | the healthcheck dials through a python/script helper, not a loopback URL — no oracle | +| `home-assistant` | path mismatch on a probe that cannot fail on it (above) | + +**Red-proofs, four, plus decoys both ways** (`scripts/test_gate_decoys.py`, now 51 cases): each of +the three real faults re-introduced one at a time and refused; the fixed tree passes; a clean app +given a wrong port refused; an `api`+`expect` probe given a wrong path refused. The decoys move the +LABEL and must not convict — the port in a YAML comment, Traefik's `loadbalancer.server.port`, a +published `ports:` mapping, a NON-probed sidecar's own healthcheck, and a path mismatch on a probe +that cannot fail on it. The gate takes `--root=` so the suite judges its clone and not the real +repo; without it every case would read identical bytes and pass for nothing. + ## Three health probes now dial where the app actually listens (2026-09-22, R-618) **Templates only, and no `image:` line moved — so no `catalog_since` moves either.** diff --git a/scripts/catalog_gates.py b/scripts/catalog_gates.py index a8cb2b7..7379558 100644 --- a/scripts/catalog_gates.py +++ b/scripts/catalog_gates.py @@ -16,6 +16,9 @@ Gates, in order (all must pass; **non-zero exit on any failure**): 2. image-resolvable network — every pinned tag still EXISTS upstream 3. volume-persistence RUNTIME — the folder a template preserves is the folder the app writes to 5. catalog-since static, needs GIT HISTORY — an image: move bumps that app's catalog_since (R-452) + 7. probe-matches-compose static, instant, whole repo — the .felhom.yml health probe dials the + port/path the app's own compose healthcheck dials (R-618). Runs in the + hook: a wrong probe stops a WORKING app at the end of a successful update. 4. engine-major static, needs GIT HISTORY — no database engine pin crosses a MAJOR version (operator ruling 2026-09-13; expires when Slice 4 / R-448 ships). Runs in the pre-push hook, which has the full clone; on a SHALLOW clone (CI fetches at @@ -84,6 +87,12 @@ GATES = [ # scope for the LANGUAGE checks only — the Hungarian freeze always runs on all 53, because a # scoped push that quietly edits a neighbour's copy is precisely what a freeze is for. ("copy-i18n", "check-copy-i18n.py", True, True, False), + # R-618 (2026-09-22): the `.felhom.yml` probe must dial the port — and, where the probe can + # actually fail on it, the path — that the SAME service's compose healthcheck dials on + # loopback. Static, instant, no git history, no network. It is `--fast` deliberately: the + # defect it catches does not merely mis-colour a badge, it makes a SUCCESSFUL update stop a + # working app (the `verifying` phase waits on this probe), so it must bite at push time. + ("probe-matches-compose", "check-probe-matches-compose.py", True, True, False), ] VERDICT = {0: "OK", 1: "FAILED", 2: "INCONCLUSIVE"} diff --git a/scripts/check-probe-matches-compose.py b/scripts/check-probe-matches-compose.py new file mode 100644 index 0000000..3686f98 --- /dev/null +++ b/scripts/check-probe-matches-compose.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""check-probe-matches-compose.py — the `.felhom.yml` probe must dial where the app LISTENS (R-618). + +WHAT WENT WRONG, and why a static gate can catch it. + +`.felhom.yml`'s `healthcheck.checks` tells the controller where to knock: it dials +`:` on the compose network. The port is therefore **the port the +process listens on INSIDE the container** — not the published port, not the Traefik port. Three +templates named a port or a path the app does not answer. On its own that is only a wrong badge; +what makes it P1 is that **the guarded update's `verifying` phase waits on that same probe**, so a +SUCCESSFUL update ends with `failAndHold` stopping a working app. Measured on 2026-09-21: tandoor +served HTTP 200 on the new version at four samples over five minutes, docker's own healthcheck +green, and the controller stopped it at +361.9 s. + +THE ORACLE. The compose `healthcheck.test` of the very same service already dials the app on +loopback — `wget http://127.0.0.1:80/accounts/login/`. That line is written by whoever added the +template, is exercised by docker on every start, and is therefore known-good. So the two can be +compared, statically, with no network and no container: + + probe port == the port the compose healthcheck dials on 127.0.0.1 + probe path == the path it dials (when the probe can FAIL on it) + +WHICH SERVICE. The one the controller would actually probe. `findProbeContainer` +(`controller/internal/stacks/healthprobe.go:297`) takes the container whose name EQUALS the stack +name, else the first whose name has it as a PREFIX. This gate resolves the same way, off +`container_name`. + +THE TWO VERDICTS, and why the path rule is narrower than the port rule — this is a measurement, not +a compromise: + + * **A wrong PORT is always fatal.** Nothing listens there, the dial is refused, every check fails. + REFUSE, for every check type including `tcp`. + * **A wrong PATH is fatal only if the probe can fail on it.** `probeHTTP` returns healthy for ANY + response when the type is `http`, and also when the type is `api` with **no `expect` block** + (healthprobe.go:253-262). So a path mismatch bites exactly when the type is `api` AND `expect` + is present. REFUSE there; WARN otherwise. Run over all 53 on 2026-09-22, that is the difference + between convicting `zipline` (`api` + `expect.status: 200`, path `/api/health` vs the app's real + `/api/healthcheck`) and merely warning about `home-assistant` (`api`, no `expect`, `/api/` vs + `/manifest.json`) — which is honest: home-assistant's badge is right today and would break the + day someone adds an `expect`. + +WHAT IT CANNOT SEE — named, not implied: + + 1. **A template whose compose healthcheck is ALSO wrong.** The oracle is the compose line; if both + agree and both are wrong, this gate passes. Only a live deploy decides that. + 2. **A path that is right but answers the wrong STATUS.** `expect.status: 200` against an endpoint + that 302s is a real fault and looks identical to a correct template here. + 3. **Anything about a service other than the probed one.** + + Each of these three is a WARN category below rather than a silent pass, wherever it is visible. + +USAGE + python3 scripts/check-probe-matches-compose.py # every app + python3 scripts/check-probe-matches-compose.py tandoor wger # only these + python3 scripts/check-probe-matches-compose.py --root= # judge another checkout +Exit: 0 clean (warnings do not convict) · 1 convicted · 2 inconclusive (unreadable template). +""" +import os +import re +import sys + +try: + import yaml +except ImportError: + print("check-probe-matches-compose: PyYAML is not installed — INCONCLUSIVE") + sys.exit(2) + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEMPLATES = os.path.join(ROOT, "templates") + + +def set_root(root): + """Point the gate at a different checkout — `--root=`. + + Without this the decoy suite would run the gate against the REAL repo while editing a clone, so + every case would read the same unchanged files and every case would pass. That is the + `constant-for-measurement` decoy shape this repo's own suite warns about, and it has fooled a + gate here before. + """ + global TEMPLATES + TEMPLATES = os.path.join(root, "templates") + +# The names that mean "this container, from inside it". `0.0.0.0` appears as a dial target in a +# couple of templates; it is a bind address rather than a destination, but it reaches the same +# listener and the PORT is what this gate compares. +LOOPBACK = ("127.0.0.1", "localhost", "0.0.0.0", "::1", "[::1]") + + +def flatten_test(test): + """The compose healthcheck test as one string, whatever shape it was written in. + + Both `["CMD", "wget", …]` and `["CMD-SHELL", "curl … || exit 1"]` and a bare string occur in + this catalog. The leading CMD/CMD-SHELL word is dropped; everything else is joined, because the + URL may sit in any argument (node's `-e` script puts it inside quotes). + """ + if test is None: + return None + if isinstance(test, str): + return test + parts = list(test) + if parts and parts[0] in ("CMD", "CMD-SHELL", "NONE"): + parts = parts[1:] + return " ".join(str(p) for p in parts) + + +URL_RE = re.compile(r'https?://([A-Za-z0-9_.\-\[\]:]+?)(?::(\d+))?(/[^\s\'"`)\\,]*)?(?=[\s\'"`)\\,]|$)') +NC_RE = re.compile(r'\bnc\b[^\n;|&]*?(\S+)\s+(\d+)\b') + + +def endpoints(test_str): + """Every loopback endpoint the compose healthcheck dials: (scheme, port, path).""" + found = [] + for m in URL_RE.finditer(test_str): + host, port, path = m.group(1), m.group(2), m.group(3) + if host not in LOOPBACK: + continue + p = int(port) if port else (443 if m.group(0).startswith("https") else 80) + found.append(("http", p, path or "/")) + for m in NC_RE.finditer(test_str): + if m.group(1) in LOOPBACK: + found.append(("tcp", int(m.group(2)), None)) + return found + + +def probed_service(app, services): + """The service the controller would probe — `findProbeContainer`'s rule, statically.""" + for name, svc in services.items(): + if (svc or {}).get("container_name") == app: + return name, svc + for name, svc in services.items(): + cn = str((svc or {}).get("container_name") or "") + if cn.startswith(app): + return name, svc + return None, None + + +def check_app(app): + """Returns (verdicts, warnings) — each a list of one-line strings.""" + d = os.path.join(TEMPLATES, app) + bad, warn = [], [] + try: + fy = yaml.safe_load(open(os.path.join(d, ".felhom.yml"), encoding="utf-8")) or {} + cy = yaml.safe_load(open(os.path.join(d, "docker-compose.yml"), encoding="utf-8")) or {} + except Exception as e: # noqa: BLE001 — report, never crash + return None, ["%s: template unreadable (%s)" % (app, e)] + + checks = ((fy.get("healthcheck") or {}).get("checks")) or [] + if not checks: + warn.append("%s: no `healthcheck.checks` in .felhom.yml — nothing to compare" % app) + return bad, warn + + services = cy.get("services") or {} + svc_name, svc = probed_service(app, services) + if svc is None: + # Not a conviction, but it is worse than a mismatch: `findProbeContainer` returns "" and + # the stack is SKIPPED, so no probe ever runs and the badge can never go red. Loud WARN. + warn.append("%s: NO container_name equals or begins with the stack name — the controller " + "would find no container to probe and SKIP this stack silently " + "(healthprobe.go:62). Containers: %s" + % (app, ", ".join(sorted(str((s or {}).get("container_name")) for s in services.values())))) + return bad, warn + + test = flatten_test((svc.get("healthcheck") or {}).get("test")) + if not test: + warn.append("%s: service %r has no compose healthcheck — this gate has no oracle for it; " + "only a live deploy can judge the probe" % (app, svc_name)) + return bad, warn + + eps = endpoints(test) + if not eps: + has_ports = bool(svc.get("ports") or svc.get("expose")) + warn.append("%s: the compose healthcheck of %r dials no loopback URL or port (%s) — " + "no oracle.%s" % (app, svc_name, test.strip()[:90], + (" Only `expose:`/`ports:` are declared, and a published " + "port is NOT the port the app listens on inside the " + "container, so it cannot stand in.") if has_ports else "")) + return bad, warn + + for c in checks: + ctype = (c or {}).get("type") + cport = (c or {}).get("port") + cpath = (c or {}).get("path") or "/" + # `tcp` is compared on the port alone — it dials a socket and has no path. + same_scheme = [e for e in eps if (e[0] == "tcp") == (ctype == "tcp")] or eps + ports = sorted({e[1] for e in same_scheme}) + if cport not in ports: + bad.append("%s: probe %s port %s, but the compose healthcheck of %r dials %s on " + "loopback. Nothing listens on %s inside the container, so EVERY probe fails " + "and a successful update is stopped by failAndHold (R-618). test: %s" + % (app, ctype, cport, svc_name, "/".join(map(str, ports)), cport, + test.strip()[:90])) + continue + if ctype == "tcp": + continue + paths = sorted({e[2] for e in same_scheme if e[1] == cport and e[2] is not None}) + if paths and cpath not in paths: + can_fail = ctype == "api" and (c or {}).get("expect") + msg = ("%s: probe %s path %r, but the compose healthcheck of %r dials %s on the same " + "port. test: %s" % (app, ctype, cpath, svc_name, " or ".join(repr(p) for p in paths), + test.strip()[:90])) + if can_fail: + bad.append(msg + " This probe CAN fail on it: type `api` with an `expect` block " + "compares the response (healthprobe.go:253-262).") + else: + warn.append(msg + " Harmless TODAY — type %r without `expect` calls ANY response " + "healthy — and fatal the day someone adds `expect`." % ctype) + return bad, warn + + +def main(argv): + for a in argv: + if a.startswith("--root="): + set_root(a[len("--root="):]) + unknown = [a for a in argv if a.startswith("-") and not a.startswith("--root=")] + if unknown: + print("unknown option(s): %s" % " ".join(unknown)) + return 2 + apps = [a for a in argv if not a.startswith("-")] + if not apps: + apps = sorted(d for d in os.listdir(TEMPLATES) + if os.path.isdir(os.path.join(TEMPLATES, d))) + print("probe-matches-compose: %d app(s)" % len(apps)) + + convicted, warnings, unreadable = [], [], 0 + for app in sorted(apps): + bad, warn = check_app(app) + if bad is None: + unreadable += 1 + warnings += warn + continue + convicted += bad + warnings += warn + + if warnings: + print("\n%d WARNING(S) — not a conviction, and not a pass either:" % len(warnings)) + for w in warnings: + print(" WARN " + w) + if unreadable: + print("\nprobe-matches-compose: %d template(s) unreadable — INCONCLUSIVE" % unreadable) + return 2 + if convicted: + print("\nprobe-matches-compose: CONVICTED — %d probe(s) do not match the app" % len(convicted)) + for b in convicted: + print(" FAIL " + b) + return 1 + print("\nprobe-matches-compose: OK — every probe dials the port (and, where it can fail, the " + "path) that the app's own compose healthcheck dials") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_gate_decoys.py b/scripts/test_gate_decoys.py index 448f2ad..7769569 100644 --- a/scripts/test_gate_decoys.py +++ b/scripts/test_gate_decoys.py @@ -48,6 +48,9 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) COVERS = { "engine-major": "the major moved in a comment/env var/README/app image, not on an engine's image: line", "catalog-since": "the date bumped in a comment/README while .felhom.yml's field stayed; or only a comment/env moved, no image (R-452)", + "probe-matches-compose": "the port/path moved in a COMMENT, in traefik's loadbalancer label, in " + "`ports:`/`expose:`, or on a NON-probed service - none of which is where " + "the app listens; vs a real probe port/path that the app does not answer (R-618)", "copy-i18n": "Hungarian edited in a COMMENT/README/display_name (label, not copy) vs a real frozen string changed; an English block that is not English, is not matched to a Hungarian twin, or rewrites a credential (R-560). Also the DEGRADED mode CI actually runs — PyYAML shadowed out, freeze only (R-595)", } @@ -116,6 +119,37 @@ def case(name, clone, edits, expect_rc, must_contain=(), gate="check-engine-majo sh(["git", "reset", "-q", "--hard", base], cwd=clone) +def case_probe(name, clone, edits, expect_rc, must_contain=(), apps=("tandoor", "zipline", "wger", + "home-assistant")): + """The probe gate reads FILES in a checkout, so its cases need `--root` and no commit. + + Without `--root` the gate would read the REAL repo while the case edits the clone, every case + would see identical bytes, and every case would pass — the constant-for-measurement shape. The + app list is passed explicitly for the same reason: a whole-repo run is dominated by the three + genuine faults and would mask whether THIS case's edit changed anything. + """ + global ran + ran += 1 + base = sh(["git", "rev-parse", "HEAD"], cwd=clone).stdout.strip() + try: + for relpath, fn in edits: + edit(clone, relpath, fn) + r = sh([sys.executable, os.path.join(ROOT, "scripts", "check-probe-matches-compose.py"), + "--root=" + clone] + list(apps), cwd=clone) + out = r.stdout + r.stderr + ok = r.returncode == expect_rc and all(m in out for m in must_contain) + if ok: + print(" ok %-52s rc=%d (expected %d)" % (name, r.returncode, expect_rc)) + else: + fails.append("%s: rc=%d expected %d; missing %s\n%s" % ( + name, r.returncode, expect_rc, + [m for m in must_contain if m not in out], out[-900:])) + return out + finally: + sh(["git", "checkout", "-q", "--", "."], cwd=clone) + sh(["git", "reset", "-q", "--hard", base], cwd=clone) + + def case_copy(name, clone, edits, expect_rc, must_contain=(), extra_args=()): """The copy-i18n gate reads FILES, not commits, so its cases need neither a commit nor a range — but they DO need --root, or the gate would read the real repo and judge files nobody edited. @@ -458,6 +492,73 @@ i18n: [("templates/privatebin/docker-compose.yml", lambda t: t.replace("services:", "# Titkosított jegyzet és szöveg megosztás\nservices:", 1))], expect_rc=0, must_contain=("copy-i18n: OK",)) + + # ── probe-matches-compose (R-618) ──────────────────────────────────────────────────────── + # The FACT is where the app LISTENS inside its container. The LABEL is every other number + # in the file that looks like a port: the traefik label, `ports:`, `expose:`, a comment, + # and a sidecar's own healthcheck. A gate that reads any of those would have passed the + # three templates that stopped a working app on 2026-09-21. + BS = "templates/bookstack/.felhom.yml" + BSC = "templates/bookstack/docker-compose.yml" + + print("\n-- probe-matches-compose: the facts (each MUST be refused)") + # The three real faults, RE-INTRODUCED rather than read off the tree. Reading them off the + # tree passed only while the tree was broken; the case would have gone green for the wrong + # reason the moment R-618 was fixed, which is the `constant-for-measurement` shape again. + case_probe("FACT: tandoor's real R-618 port fault, re-introduced", clone, + [("templates/tandoor/.felhom.yml", lambda t: t.replace(" port: 80\n", + " port: 8080\n"))], + expect_rc=1, must_contain=("FAIL tandoor", "Nothing listens on 8080"), + apps=("tandoor",)) + case_probe("FACT: wger's real R-618 port fault, re-introduced", clone, + [("templates/wger/.felhom.yml", lambda t: t.replace(" port: 8000\n", + " port: 80\n"))], + expect_rc=1, must_contain=("FAIL wger", "dials 8000 on loopback"), + apps=("wger",)) + case_probe("FACT: zipline's real R-618 path fault, re-introduced", clone, + [("templates/zipline/.felhom.yml", + lambda t: t.replace('path: "/api/healthcheck"', 'path: "/api/health"'))], + expect_rc=1, must_contain=("FAIL zipline", "This probe CAN fail on it"), + apps=("zipline",)) + case_probe("GENUINE: the fixed tree passes", clone, [], expect_rc=0, + must_contain=("probe-matches-compose: OK",), + apps=("tandoor", "zipline", "wger")) + case_probe("FACT: a clean app given a wrong probe port", clone, + [(BS, lambda t: t.replace("port: 80", "port: 8080"))], + expect_rc=1, must_contain=("FAIL bookstack", "dials 80 on loopback"), + apps=("bookstack",)) + case_probe("FACT: api+expect probe given a path the app does not answer", clone, + [(BS, lambda t: t.replace(" - type: http\n port: 80", + " - type: api\n port: 80\n" + " path: /status\n expect:\n" + " status: 200"))], + expect_rc=1, must_contain=("FAIL bookstack", "This probe CAN fail on it"), + apps=("bookstack",)) + + print("-- probe-matches-compose: the decoys (the label moves, the fact does not)") + case_probe("DECOY: the port moves in a COMMENT in .felhom.yml", clone, + [(BS, lambda t: t.replace("healthcheck:", + "# the app listens on 8080 (decoy comment)\nhealthcheck:"))], + expect_rc=0, must_contain=("probe-matches-compose: OK",), apps=("bookstack",)) + case_probe("DECOY: traefik loadbalancer.server.port changed", clone, + [(BSC, lambda t: t.replace("loadbalancer.server.port=80", + "loadbalancer.server.port=9999"))], + expect_rc=0, must_contain=("probe-matches-compose: OK",), apps=("bookstack",)) + case_probe("DECOY: a published `ports:` mapping changed", clone, + [(BSC, lambda t: t.replace(" container_name: bookstack\n", + " container_name: bookstack\n" + " ports:\n - \"9999:80\"\n", 1))], + expect_rc=0, must_contain=("probe-matches-compose: OK",), apps=("bookstack",)) + case_probe("DECOY: a NON-probed sidecar's healthcheck port changed", clone, + [(BSC, lambda t: t.replace( + '["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]', + '["CMD", "curl", "-f", "http://127.0.0.1:7777/"]'))], + expect_rc=0, must_contain=("probe-matches-compose: OK",), apps=("bookstack",)) + case_probe("DECOY: path mismatch on a probe that CANNOT fail on it -> WARN, not FAIL", clone, + [], expect_rc=0, + must_contain=("WARN home-assistant", "Harmless TODAY"), + apps=("home-assistant",)) + finally: shutil.rmtree(clone, ignore_errors=True)