gates: probe-matches-compose runs in CI's no-PyYAML mode (R-618)
gates / gates (push) Successful in 2s

CI job 877 on 15d7c2b failed: the runner has no PyYAML and the gate answered INCONCLUSIVE, which
the runner rightly refuses to call a pass. A gate red on every push is bypassed within a week.

Falls back to a line reader and announces mode: DEGRADED. Over all 53 apps it returns exactly what
the full reader returns. Five more decoy cases with PyYAML shadowed out prove it still convicts the
three real faults; suite now 56 cases.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-22 11:02:53 +02:00
parent 15d7c2b7ed
commit 1ad1f34b6e
3 changed files with 168 additions and 11 deletions
+15
View File
@@ -1,3 +1,18 @@
## The probe gate now runs in CI's own degraded mode (2026-09-22, R-618)
**Caught by checking the push's CI run by job id, not by assuming it went green.** Job **877** on
`15d7c2b` read `conclusion: failure`. The CI runner has no PyYAML, the new gate answered
INCONCLUSIVE there, and the runner correctly refuses to call an undetermined result a pass.
A gate that is red on every CI push is bypassed within a week, so it would enforce nothing — the
R-421 shape one level up. Both files it reads use a tiny regular subset of YAML, so it now falls
back to a line reader and says `mode: DEGRADED` when it does. Over all 53 apps the degraded reader
returns **exactly** what the full reader returns: no convictions and the same six warnings.
Pinned by five more decoy cases with PyYAML shadowed out (suite now **56 cases**): the three real
faults re-introduced one at a time are still REFUSED, the fixed tree passes and says it is degraded,
and moving Traefik's label still convicts nothing.
## Fifteen proven versions move to the catalog (2026-09-22, R-462) ## Fifteen proven versions move to the catalog (2026-09-22, R-462)
Every one was walked on scratch guest 9202 **through the product's own guarded Update**, seeded and Every one was walked on scratch guest 9202 **through the product's own guarded Update**, seeded and
+95 -10
View File
@@ -56,15 +56,15 @@ USAGE
python3 scripts/check-probe-matches-compose.py --root=<dir> # judge another checkout python3 scripts/check-probe-matches-compose.py --root=<dir> # judge another checkout
Exit: 0 clean (warnings do not convict) · 1 convicted · 2 inconclusive (unreadable template). Exit: 0 clean (warnings do not convict) · 1 convicted · 2 inconclusive (unreadable template).
""" """
import io
import os import os
import re import re
import sys import sys
try: try:
import yaml import yaml
except ImportError: except ImportError: # pragma: no cover — this IS the CI path
print("check-probe-matches-compose: PyYAML is not installed — INCONCLUSIVE") yaml = None
sys.exit(2)
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TEMPLATES = os.path.join(ROOT, "templates") TEMPLATES = os.path.join(ROOT, "templates")
@@ -123,6 +123,84 @@ def endpoints(test_str):
return found return found
# ── DEGRADED MODE — the shape CI actually runs ───────────────────────────────────────────────────
#
# The CI runner has no PyYAML. A gate that answers INCONCLUSIVE there is a gate that is red on every
# push, and a gate that is red on every push is bypassed within a week — so it would enforce
# nothing, which is the R-421 shape one level up. Both files this gate reads are written in a tiny,
# regular subset of YAML (two-space indent, one key per line, a flow-sequence `test:`), so the two
# facts it needs can be read without a YAML parser at all.
#
# It is NOT a YAML parser and does not pretend to be one. It reads exactly two shapes, and when a
# file does not match them it says so and the app becomes a WARN — never a silent pass.
def _deg_healthchecks(path):
"""`healthcheck: / checks: / - type: … ` out of a .felhom.yml, without PyYAML."""
lines = io.open(path, encoding="utf-8").read().split("\n")
checks, inside, cur = [], False, None
for raw in lines:
line = raw.split("#", 1)[0].rstrip() if not raw.strip().startswith("#") else ""
if re.match(r"^healthcheck:\s*$", line):
inside = True
continue
if inside and line and not line.startswith(" "):
break # a new top-level key ends the block
if not inside or not line.strip():
continue
m = re.match(r"^\s*-\s*(\w+):\s*(.*)$", line)
if m: # a new list item
cur = {}
checks.append(cur)
k, v = m.group(1), m.group(2)
else:
m2 = re.match(r"^\s*(\w+):\s*(.*)$", line)
if not m2 or cur is None:
continue
k, v = m2.group(1), m2.group(2)
v = v.strip().strip('"').strip("'")
if k == "port" and v.isdigit():
cur["port"] = int(v)
elif k in ("type", "path", "method"):
cur[k] = v
elif k == "expect":
cur["expect"] = True # presence is all this gate needs
elif k == "status" and cur.get("expect"):
pass
return [c for c in checks if c.get("type")]
def _deg_services(path):
"""{service: {container_name, healthcheck_test}} out of a docker-compose.yml, without PyYAML."""
lines = io.open(path, encoding="utf-8").read().split("\n")
services, cur, in_services = {}, None, False
for raw in lines:
if raw.strip().startswith("#") or not raw.strip():
continue
if re.match(r"^services:\s*$", raw):
in_services = True
continue
if in_services and raw[:1] not in (" ", "\t") and raw.strip():
break
if not in_services:
continue
m = re.match(r"^ ([A-Za-z0-9_.\-]+):\s*$", raw)
if m: # a service block begins
cur = m.group(1)
services[cur] = {"container_name": None, "test": None}
continue
if cur is None:
continue
m = re.match(r"^ container_name:\s*(\S+)", raw)
if m:
services[cur]["container_name"] = m.group(1).strip('"').strip("'")
continue
m = re.match(r"^\s+test:\s*(.+)$", raw)
if m and services[cur]["test"] is None:
services[cur]["test"] = m.group(1)
return services
def probed_service(app, services): def probed_service(app, services):
"""The service the controller would probe — `findProbeContainer`'s rule, statically.""" """The service the controller would probe — `findProbeContainer`'s rule, statically."""
for name, svc in services.items(): for name, svc in services.items():
@@ -140,17 +218,23 @@ def check_app(app):
d = os.path.join(TEMPLATES, app) d = os.path.join(TEMPLATES, app)
bad, warn = [], [] bad, warn = [], []
try: try:
fy = yaml.safe_load(open(os.path.join(d, ".felhom.yml"), encoding="utf-8")) or {} if yaml is not None:
cy = yaml.safe_load(open(os.path.join(d, "docker-compose.yml"), encoding="utf-8")) or {} fy = yaml.safe_load(open(os.path.join(d, ".felhom.yml"), encoding="utf-8")) or {}
cy = yaml.safe_load(open(os.path.join(d, "docker-compose.yml"), encoding="utf-8")) or {}
checks = ((fy.get("healthcheck") or {}).get("checks")) or []
services = cy.get("services") or {}
else:
checks = _deg_healthchecks(os.path.join(d, ".felhom.yml"))
services = {n: {"container_name": v["container_name"],
"healthcheck": {"test": v["test"]}}
for n, v in _deg_services(os.path.join(d, "docker-compose.yml")).items()}
except Exception as e: # noqa: BLE001 — report, never crash except Exception as e: # noqa: BLE001 — report, never crash
return None, ["%s: template unreadable (%s)" % (app, e)] return None, ["%s: template unreadable (%s)" % (app, e)]
checks = ((fy.get("healthcheck") or {}).get("checks")) or []
if not checks: if not checks:
warn.append("%s: no `healthcheck.checks` in .felhom.yml — nothing to compare" % app) warn.append("%s: no `healthcheck.checks` in .felhom.yml — nothing to compare" % app)
return bad, warn return bad, warn
services = cy.get("services") or {}
svc_name, svc = probed_service(app, services) svc_name, svc = probed_service(app, services)
if svc is None: if svc is None:
# Not a conviction, but it is worse than a mismatch: `findProbeContainer` returns "" and # Not a conviction, but it is worse than a mismatch: `findProbeContainer` returns "" and
@@ -220,7 +304,8 @@ def main(argv):
if not apps: if not apps:
apps = sorted(d for d in os.listdir(TEMPLATES) apps = sorted(d for d in os.listdir(TEMPLATES)
if os.path.isdir(os.path.join(TEMPLATES, d))) if os.path.isdir(os.path.join(TEMPLATES, d)))
print("probe-matches-compose: %d app(s)" % len(apps)) mode = "full" if yaml is not None else "DEGRADED (no PyYAML — line reader; this is the CI path)"
print("probe-matches-compose: %d app(s), mode: %s" % (len(apps), mode))
convicted, warnings, unreadable = [], [], 0 convicted, warnings, unreadable = [], [], 0
for app in sorted(apps): for app in sorted(apps):
@@ -244,8 +329,8 @@ def main(argv):
for b in convicted: for b in convicted:
print(" FAIL " + b) print(" FAIL " + b)
return 1 return 1
print("\nprobe-matches-compose: OK — every probe dials the port (and, where it can fail, the " print("\nprobe-matches-compose: OK%s — every probe dials the port (and, where it can fail, the "
"path) that the app's own compose healthcheck dials") "path) that the app's own compose healthcheck dials" % (" (degraded)" if yaml is None else ""))
return 0 return 0
+58 -1
View File
@@ -48,7 +48,7 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
COVERS = { COVERS = {
"engine-major": "the major moved in a comment/env var/README/app image, not on an engine's image: line", "engine-major": "the major moved in a comment/env var/README/app image, not on an engine's image: line",
"catalog-since": "the date bumped in a comment/README while .felhom.yml's field stayed; or only a comment/env moved, no image (R-452)", "catalog-since": "the date bumped in a comment/README while .felhom.yml's field stayed; or only a comment/env moved, no image (R-452)",
"probe-matches-compose": "the port/path moved in a COMMENT, in traefik's loadbalancer label, in " "probe-matches-compose": "the DEGRADED no-PyYAML mode CI actually runs; the port/path moved in a COMMENT, in traefik's loadbalancer label, in "
"`ports:`/`expose:`, or on a NON-probed service - none of which is where " "`ports:`/`expose:`, or on a NON-probed service - none of which is where "
"the app listens; vs a real probe port/path that the app does not answer (R-618)", "the app listens; vs a real probe port/path that the app does not answer (R-618)",
"copy-i18n": "Hungarian edited in a COMMENT/README/display_name (label, not copy) vs a real frozen string changed; an English block that is not English, is not matched to a Hungarian twin, or rewrites a credential (R-560). Also the DEGRADED mode CI actually runs — PyYAML shadowed out, freeze only (R-595)", "copy-i18n": "Hungarian edited in a COMMENT/README/display_name (label, not copy) vs a real frozen string changed; an English block that is not English, is not matched to a Hungarian twin, or rewrites a credential (R-560). Also the DEGRADED mode CI actually runs — PyYAML shadowed out, freeze only (R-595)",
@@ -150,6 +150,38 @@ def case_probe(name, clone, edits, expect_rc, must_contain=(), apps=("tandoor",
sh(["git", "reset", "-q", "--hard", base], cwd=clone) sh(["git", "reset", "-q", "--hard", base], cwd=clone)
def case_probe_noyaml(name, clone, edits, expect_rc, must_contain=(), apps=()):
"""The same cases with PyYAML SHADOWED OUT — the mode the CI runner actually has.
A degraded mode that always passes is worse than no gate, because the summary says OK. So the
three real faults are re-introduced here too and must still be REFUSED by the line reader.
"""
global ran
ran += 1
base = sh(["git", "rev-parse", "HEAD"], cwd=clone).stdout.strip()
nd = noyaml_dir()
env = dict(os.environ, PYTHONPATH=nd + os.pathsep + os.environ.get("PYTHONPATH", ""))
try:
for relpath, fn in edits:
edit(clone, relpath, fn)
r = subprocess.run([sys.executable, os.path.join(ROOT, "scripts",
"check-probe-matches-compose.py"),
"--root=" + clone] + list(apps),
cwd=clone, capture_output=True, text=True, env=env)
out = r.stdout + r.stderr
ok = r.returncode == expect_rc and all(m in out for m in must_contain)
if ok:
print(" ok %-52s rc=%d (expected %d)" % (name, r.returncode, expect_rc))
else:
fails.append("%s: rc=%d expected %d; missing %s\n%s" % (
name, r.returncode, expect_rc,
[m for m in must_contain if m not in out], out[-900:]))
return out
finally:
sh(["git", "checkout", "-q", "--", "."], cwd=clone)
sh(["git", "reset", "-q", "--hard", base], cwd=clone)
def case_copy(name, clone, edits, expect_rc, must_contain=(), extra_args=()): def case_copy(name, clone, edits, expect_rc, must_contain=(), extra_args=()):
"""The copy-i18n gate reads FILES, not commits, so its cases need neither a commit nor a range — """The copy-i18n gate reads FILES, not commits, so its cases need neither a commit nor a range —
but they DO need --root, or the gate would read the real repo and judge files nobody edited. but they DO need --root, or the gate would read the real repo and judge files nobody edited.
@@ -559,6 +591,31 @@ i18n:
must_contain=("WARN home-assistant", "Harmless TODAY"), must_contain=("WARN home-assistant", "Harmless TODAY"),
apps=("home-assistant",)) apps=("home-assistant",))
print("-- probe-matches-compose: the DEGRADED mode CI actually runs (no PyYAML)")
case_probe_noyaml("DEGRADED: the fixed tree passes, and SAYS it is degraded", clone, [],
expect_rc=0, must_contain=("mode: DEGRADED", "OK (degraded)"),
apps=("tandoor", "zipline", "wger"))
case_probe_noyaml("DEGRADED FACT: tandoor's port fault still refused", clone,
[("templates/tandoor/.felhom.yml",
lambda t: t.replace(" port: 80\n", " port: 8080\n"))],
expect_rc=1, must_contain=("FAIL tandoor", "Nothing listens on 8080"),
apps=("tandoor",))
case_probe_noyaml("DEGRADED FACT: wger's port fault still refused", clone,
[("templates/wger/.felhom.yml",
lambda t: t.replace(" port: 8000\n", " port: 80\n"))],
expect_rc=1, must_contain=("FAIL wger",), apps=("wger",))
case_probe_noyaml("DEGRADED FACT: zipline's path fault still refused", clone,
[("templates/zipline/.felhom.yml",
lambda t: t.replace('path: "/api/healthcheck"', 'path: "/api/health"'))],
expect_rc=1, must_contain=("FAIL zipline", "This probe CAN fail on it"),
apps=("zipline",))
case_probe_noyaml("DEGRADED DECOY: traefik label moves, the fact does not", clone,
[("templates/bookstack/docker-compose.yml",
lambda t: t.replace("loadbalancer.server.port=80",
"loadbalancer.server.port=9999"))],
expect_rc=0, must_contain=("OK (degraded)",), apps=("bookstack",))
finally: finally:
shutil.rmtree(clone, ignore_errors=True) shutil.rmtree(clone, ignore_errors=True)