From ab2b304987c50b5198f6fb4af87ba332727ce4c7 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 5 Oct 2026 16:48:12 +0200 Subject: [PATCH] R-885: script-tests gate runs every scripts/**/test_*.py on every push (decoys, red-proofs); closed (336 -> 335) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../burndown-2026-10-05/r885-red-proof.txt | 16 ++++ scripts/CHANGELOG.md | 11 +++ scripts/hub-db-backup/test_hub_db_backup.py | 23 +++++- scripts/repo_gates.py | 3 + scripts/script_tests_gate.py | 73 +++++++++++++++++++ scripts/test_gate_decoys.py | 16 ++++ scripts/test_script_tests_gate.py | 62 ++++++++++++++++ 7 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 documentation/audits/burndown-2026-10-05/r885-red-proof.txt create mode 100644 scripts/script_tests_gate.py create mode 100644 scripts/test_script_tests_gate.py diff --git a/documentation/audits/burndown-2026-10-05/r885-red-proof.txt b/documentation/audits/burndown-2026-10-05/r885-red-proof.txt new file mode 100644 index 00000000..56d1b317 --- /dev/null +++ b/documentation/audits/burndown-2026-10-05/r885-red-proof.txt @@ -0,0 +1,16 @@ +### R885-a: gate ignores a suite's exit code +FAIL prints OK, exits 1 want rc=1 got rc=0 +FAIL failing suite 3 levels deep want rc=1 got rc=0 +ok empty scripts/ want rc=1 got rc=1 +ok genuine: two passing suites want rc=0 got rc=0 +ok nested run steps aside want rc=0 got rc=0 +script-tests decoys: 3/5 +rc=1 +### R885-b: gate passes when it found nothing +ok prints OK, exits 1 want rc=1 got rc=1 +ok failing suite 3 levels deep want rc=1 got rc=1 +FAIL empty scripts/ want rc=1 got rc=0 +ok genuine: two passing suites want rc=0 got rc=0 +ok nested run steps aside want rc=0 got rc=0 +script-tests decoys: 4/5 +rc=1 diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index b925a22c..a9b8751f 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -1,3 +1,14 @@ +## gates — `script-tests`: every Python test suite under scripts/ runs on every push (R-885) (2026-10-05) + +- `scripts/script_tests_gate.py` (registered in `repo_gates.py`, fast): walks `scripts/` for `test_*.py` and runs each; + any non-zero exit, a timeout (300 s) or finding no suite fails the gate. Nested runs (`test_repo_gates.py` runs the + runner) step aside via `FELHOM_SCRIPT_TESTS_GATE`. 13 suites today, ~20 s. +- `scripts/test_script_tests_gate.py`: 5 decoys (prints OK but exits 1; failing suite 3 levels deep; empty tree; + genuine; nested), run from `test_gate_decoys.py`, which declares the gate in `COVERS`. Red-proofs: ignoring the exit + code → 2 decoys fail; passing an empty tree → 1 decoy fails. +- `scripts/hub-db-backup/test_hub_db_backup.py`: a Python-sqlite3 stand-in when the `sqlite3` CLI is absent (the CI + runner has python3 and git only); 15/15 pass with it. + ## hub-db-backup 1.0 — the hub database leaves DooPlex every night, encrypted, and is restore-tested weekly (R-173) (2026-10-05) New, in `scripts/hub-db-backup/` (versioned from day one, R-231): `felhom-hub-db-backup` (02:30 — copy the hub's newest diff --git a/scripts/hub-db-backup/test_hub_db_backup.py b/scripts/hub-db-backup/test_hub_db_backup.py index 6b092071..fdbabf33 100644 --- a/scripts/hub-db-backup/test_hub_db_backup.py +++ b/scripts/hub-db-backup/test_hub_db_backup.py @@ -64,6 +64,24 @@ sys.exit(98) ''' +# The CI runner carries python3 and git only (no sqlite3 CLI). When the real `sqlite3` is absent, a stand-in built on +# Python's own sqlite3 module (the SAME SQLite library) answers exactly the calls the scripts make: +# `sqlite3 -readonly ''`, one row per line, "|"-separated, exit 1 + stderr on an error. Said out loud below. +FAKE_SQLITE3 = r'''#!/usr/bin/env python3 +import sqlite3, sys +a = [x for x in sys.argv[1:] if x != "-readonly"] +try: + db = sqlite3.connect("file:" + a[0] + "?mode=ro", uri=True) + for row in db.execute(a[1]): + print("|".join("" if v is None else str(v) for v in row)) +except Exception as e: + print("Error: " + str(e), file=sys.stderr); sys.exit(1) +''' +REAL_SQLITE3 = shutil.which("sqlite3") +if not REAL_SQLITE3: + print("test_hub_db_backup: no sqlite3 CLI here — using the Python-sqlite3 stand-in (same SQLite library)") + + def make_db(path, hosts=2, recovery=("enc:v1:abc", "enc:v1:def"), corrupt=False): db = sqlite3.connect(path) db.execute("CREATE TABLE hosts (host_id TEXT)") @@ -90,7 +108,10 @@ class Base(unittest.TestCase): j = lambda *p: os.path.join(self.t, *p) for d in ("bin", "pod/snapshots", "conf", "state", "textfile", "pbs"): os.makedirs(j(d), exist_ok=True) - for name, body in (("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS)): + fakes = [("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS)] + if not REAL_SQLITE3: + fakes.append(("sqlite3", FAKE_SQLITE3)) + for name, body in fakes: p = j("bin", name); open(p, "w").write(body); os.chmod(p, 0o755) open(j("conf", "env"), "w").write( "PBS_REPOSITORY_PUSH='dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite'\n" diff --git a/scripts/repo_gates.py b/scripts/repo_gates.py index 059b3db2..dcef373b 100644 --- a/scripts/repo_gates.py +++ b/scripts/repo_gates.py @@ -152,6 +152,9 @@ GATES = [ # nothing noticed until a person read the file. R-254 had been missing its state cell since # 2026-08-08 — 45 days — for the same reason. Fast: one file read. ("register-shape", os.path.join(SCRIPTS, "register_shape_gate.py"), [], True, False), + # R-885 — the Python test suites under scripts/ (found by a walk) ran only by hand; a change that broke the + # 15 tests of the DooPlex hub-DB scripts, or the 73 of the instructions gate, reached main green. ~20 s. + ("script-tests", os.path.join(SCRIPTS, "script_tests_gate.py"), [ROOT], True, False), # R-421 — every registered gate across all four repos ships with a decoy test, or is # named in the exemption list with its row. Registered LAST, after every runner was green: # a failing gate refuses every push, which is what instructions_gate learned the hard way. diff --git a/scripts/script_tests_gate.py b/scripts/script_tests_gate.py new file mode 100644 index 00000000..0daa54fa --- /dev/null +++ b/scripts/script_tests_gate.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""script_tests_gate.py — every Python test suite under scripts/ runs on every push (R-885). + +Usage: python3 scripts/script_tests_gate.py [] +Exit 0 every suite exited 0 · 1 a suite failed, timed out, or NO suite was found. + +WHY. `repo_gates.py` ran gates, not the `test_*.py` suites beside them, so a change that broke one — the 15 tests of +the DooPlex hub-DB push/restore scripts, the 73 of the instructions gate — was caught only if someone ran it by hand. + +SCOPE IS A FACT (R-421): the suites are FOUND by walking scripts/ (os.walk, every depth), never listed by hand, so a new +suite is covered the day it is added. Finding none is a FAILURE — a gate that checked nothing must not read as green. +A suite's verdict is its EXIT CODE, never its output: a suite that prints "OK" and exits 1 has failed. + +NESTING. `test_repo_gates.py` runs `repo_gates.py`, which runs this gate, which would run `test_repo_gates.py` again. +The gate marks its children with FELHOM_SCRIPT_TESTS_GATE=1 and, when it finds the mark, steps aside (exit 0, said +out loud). Pinned by scripts/test_script_tests_gate.py (decoys). +""" +import os +import subprocess +import sys +import time + +MARK = "FELHOM_SCRIPT_TESTS_GATE" +TIMEOUT_S = 300 +SKIP_DIRS = {"__pycache__", ".git", "node_modules"} + + +def find_suites(scripts_dir): + out = [] + for dp, dns, fns in os.walk(scripts_dir): + dns[:] = sorted(d for d in dns if d not in SKIP_DIRS) + for f in sorted(fns): + if f.startswith("test_") and f.endswith(".py"): + out.append(os.path.join(dp, f)) + return out + + +def main(argv): + if os.environ.get(MARK): + print("script-tests: nested run (inside a suite this gate started) — stepping aside") + return 0 + root = os.path.abspath(argv[1]) if len(argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + suites = find_suites(os.path.join(root, "scripts")) + if not suites: + print("script-tests: FAILED — no test_*.py found under %s/scripts (a gate that checked nothing is not green)" % root) + return 1 + env = dict(os.environ, **{MARK: "1"}) + failed = [] + for s in suites: + rel = os.path.relpath(s, root) + t0 = time.time() + try: + p = subprocess.run([sys.executable, "-W", "ignore", s], cwd=root, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=TIMEOUT_S) + rc, tail = p.returncode, p.stdout.decode("utf-8", "replace").strip().splitlines()[-3:] + except subprocess.TimeoutExpired: + rc, tail = "timeout", ["(killed after %d s)" % TIMEOUT_S] + verdict = "OK " if rc == 0 else "FAIL" + print(" %s %-55s %5.1fs rc=%s" % (verdict, rel, time.time() - t0, rc)) + if rc != 0: + failed.append(rel) + for line in tail: + print(" | " + line[:200]) + if failed: + print("script-tests: FAILED — %d of %d suite(s): %s" % (len(failed), len(suites), ", ".join(failed))) + return 1 + print("script-tests: OK — %d suite(s), every one exited 0" % len(suites)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/test_gate_decoys.py b/scripts/test_gate_decoys.py index e3a6fdcf..74d09e3f 100644 --- a/scripts/test_gate_decoys.py +++ b/scripts/test_gate_decoys.py @@ -52,6 +52,9 @@ COVERS = { "(2026-10-03) an old-shape row under the new header, a near-miss category, an " "old rank tag as Sev, an undefined state word, and a pipe outside backticks"), "decoy-coverage": "a gate registered in a runner with no decoy and no exemption (its red-proof)", + "script-tests": ("R-885: FIVE cases in scripts/test_script_tests_gate.py, run from here: a suite that PRINTS " + "OK and exits 1 (label without fact), a failing suite three levels deep (scope is a walk), " + "an empty scripts/ tree (checked nothing), the genuine article (must pass), the nested mark"), "guide-quote": ("R-596: SEVEN cases in scripts/test_guide_quote_gate.py, run from here so " "this suite stays the single entry point. The load-bearing one is " "name-for-fact: a guide that lists every key in a table and quotes none of " @@ -353,6 +356,19 @@ else: _n = _gq.stdout.strip().splitlines()[-1] if _gq.stdout.strip() else "?" print(" ok %-20s %s" % ("guide-quote", _n)) +# ── script-tests (R-885) ───────────────────────────────────────────────────────────────────────── +# +# Its decoys are whole fake repos, so they live in their own file and are RUN from here (guide-quote's shape). +ran += 1 +_st = subprocess.run([sys.executable, os.path.join("scripts", "test_script_tests_gate.py")], + cwd=ROOT, capture_output=True, text=True) +if _st.returncode != 0: + fails.append("script-tests: its decoy suite FAILED — a decoy did not convict\n%s" + % (_st.stdout + _st.stderr)[-800:]) +else: + _n = _st.stdout.strip().splitlines()[-1] if _st.stdout.strip() else "?" + print(" ok %-20s %s" % ("script-tests", _n)) + # ── register-shape (R-627) ─────────────────────────────────────────────────────────────────────── # # THE DECOYS ARE THE REAL DAMAGE, not invented shapes. On 2026-09-21 an append regex ate two rows' diff --git a/scripts/test_script_tests_gate.py b/scripts/test_script_tests_gate.py new file mode 100644 index 00000000..74150c15 --- /dev/null +++ b/scripts/test_script_tests_gate.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Decoys for scripts/script_tests_gate.py (R-885, R-421). Run: python3 scripts/test_script_tests_gate.py + +Each decoy is a fake repo with a scripts/ tree. The gate must REFUSE: a suite that prints "OK" but exits 1 (the label +without the fact); a failing suite three directories deep (scope is a walk, not a list); an empty scripts/ (a gate that +checked nothing). It must PASS the genuine article, and step aside when nested. +""" +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +GATE = os.path.join(HERE, "script_tests_gate.py") + +COVERS = { + "script-tests": ("a suite that PRINTS OK and exits 1; a failing suite three levels deep; an empty scripts/ " + "tree; plus the genuine article (must pass) and the nested-run mark (must step aside)"), +} + + +def fake_repo(files): + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "scripts")) + for rel, body in files.items(): + p = os.path.join(root, "scripts", rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + open(p, "w").write(body) + return root + + +def gate(root, nested=False): + env = {k: v for k, v in os.environ.items() if k != "FELHOM_SCRIPT_TESTS_GATE"} + if nested: + env["FELHOM_SCRIPT_TESTS_GATE"] = "1" + return subprocess.run([sys.executable, GATE, root], env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).returncode + + +CASES = [ + ("prints OK, exits 1", {"test_a.py": "print('OK')\nraise SystemExit(1)\n"}, False, 1), + ("failing suite 3 levels deep", {"test_ok.py": "pass\n", "x/y/z/test_deep.py": "raise SystemExit(2)\n"}, False, 1), + ("empty scripts/", {"README": "no tests here\n"}, False, 1), + ("genuine: two passing suites", {"test_a.py": "pass\n", "sub/test_b.py": "print('fine')\n"}, False, 0), + ("nested run steps aside", {"test_a.py": "raise SystemExit(1)\n"}, True, 0), +] + + +def main(): + bad = 0 + for name, files, nested, want in CASES: + got = gate(fake_repo(files), nested) + ok = got == want + bad += not ok + print("%s %-32s want rc=%d got rc=%d" % ("ok " if ok else "FAIL", name, want, got)) + print("script-tests decoys: %d/%d" % (len(CASES) - bad, len(CASES))) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main())