Files
felhom.eu/scripts/test_instructions_gate.py
T
admin f49b1f390b
gates / gates (push) Successful in 7s
workspace: the root CLAUDE.md becomes a symlink; check 5 learns two shapes (R-230(b))
Two files kept identical by hand and one check is a divergence class policed forever; one file
reachable by two paths cannot diverge at all. install_workspace.py now links by default, MIGRATES
an existing regular file (backing it up first and SAYING SO if it differed -- that difference is
the last chance to notice an unsynced edit), and keeps --copy for a clone that wants the old shape.

Check 5 asserts a different thing per shape: for a link, that it points at the versioned copy and
resolves to a real file; for two files, byte-identity as before. A dangling link is worse than a
diverged copy -- the instructions load NOTHING and there is no content left to notice is wrong --
so that case is red-proofed.

NOT yet proven to LOAD: that needs a fresh session and a hook line, which is Phase 7. If it does
not load, this reverts to the copy.
2026-08-06 11:41:36 +02:00

533 lines
23 KiB
Python

# -*- coding: utf-8 -*-
"""Fixture tests for instructions_gate.py.
Run: python3 scripts/test_instructions_gate.py
Every test asserts the EFFECT — the gate's exit code AND that its message names the file and the
reason — not merely that "it ran". A gate that exits non-zero for the wrong reason is not a gate.
The negatives matter as much as the positives here: the whole point of the HTML-comment convention
is that commented content costs nothing, so `test_comments_do_not_count_toward_ceiling` and
`test_version_literal_in_comment_is_allowed` are what stop the gate from punishing the very move it
is meant to encourage.
"""
import os
import shutil
import subprocess
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
GATE = os.path.join(HERE, "instructions_gate.py")
PASSED = []
FAILED = []
def run_gate(root, today="2026-08-06"):
env = dict(os.environ, FELHOM_GATE_TODAY=today)
p = subprocess.run(
[sys.executable, GATE, root],
capture_output=True,
text=True,
env=env,
)
return p.returncode, p.stdout + p.stderr
def make_repo(tmp, claude_md, rules=None):
root = os.path.join(tmp, "repo")
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "CLAUDE.md"), "w", encoding="utf-8") as fh:
fh.write(claude_md)
if rules:
rd = os.path.join(root, ".claude", "rules")
os.makedirs(rd, exist_ok=True)
for name, body in rules.items():
with open(os.path.join(rd, name), "w", encoding="utf-8") as fh:
fh.write(body)
return root
def check(name, cond, detail=""):
(PASSED if cond else FAILED).append(name + (("" + detail) if detail else ""))
print((" PASS " if cond else " FAIL ") + name + ((" " + detail) if detail else ""))
def test_short_file_passes():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n" + "a line\n" * 150)
rc, out = run_gate(root)
check("150-line CLAUDE.md exits 0", rc == 0, "rc=%d" % rc)
def test_long_file_fails_and_names_file_and_count():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n" + "a line\n" * 204)
rc, out = run_gate(root)
check("205-line CLAUDE.md exits non-zero", rc != 0, "rc=%d" % rc)
check("message names the file", "CLAUDE.md" in out)
check("message names the effective line count", "205 effective lines" in out)
check(
"message says ADHERENCE, not space",
"ADHERENCE limit, not a space limit" in out,
)
def test_comments_do_not_count_toward_ceiling():
"""The convention's load-bearing negative: 400 commented lines must not trip the ceiling."""
with tempfile.TemporaryDirectory() as tmp:
body = "# Repo\n" + "a line\n" * 100 + "<!--\n" + "history\n" * 400 + "-->\n"
root = make_repo(tmp, body)
rc, out = run_gate(root)
check("400 commented lines do not trip the ceiling", rc == 0, "rc=%d" % rc)
def test_rule_with_paths_passes():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(
tmp,
"# Repo\n",
rules={"scoped.md": '---\npaths: ["**/*.go"]\n---\n\n# Scoped\n'},
)
rc, out = run_gate(root)
check("rule with paths: exits 0", rc == 0, "rc=%d" % rc)
def test_rule_without_paths_fails():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(
tmp, "# Repo\n", rules={"bare.md": "---\nname: bare\n---\n\n# Bare\n"}
)
rc, out = run_gate(root)
check("rule with neither marker exits non-zero", rc != 0, "rc=%d" % rc)
check("message names the rule file", "bare.md" in out)
def test_rule_with_unconditional_marker_passes():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(
tmp,
"# Repo\n",
rules={"always.md": "---\nunconditional: true\n---\n\n# Always\n"},
)
rc, out = run_gate(root)
check("explicit unconditional: true exits 0", rc == 0, "rc=%d" % rc)
def test_version_literal_fails():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n\nThe box runs agent 0.93.0 today.\n")
rc, out = run_gate(root)
check("version literal exits non-zero", rc != 0, "rc=%d" % rc)
check("message quotes the version", "0.93.0" in out)
def test_ipv4_is_not_a_version():
"""A bare \\d+\\.\\d+\\.\\d+ matches the first three octets of every IPv4."""
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n\nDooPlex is 192.168.0.180 and the demo box is 10.0.0.1.\n")
rc, out = run_gate(root)
check("IPv4 addresses are not flagged as versions", rc == 0, "rc=%d" % rc)
def test_version_literal_in_comment_is_allowed():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n\n<!-- fixed in hub v0.97.0 -->\n")
rc, out = run_gate(root)
check("version literal inside a comment is allowed", rc == 0, "rc=%d" % rc)
def test_expired_temporary_fails():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(
tmp,
"# Repo\n\n> **TEMPORARY — host is away (until ~2026-08-02).**\n> Delete on return.\n",
)
rc, out = run_gate(root)
check("expired TEMPORARY block exits non-zero", rc != 0, "rc=%d" % rc)
check("message quotes the past date", "2026-08-02" in out)
def test_future_temporary_passes():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(
tmp, "# Repo\n\n> **TEMPORARY — host is away (until ~2026-12-31).**\n"
)
rc, out = run_gate(root)
check("TEMPORARY with a future date exits 0", rc == 0, "rc=%d" % rc)
def test_missing_claude_md_fails():
with tempfile.TemporaryDirectory() as tmp:
root = os.path.join(tmp, "repo")
os.makedirs(root)
rc, out = run_gate(root)
check("absent CLAUDE.md is a FAILURE, not a skip", rc != 0, "rc=%d" % rc)
def test_diverged_workspace_copy_fails():
with tempfile.TemporaryDirectory() as tmp:
ws = os.path.join(tmp, "ws")
root = os.path.join(ws, "repo")
os.makedirs(root)
with open(os.path.join(root, "CLAUDE.md"), "w", encoding="utf-8") as fh:
fh.write("# Repo\n")
with open(os.path.join(ws, "CLAUDE.md"), "w", encoding="utf-8") as fh:
fh.write("# Workspace live\n")
cp = os.path.join(ws, "felhom.eu", "documentation", "runbooks")
os.makedirs(cp)
with open(os.path.join(cp, "workspace-CLAUDE.md"), "w", encoding="utf-8") as fh:
fh.write("# Workspace copy — DIVERGED\n")
rc, out = run_gate(root)
check("diverged workspace copy exits non-zero", rc != 0, "rc=%d" % rc)
check("message names both files", "workspace-CLAUDE.md" in out)
# --- memory-index checks (gate check 6) ---------------------------------------------------------
#
# The three outcomes are deliberately different and each has a test proving it is the one intended:
# over-limit FAILS (silent truncation), an orphan WARNS (the store is outside git), and an absent
# store PASSES *while printing its reason* (machine-local by design). The last one is the dangerous
# grant — an exception to "a missing input is a FAILURE" — so it is asserted on the reason text, not
# just on rc == 0. A pass with no reason is indistinguishable from a gate that stopped running.
def make_memory(tmp, index, topics=None, archived=None):
"""Create <tmp>/.claude-memory. The gate resolves the store from the WORKSPACE root, which for
make_repo(tmp, ...) is tmp itself."""
store = os.path.join(tmp, ".claude-memory")
os.makedirs(store, exist_ok=True)
if index is not None:
with open(os.path.join(store, "MEMORY.md"), "w", encoding="utf-8") as fh:
fh.write(index)
for name, body in (topics or {}).items():
with open(os.path.join(store, name), "w", encoding="utf-8") as fh:
fh.write(body)
if archived:
arc = os.path.join(store, "archive")
os.makedirs(arc, exist_ok=True)
for name, body in archived.items():
with open(os.path.join(arc, name), "w", encoding="utf-8") as fh:
fh.write(body)
return store
def test_memory_absent_passes_and_prints_the_reason():
with tempfile.TemporaryDirectory() as tmp:
root = make_repo(tmp, "# Repo\n") # no .claude-memory at all
rc, out = run_gate(root)
check("absent memory store exits 0", rc == 0, "rc=%d" % rc)
check("absent memory store PRINTS its reason", "machine-local by design" in out)
check("absent memory store is not reported as a skip", "PASS with reason" in out)
def test_memory_within_limits_passes():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md)\n", topics={"a.md": "# A\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("memory index within limits exits 0", rc == 0, "rc=%d" % rc)
check("tally reports indexed/orphaned/archived", "1 indexed, 0 orphaned" in out)
def test_memory_over_line_limit_fails_and_names_file_and_count():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n" + "- entry\n" * 250)
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("over-length memory index exits non-zero", rc != 0, "rc=%d" % rc)
check("message names MEMORY.md", "MEMORY.md" in out)
check("message names the line count", "251 lines" in out)
check("message says silent truncation, not space", "DROPPED WITH NO" in out)
def test_memory_over_byte_limit_fails():
"""Few lines, many bytes — the byte ceiling must bite independently of the line ceiling."""
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n" + ("x" * 30000) + "\n")
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("over-byte memory index exits non-zero", rc != 0, "rc=%d" % rc)
check("byte failure names the byte ceiling", "25600" in out)
def test_memory_orphan_warns_but_does_not_fail():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md)\n",
topics={"a.md": "# A\n", "lonely.md": "# Lonely\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("an orphan does NOT fail the gate", rc == 0, "rc=%d" % rc)
check("an orphan produces a WARNING", "WARNING" in out)
check("the warning names the orphan", "lonely.md" in out)
def test_memory_wikilink_counts_as_a_reference():
"""Load-bearing negative: a file reachable only via [[wikilink]] is indexed, not orphaned.
Calling it an orphan would push someone to add a duplicate row for it."""
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md) — see [[deep]]\n",
topics={"a.md": "# A\n", "deep.md": "# Deep\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("wikilink-only file is not an orphan", rc == 0 and "WARNING" not in out, "rc=%d" % rc)
def test_memory_archived_file_is_not_an_orphan():
"""archive/ is where deliberately set-aside records live; they are not top-level and must not
be counted as unreferenced."""
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md)\n", topics={"a.md": "# A\n"},
archived={"old-campaign.md": "# Old\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("archived file is not an orphan", rc == 0 and "WARNING" not in out, "rc=%d" % rc)
check("tally counts the archived file", "1 archived" in out)
def test_memory_store_without_index_fails():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, None, topics={"a.md": "# A\n"}) # store exists, no MEMORY.md
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("store with no index exits non-zero", rc != 0, "rc=%d" % rc)
check("message says unreachable knowledge", "unreachable knowledge" in out)
# --- check 7: register citations ----------------------------------------------------------------
#
# The founding case is "CI is still owed (R-168)" shipped four days after R-168 closed, in four
# files, one of which contradicted itself. The trigger is an OPENNESS CLAIM, not any citation:
# requiring every mention of a non-open item to carry "CLOSED" would fire on ~30 legitimate
# provenance citations like "(R-161)" or "R-117 spike §6.3", and a gate that noisy gets switched
# off — which is the fate R-29 documented.
def make_register(tmp, rows):
"""rows: list of (item, what, state-cell). Mirrors the real 6-column table shape."""
d = os.path.join(tmp, "felhom.eu", "documentation", "backlog")
os.makedirs(d, exist_ok=True)
body = ["| ID | What | State | Blocked on | Next action | Owner |", "|---|---|---|---|---|---|"]
for item, what, st in rows:
body.append("| **%s** | %s | %s | — | — | operator |" % (item, what, st))
with open(os.path.join(d, "OPEN-ITEMS.md"), "w", encoding="utf-8") as fh:
fh.write("\n".join(body) + "\n")
with open(os.path.join(d, "ROADMAP.md"), "w", encoding="utf-8") as fh:
fh.write("| R-900 | a historical item, closed long ago and pruned from OPEN-ITEMS |\n")
REG_ROWS = [
("R-168", "~~CI: no runner exists~~", "**SHIPPED — and the alarm is DEMONSTRATED** (2026-08-02)"),
("R-161", "volume-persistence gate", "**REDUCED SCOPE — open** (operator ruling)"),
]
def test_citation_claiming_open_about_a_closed_item_fails():
"""The founding case, verbatim."""
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(tmp, "# Repo\n\nBoth facts are why CI is still owed (`OPEN-ITEMS.md` R-168).\n")
rc, out = run_gate(root)
check("stale-open citation exits non-zero", rc != 0, "rc=%d" % rc)
check("message names the item", "R-168" in out)
check("message says the register disagrees", "register says it is closed" in out)
def test_citation_that_says_closed_passes():
"""Explicitly required by the spec: a correct citation must survive."""
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(
tmp, "# Repo\n\na bypass is noticed even though it is not blocked (R-168, CLOSED 2026-08-02).\n"
)
rc, out = run_gate(root)
check("citation saying CLOSED exits 0", rc == 0, "rc=%d" % rc)
def test_bare_provenance_citation_of_a_closed_item_passes():
"""Load-bearing negative: citing a closed item as the SOURCE OF A FACT is not a claim it is open."""
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(tmp, "# Repo\n\nThe canonical runner shape (R-168) is what survives.\n")
rc, out = run_gate(root)
check("provenance citation of a closed item exits 0", rc == 0, "rc=%d" % rc)
def test_openness_claim_about_an_actually_open_item_passes():
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(tmp, "# Repo\n\n**R-161 stays open at reduced scope** — enforcement is by convention.\n")
rc, out = run_gate(root)
check("openness claim about an OPEN item exits 0", rc == 0, "rc=%d" % rc)
def test_closed_word_elsewhere_on_the_line_does_not_pardon_a_stale_claim():
"""Regression. The exemption is scoped to the citation's CLAUSE, not the whole line.
`- [Customer RESET shipped](customer-reset-shipped-2026-07-17.md) — …; OPEN R-25b` contains
"shipped" twice — once in a title, once in a filename — and a line-wide exemption pardoned the
stale claim sitting at the end of it. Found by this check's own red-proof failing to go red,
which is the only reason it was found at all."""
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(
tmp,
"# Repo\n\n- [Customer RESET shipped](customer-reset-shipped-2026-07-17.md) — "
"teardown first; OPEN R-168\n",
)
rc, out = run_gate(root)
check("a distant 'shipped' does not pardon the claim", rc != 0, "rc=%d" % rc)
check("the stale item is still named", "R-168" in out)
def test_citation_of_an_item_in_no_register_fails():
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(tmp, "# Repo\n\nSee R-4242 for the rationale.\n")
rc, out = run_gate(root)
check("citation of a nonexistent item exits non-zero", rc != 0, "rc=%d" % rc)
check("message says it is in neither register", "NEITHER" in out)
def test_citation_in_an_html_comment_is_not_policed():
"""Comments are stripped before injection, so a historical citation parked there costs nothing."""
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(tmp, "# Repo\n\n<!-- history: CI was still owed (R-168) at the time -->\n")
rc, out = run_gate(root)
check("citation inside a comment is not policed", rc == 0, "rc=%d" % rc)
def test_citation_check_covers_rule_files_too():
with tempfile.TemporaryDirectory() as tmp:
make_register(tmp, REG_ROWS)
root = make_repo(
tmp, "# Repo\n",
rules={"g.md": '---\npaths: ["**/*.go"]\n---\n\nCI is still owed (R-168).\n'},
)
rc, out = run_gate(root)
check("a rule file's stale citation also fails", rc != 0, "rc=%d" % rc)
check("message names the rule file", "g.md" in out)
# --- check 6: content WARNings on the memory index ------------------------------------------------
def test_memory_expired_statement_warns_but_does_not_fail():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md) — demo boxes REMOTE till ~08-02\n", topics={"a.md": "# A\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("expired statement in the index does NOT fail", rc == 0, "rc=%d" % rc)
check("expired statement WARNs", "EXPIRED statement" in out)
def test_memory_date_inside_a_link_target_is_not_an_expired_statement():
"""THE load-bearing negative. On 2026-08-06 a scan matched the ISO date in the link TARGET and
reported three expired statements, every one of them false, while missing the one real expired
claim in the same file — whose deadline was written `~08-02` with no ISO date at all."""
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [Customer RESET](customer-reset-shipped-2026-07-17.md) — teardown first\n",
topics={"customer-reset-shipped-2026-07-17.md": "# X\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("a dated FILENAME is not an expired statement", rc == 0 and "EXPIRED" not in out, "rc=%d" % rc)
def test_memory_future_deadline_does_not_warn():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md) — rig stays up until 2026-12-31\n", topics={"a.md": "# A\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("a future deadline does not warn", "EXPIRED" not in out)
def test_memory_version_literal_and_address_warn_but_do_not_fail():
with tempfile.TemporaryDirectory() as tmp:
make_memory(tmp, "# Index\n- [a](a.md) — ctrl 0.162.0 on 192.168.0.180\n", topics={"a.md": "# A\n"})
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("version literal in the index does NOT fail", rc == 0, "rc=%d" % rc)
check("version literal WARNs", "version literal" in out)
check("host address WARNs", "host address" in out)
check("loopback is not reported", "127.0.0.1" not in out)
# --- check 5: the workspace file has TWO legal shapes --------------------------------------------
def make_workspace_pair(tmp, live_content=None, link=False, link_target=None):
"""Build <tmp>/CLAUDE.md + the versioned copy, as either a copy or a symlink."""
cp = os.path.join(tmp, "felhom.eu", "documentation", "runbooks")
os.makedirs(cp, exist_ok=True)
versioned = os.path.join(cp, "workspace-CLAUDE.md")
with open(versioned, "w", encoding="utf-8") as fh:
fh.write("# Workspace\n")
live = os.path.join(tmp, "CLAUDE.md")
if link:
os.symlink(link_target or os.path.relpath(versioned, tmp), live)
else:
with open(live, "w", encoding="utf-8") as fh:
fh.write(live_content if live_content is not None else "# Workspace\n")
return live, versioned
def test_workspace_symlink_shape_passes():
with tempfile.TemporaryDirectory() as tmp:
make_workspace_pair(tmp, link=True)
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("symlink shape exits 0", rc == 0, "rc=%d" % rc)
check("tally reports the symlink", "SYMLINK" in out)
def test_dangling_workspace_symlink_fails():
"""A dangling link is worse than a diverged copy: the instructions load NOTHING and there is no
content left to notice is wrong."""
with tempfile.TemporaryDirectory() as tmp:
_, versioned = make_workspace_pair(tmp, link=True)
os.unlink(versioned)
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("dangling workspace symlink exits non-zero", rc != 0, "rc=%d" % rc)
check("message says the instructions load nothing", "load NOTHING" in out)
def test_workspace_symlink_to_the_wrong_file_fails():
with tempfile.TemporaryDirectory() as tmp:
other = os.path.join(tmp, "somewhere-else.md")
with open(other, "w", encoding="utf-8") as fh:
fh.write("# Not the versioned copy\n")
make_workspace_pair(tmp, link=True, link_target="somewhere-else.md")
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("symlink to the wrong target exits non-zero", rc != 0, "rc=%d" % rc)
check("message says it reintroduces divergence", "reintroduces the divergence" in out)
def test_two_file_shape_still_checked_byte_identical():
"""A clone elsewhere may legitimately have two files; the link shape is not forced on anyone."""
with tempfile.TemporaryDirectory() as tmp:
make_workspace_pair(tmp, live_content="# Workspace DIVERGED\n")
root = make_repo(tmp, "# Repo\n")
rc, out = run_gate(root)
check("two-file shape still catches divergence", rc != 0, "rc=%d" % rc)
check("tally names the two-file shape", "two-file shape" in out)
def main():
print("test_instructions_gate")
for fn in sorted(
(v for k, v in globals().items() if k.startswith("test_")),
key=lambda f: f.__name__,
):
fn()
print("")
print("passed: %d failed: %d" % (len(PASSED), len(FAILED)))
return 1 if FAILED else 0
if __name__ == "__main__":
sys.exit(main())