admin
f27aed87cd
gates / gates (push) Successful in 15s
MEMORY.md is the larger half of what loads before a word is typed (8.4k tokens vs the root CLAUDE.md's 6.6k) and is the one instruction file nobody hand-edits, so nothing was watching it. Three deliberately different outcomes, each pinned by a test: over-ceiling FAILS (auto-memory drops content past the limit with no error), an orphan WARNS (the store is outside git), and an absent store PASSES while PRINTING its reason -- asserted on the reason text, because a pass with no reason is indistinguishable from a gate that stopped running. 39 assertions (was 20). Red-proof run against the real store, not a fixture.
312 lines
12 KiB
Python
312 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Fixture tests for instructions_gate.py.
|
|
|
|
Run: python3 scripts/test_instructions_gate.py
|
|
|
|
Every test asserts the EFFECT — the gate's exit code AND that its message names the file and the
|
|
reason — not merely that "it ran". A gate that exits non-zero for the wrong reason is not a gate.
|
|
|
|
The negatives matter as much as the positives here: the whole point of the HTML-comment convention
|
|
is that commented content costs nothing, so `test_comments_do_not_count_toward_ceiling` and
|
|
`test_version_literal_in_comment_is_allowed` are what stop the gate from punishing the very move it
|
|
is meant to encourage.
|
|
"""
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
GATE = os.path.join(HERE, "instructions_gate.py")
|
|
|
|
PASSED = []
|
|
FAILED = []
|
|
|
|
|
|
def run_gate(root, today="2026-08-06"):
|
|
env = dict(os.environ, FELHOM_GATE_TODAY=today)
|
|
p = subprocess.run(
|
|
[sys.executable, GATE, root],
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
|
|
def make_repo(tmp, claude_md, rules=None):
|
|
root = os.path.join(tmp, "repo")
|
|
os.makedirs(root, exist_ok=True)
|
|
with open(os.path.join(root, "CLAUDE.md"), "w", encoding="utf-8") as fh:
|
|
fh.write(claude_md)
|
|
if rules:
|
|
rd = os.path.join(root, ".claude", "rules")
|
|
os.makedirs(rd, exist_ok=True)
|
|
for name, body in rules.items():
|
|
with open(os.path.join(rd, name), "w", encoding="utf-8") as fh:
|
|
fh.write(body)
|
|
return root
|
|
|
|
|
|
def check(name, cond, detail=""):
|
|
(PASSED if cond else FAILED).append(name + ((" — " + detail) if detail else ""))
|
|
print((" PASS " if cond else " FAIL ") + name + ((" " + detail) if detail else ""))
|
|
|
|
|
|
def test_short_file_passes():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n" + "a line\n" * 150)
|
|
rc, out = run_gate(root)
|
|
check("150-line CLAUDE.md exits 0", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_long_file_fails_and_names_file_and_count():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n" + "a line\n" * 204)
|
|
rc, out = run_gate(root)
|
|
check("205-line CLAUDE.md exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message names the file", "CLAUDE.md" in out)
|
|
check("message names the effective line count", "205 effective lines" in out)
|
|
check(
|
|
"message says ADHERENCE, not space",
|
|
"ADHERENCE limit, not a space limit" in out,
|
|
)
|
|
|
|
|
|
def test_comments_do_not_count_toward_ceiling():
|
|
"""The convention's load-bearing negative: 400 commented lines must not trip the ceiling."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
body = "# Repo\n" + "a line\n" * 100 + "<!--\n" + "history\n" * 400 + "-->\n"
|
|
root = make_repo(tmp, body)
|
|
rc, out = run_gate(root)
|
|
check("400 commented lines do not trip the ceiling", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_rule_with_paths_passes():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(
|
|
tmp,
|
|
"# Repo\n",
|
|
rules={"scoped.md": '---\npaths: ["**/*.go"]\n---\n\n# Scoped\n'},
|
|
)
|
|
rc, out = run_gate(root)
|
|
check("rule with paths: exits 0", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_rule_without_paths_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(
|
|
tmp, "# Repo\n", rules={"bare.md": "---\nname: bare\n---\n\n# Bare\n"}
|
|
)
|
|
rc, out = run_gate(root)
|
|
check("rule with neither marker exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message names the rule file", "bare.md" in out)
|
|
|
|
|
|
def test_rule_with_unconditional_marker_passes():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(
|
|
tmp,
|
|
"# Repo\n",
|
|
rules={"always.md": "---\nunconditional: true\n---\n\n# Always\n"},
|
|
)
|
|
rc, out = run_gate(root)
|
|
check("explicit unconditional: true exits 0", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_version_literal_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n\nThe box runs agent 0.93.0 today.\n")
|
|
rc, out = run_gate(root)
|
|
check("version literal exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message quotes the version", "0.93.0" in out)
|
|
|
|
|
|
def test_ipv4_is_not_a_version():
|
|
"""A bare \\d+\\.\\d+\\.\\d+ matches the first three octets of every IPv4."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n\nDooPlex is 192.168.0.180 and the demo box is 10.0.0.1.\n")
|
|
rc, out = run_gate(root)
|
|
check("IPv4 addresses are not flagged as versions", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_version_literal_in_comment_is_allowed():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n\n<!-- fixed in hub v0.97.0 -->\n")
|
|
rc, out = run_gate(root)
|
|
check("version literal inside a comment is allowed", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_expired_temporary_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(
|
|
tmp,
|
|
"# Repo\n\n> **TEMPORARY — host is away (until ~2026-08-02).**\n> Delete on return.\n",
|
|
)
|
|
rc, out = run_gate(root)
|
|
check("expired TEMPORARY block exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message quotes the past date", "2026-08-02" in out)
|
|
|
|
|
|
def test_future_temporary_passes():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(
|
|
tmp, "# Repo\n\n> **TEMPORARY — host is away (until ~2026-12-31).**\n"
|
|
)
|
|
rc, out = run_gate(root)
|
|
check("TEMPORARY with a future date exits 0", rc == 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_missing_claude_md_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = os.path.join(tmp, "repo")
|
|
os.makedirs(root)
|
|
rc, out = run_gate(root)
|
|
check("absent CLAUDE.md is a FAILURE, not a skip", rc != 0, "rc=%d" % rc)
|
|
|
|
|
|
def test_diverged_workspace_copy_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
ws = os.path.join(tmp, "ws")
|
|
root = os.path.join(ws, "repo")
|
|
os.makedirs(root)
|
|
with open(os.path.join(root, "CLAUDE.md"), "w", encoding="utf-8") as fh:
|
|
fh.write("# Repo\n")
|
|
with open(os.path.join(ws, "CLAUDE.md"), "w", encoding="utf-8") as fh:
|
|
fh.write("# Workspace live\n")
|
|
cp = os.path.join(ws, "felhom.eu", "documentation", "runbooks")
|
|
os.makedirs(cp)
|
|
with open(os.path.join(cp, "workspace-CLAUDE.md"), "w", encoding="utf-8") as fh:
|
|
fh.write("# Workspace copy — DIVERGED\n")
|
|
rc, out = run_gate(root)
|
|
check("diverged workspace copy exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message names both files", "workspace-CLAUDE.md" in out)
|
|
|
|
|
|
# --- memory-index checks (gate check 6) ---------------------------------------------------------
|
|
#
|
|
# The three outcomes are deliberately different and each has a test proving it is the one intended:
|
|
# over-limit FAILS (silent truncation), an orphan WARNS (the store is outside git), and an absent
|
|
# store PASSES *while printing its reason* (machine-local by design). The last one is the dangerous
|
|
# grant — an exception to "a missing input is a FAILURE" — so it is asserted on the reason text, not
|
|
# just on rc == 0. A pass with no reason is indistinguishable from a gate that stopped running.
|
|
|
|
|
|
def make_memory(tmp, index, topics=None, archived=None):
|
|
"""Create <tmp>/.claude-memory. The gate resolves the store from the WORKSPACE root, which for
|
|
make_repo(tmp, ...) is tmp itself."""
|
|
store = os.path.join(tmp, ".claude-memory")
|
|
os.makedirs(store, exist_ok=True)
|
|
if index is not None:
|
|
with open(os.path.join(store, "MEMORY.md"), "w", encoding="utf-8") as fh:
|
|
fh.write(index)
|
|
for name, body in (topics or {}).items():
|
|
with open(os.path.join(store, name), "w", encoding="utf-8") as fh:
|
|
fh.write(body)
|
|
if archived:
|
|
arc = os.path.join(store, "archive")
|
|
os.makedirs(arc, exist_ok=True)
|
|
for name, body in archived.items():
|
|
with open(os.path.join(arc, name), "w", encoding="utf-8") as fh:
|
|
fh.write(body)
|
|
return store
|
|
|
|
|
|
def test_memory_absent_passes_and_prints_the_reason():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = make_repo(tmp, "# Repo\n") # no .claude-memory at all
|
|
rc, out = run_gate(root)
|
|
check("absent memory store exits 0", rc == 0, "rc=%d" % rc)
|
|
check("absent memory store PRINTS its reason", "machine-local by design" in out)
|
|
check("absent memory store is not reported as a skip", "PASS with reason" in out)
|
|
|
|
|
|
def test_memory_within_limits_passes():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n- [a](a.md)\n", topics={"a.md": "# A\n"})
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("memory index within limits exits 0", rc == 0, "rc=%d" % rc)
|
|
check("tally reports indexed/orphaned/archived", "1 indexed, 0 orphaned" in out)
|
|
|
|
|
|
def test_memory_over_line_limit_fails_and_names_file_and_count():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n" + "- entry\n" * 250)
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("over-length memory index exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message names MEMORY.md", "MEMORY.md" in out)
|
|
check("message names the line count", "251 lines" in out)
|
|
check("message says silent truncation, not space", "DROPPED WITH NO" in out)
|
|
|
|
|
|
def test_memory_over_byte_limit_fails():
|
|
"""Few lines, many bytes — the byte ceiling must bite independently of the line ceiling."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n" + ("x" * 30000) + "\n")
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("over-byte memory index exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("byte failure names the byte ceiling", "25600" in out)
|
|
|
|
|
|
def test_memory_orphan_warns_but_does_not_fail():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n- [a](a.md)\n",
|
|
topics={"a.md": "# A\n", "lonely.md": "# Lonely\n"})
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("an orphan does NOT fail the gate", rc == 0, "rc=%d" % rc)
|
|
check("an orphan produces a WARNING", "WARNING" in out)
|
|
check("the warning names the orphan", "lonely.md" in out)
|
|
|
|
|
|
def test_memory_wikilink_counts_as_a_reference():
|
|
"""Load-bearing negative: a file reachable only via [[wikilink]] is indexed, not orphaned.
|
|
Calling it an orphan would push someone to add a duplicate row for it."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n- [a](a.md) — see [[deep]]\n",
|
|
topics={"a.md": "# A\n", "deep.md": "# Deep\n"})
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("wikilink-only file is not an orphan", rc == 0 and "WARNING" not in out, "rc=%d" % rc)
|
|
|
|
|
|
def test_memory_archived_file_is_not_an_orphan():
|
|
"""archive/ is where deliberately set-aside records live; they are not top-level and must not
|
|
be counted as unreferenced."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, "# Index\n- [a](a.md)\n", topics={"a.md": "# A\n"},
|
|
archived={"old-campaign.md": "# Old\n"})
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("archived file is not an orphan", rc == 0 and "WARNING" not in out, "rc=%d" % rc)
|
|
check("tally counts the archived file", "1 archived" in out)
|
|
|
|
|
|
def test_memory_store_without_index_fails():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
make_memory(tmp, None, topics={"a.md": "# A\n"}) # store exists, no MEMORY.md
|
|
root = make_repo(tmp, "# Repo\n")
|
|
rc, out = run_gate(root)
|
|
check("store with no index exits non-zero", rc != 0, "rc=%d" % rc)
|
|
check("message says unreachable knowledge", "unreachable knowledge" in out)
|
|
|
|
|
|
def main():
|
|
print("test_instructions_gate")
|
|
for fn in sorted(
|
|
(v for k, v in globals().items() if k.startswith("test_")),
|
|
key=lambda f: f.__name__,
|
|
):
|
|
fn()
|
|
print("")
|
|
print("passed: %d failed: %d" % (len(PASSED), len(FAILED)))
|
|
return 1 if FAILED else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|