felhom.eu/scripts/test_guide_quote_gate.py

#!/usr/bin/env python3
"""Decoys for guide_quote_gate (R-421). Run: python3 scripts/test_guide_quote_gate.py

A decoy is the LABEL without the FACT. The shape this gate is most at risk of is
**name-for-fact**: matching the KEY NAME where the fact is the key's VALUE. A guide that lists
`claim.msg.bad_code` in a table of "messages covered" would satisfy a name-matching gate and tell a
tester nothing, because a tester reads sentences, not keys.

The second shape checked here is **declaration-for-reachability** in its sibling form: a gate that
reports OK when it could not read the bundle at all. Scope is a fact -- an absent controller clone
must be INCONCLUSIVE (2), never a pass.

Each case runs the real gate against a built tree and asserts the exit code.
"""
from __future__ import annotations

import io
import json
import os
import shutil
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
WORKSPACE = os.path.dirname(REPO)
REAL_BUNDLE = os.path.join(
    WORKSPACE, "felhom-controller", "controller", "internal", "i18n", "locales", "en.json"
)

GATE_SRC = os.path.join(HERE, "guide_quote_gate.py")


def build(tmp, guide_text, bundle=None, with_bundle=True):
    """Lay out a fake workspace: <tmp>/felhom.eu/{scripts,documentation/...} + the sibling clone."""
    repo = os.path.join(tmp, "felhom.eu")
    scripts = os.path.join(repo, "scripts")
    runbooks = os.path.join(repo, "documentation", "runbooks")
    os.makedirs(scripts)
    os.makedirs(runbooks)
    shutil.copy(GATE_SRC, os.path.join(scripts, "guide_quote_gate.py"))
    io.open(os.path.join(runbooks, "VOLUNTEER-first-hour.en.md"), "w", encoding="utf-8").write(guide_text)
    if with_bundle:
        locales = os.path.join(tmp, "felhom-controller", "controller", "internal", "i18n", "locales")
        os.makedirs(locales)
        if bundle is None:
            bundle = json.load(io.open(REAL_BUNDLE, encoding="utf-8"))
        io.open(os.path.join(locales, "en.json"), "w", encoding="utf-8").write(
            json.dumps(bundle, ensure_ascii=False, indent=2)
        )
    return os.path.join(scripts, "guide_quote_gate.py")


def run(gate):
    p = subprocess.run([sys.executable, gate], capture_output=True, text=True)
    return p.returncode, p.stdout + p.stderr


def real_bundle():
    return json.load(io.open(REAL_BUNDLE, encoding="utf-8"))


def honest_guide(b):
    return (
        "# guide\n\n"
        "A wrong code answers **\"%s\"**, and after five tries **\"%s\"**.\n"
        "A short password answers **\"%s\"**.\n\n> %s\n"
        % (
            b["claim.msg.bad_code"],
            b["claim.msg.too_many"],
            b["claim.msg.password_too_short"] % 12,
            b["backup.target.degraded"],
        )
    )


CASES = []


def case(name):
    def deco(fn):
        CASES.append((name, fn))
        return fn
    return deco


@case("control: an honest guide passes")
def _control(tmp):
    gate = build(tmp, honest_guide(real_bundle()))
    rc, out = run(gate)
    return rc == 0, "rc=%d\n%s" % (rc, out)


@case("DECOY name-for-fact: the guide NAMES every key and quotes none")
def _names(tmp):
    text = (
        "# guide\n\nMessages covered by this guide:\n\n"
        "| key | section |\n|---|---|\n"
        "| claim.msg.bad_code | 13 |\n"
        "| claim.msg.too_many | 13 |\n"
        "| claim.msg.password_too_short | 13 |\n"
        "| backup.target.degraded | 9 |\n\n"
        "All four messages are shown in English.\n"
    )
    gate = build(tmp, text)
    rc, out = run(gate)
    return rc == 1 and "NOT QUOTED" in out, "rc=%d\n%s" % (rc, out)


@case("DECOY substring: the guide quotes a PREFIX of the real sentence")
def _prefix(tmp):
    b = real_bundle()
    text = honest_guide(b).replace(b["backup.target.degraded"], b["backup.target.degraded"][:40])
    gate = build(tmp, text)
    rc, out = run(gate)
    return rc == 1 and "backup.target.degraded" in out, "rc=%d\n%s" % (rc, out)


@case("DECOY the OLD Hungarian quote: the drill-era guide must convict")
def _hungarian(tmp):
    b = real_bundle()
    text = honest_guide(b).replace(b["claim.msg.bad_code"], "Hibás vagy lejárt kód")
    gate = build(tmp, text)
    rc, out = run(gate)
    return rc == 1 and "claim.msg.bad_code" in out, "rc=%d\n%s" % (rc, out)


@case("DECOY reworded screen: the bundle changes and the guide does not")
def _reworded(tmp):
    b = real_bundle()
    text = honest_guide(b)
    b2 = dict(b)
    b2["claim.msg.too_many"] = "Too many attempts - try again later."
    gate = build(tmp, text, bundle=b2)
    rc, out = run(gate)
    return rc == 1 and "claim.msg.too_many" in out, "rc=%d\n%s" % (rc, out)


@case("SCOPE: no controller clone is INCONCLUSIVE (2), never a pass")
def _noclone(tmp):
    gate = build(tmp, honest_guide(real_bundle()), with_bundle=False)
    rc, out = run(gate)
    return rc == 2 and "INCONCLUSIVE" in out, "rc=%d\n%s" % (rc, out)


@case("SCOPE: a key deleted from the bundle convicts, it does not vanish")
def _deleted(tmp):
    b = real_bundle()
    text = honest_guide(b)
    b2 = dict(b)
    del b2["claim.msg.bad_code"]
    gate = build(tmp, text, bundle=b2)
    rc, out = run(gate)
    return rc == 1 and "MISSING KEY" in out, "rc=%d\n%s" % (rc, out)


def main():
    if not os.path.exists(REAL_BUNDLE):
        print("SKIP: the felhom-controller clone is not beside this one", file=sys.stderr)
        return 2
    failed = 0
    for name, fn in CASES:
        tmp = tempfile.mkdtemp(prefix="gqg-")
        try:
            ok, detail = fn(tmp)
        finally:
            shutil.rmtree(tmp, ignore_errors=True)
        print(("  PASS  " if ok else "  FAIL  ") + name)
        if not ok:
            failed += 1
            print("        " + detail.replace("\n", "\n        "))
    print("\nguide-quote decoys: %d/%d" % (len(CASES) - failed, len(CASES)))
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())