felhom.eu/scripts/test_repo_gates_scope.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""test_repo_gates_scope.py — R-404's acceptance test: the exemption is ONE GATE WIDE.

R3 IS THE ONE THAT MATTERS AND IT WAS WRITTEN FIRST. Everything else here is scaffolding around
it. The change this file guards makes `repo_gates.py` let a documents-only push through when
`golden-currency` convicts. The danger is not that the exemption exists — it is that it quietly
becomes general, at which point a documents push stops being checked for anything and nobody finds
out until a broken document ships.

WHY STUB GATES AND NOT THE REAL ONES. The runner's DECISION is the thing under test: given a set of
exit codes and a scope, what does it exit and what does it print? Driving that with the real gates
would make the test depend on the repo being in a particular state — the R-410 lesson, where a gate
that read a directory NAME passed with no bake behind it. Stubs make the input exact.

Run:  python3 scripts/test_repo_gates_scope.py
Exit 0 all pass · 1 a case failed.
"""
import io
import os
import shutil
import sys
import tempfile

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "scripts"))

import repo_gates  # noqa: E402


class Capture(object):
    """Collect stdout while the runner writes to it."""

    def __enter__(self):
        self._real = sys.stdout
        self.buf = io.StringIO()
        sys.stdout = self.buf
        return self

    def __exit__(self, *a):
        sys.stdout = self._real
        self.text = self.buf.getvalue()
        return False


def stub(tmp, name, rc):
    """A gate that does nothing but exit with `rc`, and says so, so output is readable."""
    p = os.path.join(tmp, name)
    with io.open(p, "w", encoding="utf-8") as fh:
        fh.write(u"import sys\nprint('stub %s exiting %d')\nsys.exit(%d)\n" % (name, rc, rc))
    return p


def run(gates, argv):
    """Run the runner's main() with GATES swapped for `gates`. Returns (exit code, output)."""
    saved = repo_gates.GATES
    repo_gates.GATES = gates
    try:
        with Capture() as cap:
            rc = repo_gates.main(argv)
        return rc, cap.text
    finally:
        repo_gates.GATES = saved


def main():
    tmp = tempfile.mkdtemp(prefix="gatescope-")
    fails = []
    try:
        ok = stub(tmp, "ok.py", 0)
        bad = stub(tmp, "bad.py", 1)
        inc = stub(tmp, "inc.py", 2)

        # The real table's shape, with the fifth field: exemptible on a documents-only push.
        def table(golden_rc, other_rc=0):
            return [
                ("site", {0: ok, 1: bad, 2: inc}[other_rc], [], True, False),
                ("golden-currency", {0: ok, 1: bad, 2: inc}[golden_rc], [], True, True),
            ]

        # --- R1: docs scope, golden-currency convicting -> ADVISORY, exit 0 -------------------
        rc, out = run(table(1), ["--fast", "--scope=docs"])
        if rc != 0:
            fails.append("R1: a documents-only push with only golden-currency convicting must be "
                         "ALLOWED; got exit %d" % rc)
        elif "ADVISORY" not in out:
            fails.append("R1: the word ADVISORY never appeared. A silent pass is a SILENCING, "
                         "not a re-aim — the whole point is that it still says so:\n%s" % out)
        else:
            print("R1 ok: docs + golden-currency convicting -> exit 0, ADVISORY printed")

        # --- R2: code scope, same conviction -> refused ---------------------------------------
        rc, out = run(table(1), ["--fast", "--scope=code"])
        if rc == 0:
            fails.append("R2: a CODE push with golden-currency convicting must still be REFUSED")
        elif "ADVISORY" in out:
            fails.append("R2: a code push must not print ADVISORY — it is refused, not advised")
        else:
            print("R2 ok: code + golden-currency convicting -> exit %d" % rc)

        # --- R3: SCENARIO C -- the exemption is ONE GATE WIDE ---------------------------------
        # A documents push with a NON-exemptible gate convicting must still be refused.
        # RED-PROOF: mark every gate exemptible (change the 5th field of 'site' to True) and this
        # fails, which is exactly what a general exemption would look like in production.
        rc, out = run(table(0, other_rc=1), ["--fast", "--scope=docs"])
        if rc == 0:
            fails.append("R3 (SCENARIO C): a documents-only push with the SITE gate convicting was "
                         "ALLOWED. The exemption has become general — every other gate must still "
                         "block every push:\n%s" % out)
        else:
            print("R3 ok: docs + a NON-exemptible gate convicting -> exit %d (refused)" % rc)

        # --- R4: inconclusive is not a conviction, in either scope -----------------------------
        rc_docs, out_docs = run(table(2), ["--fast", "--scope=docs"])
        rc_code, _ = run(table(2), ["--fast", "--scope=code"])
        if rc_docs != rc_code:
            fails.append("R4: an INCONCLUSIVE golden-currency must behave identically in both "
                         "scopes (docs=%d code=%d) — the exemption is for CONVICTIONS, and an "
                         "undetermined result was never a conviction" % (rc_docs, rc_code))
        elif rc_docs == 0:
            fails.append("R4: INCONCLUSIVE must not pass in either scope; got exit 0")
        elif "ADVISORY" in out_docs:
            fails.append("R4: an inconclusive gate must not be rendered as ADVISORY")
        else:
            print("R4 ok: INCONCLUSIVE unchanged in both scopes (exit %d)" % rc_docs)

        # --- R5: no --scope behaves exactly like today ----------------------------------------
        rc_none, out_none = run(table(1), ["--fast"])
        rc_code2, out_code2 = run(table(1), ["--fast", "--scope=code"])
        if rc_none != rc_code2:
            fails.append("R5: an unscoped run must be IDENTICAL to --scope=code (a human typing "
                         "the command must see today's behaviour); got %d vs %d" % (rc_none, rc_code2))
        else:
            # compare the summary lines, ignoring the header that names the scope
            a = [l for l in out_none.splitlines() if l.startswith("  ") and "exit" in l]
            b = [l for l in out_code2.splitlines() if l.startswith("  ") and "exit" in l]
            if a != b:
                fails.append("R5: unscoped and --scope=code printed different verdicts:\n%s\n%s"
                             % (a, b))
            else:
                print("R5 ok: unscoped == --scope=code, exit %d" % rc_none)

        # --- R6: a bad scope value is refused, never silently treated as docs ------------------
        rc, out = run(table(1), ["--fast", "--scope=banana"])
        if rc == 0:
            fails.append("R6: an unrecognised --scope value must NOT pass. Fail closed.")
        else:
            print("R6 ok: --scope=banana refused (exit %d)" % rc)
    finally:
        shutil.rmtree(tmp, ignore_errors=True)

    if fails:
        print()
        for f in fails:
            print("FAIL: %s" % f)
        return 1
    print("\nrepo_gates scope tests OK — the exemption is one gate wide (R3 is the acceptance)")
    return 0


sys.exit(main())