169 lines
7.8 KiB
Python
169 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""test_security_scan_gate.py — the decoys of security_scan_gate.py (R-986). Run from felhom.eu's decoy suite and by
|
|
the script-tests gate; each other repo's decoy suite runs the shared gate on its own scratch clone.
|
|
|
|
Every case builds a SCRATCH git repo in a temp dir (never this tree) and judges the gate's OUTPUT — warn-first, so the
|
|
exit code is 0 in every case and the listing is the fact:
|
|
|
|
1. FACT: a commit that adds a GitHub-token-shaped string → WARN with the rule and the file; the value is NOT printed.
|
|
2. GENUINE: a clean commit → 0 hits, no WARN.
|
|
3. ACCEPTED: the same hit listed in .security-accepted (rule@line-hash + file + reason) → ACCEPTED, 0 new.
|
|
4. FACT: a SECOND secret on another line of an accepted file → still WARN (acceptance is per line, not per file).
|
|
5. FACT: an accepted line with no reason → WARN naming the line.
|
|
6. govulncheck (its JSON from a file, the seam): one CALLED finding is listed by id, an imported-only one is not; the
|
|
same id accepted → ACCEPTED.
|
|
7. FACT: no gitleaks anywhere (an empty tools dir, an empty PATH) → NOT CHECKED, out loud, exit 0 — never a silent
|
|
pass.
|
|
|
|
The fake token is built at run time, so no secret-shaped literal sits in this file. Without gitleaks on this machine
|
|
cases 1-5 say NOT CHECKED HERE and pass out loud (the workspace and CI with FELHOM_SCAN_FETCH=1 run them).
|
|
"""
|
|
import json
|
|
import os
|
|
import random
|
|
import shutil
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
GATE = os.path.join(HERE, "security_scan_gate.py")
|
|
fails, ran = [], 0
|
|
|
|
|
|
def fake_token(seed=1986):
|
|
rnd = random.Random(seed)
|
|
return "ghp" + "_" + "".join(rnd.choice(string.ascii_letters + string.digits) for _ in range(36))
|
|
|
|
|
|
def run(cwd, argv, env=None):
|
|
p = subprocess.run(argv, cwd=cwd, capture_output=True, text=True, env=env)
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
|
|
def scratch_repo(base):
|
|
r = os.path.join(base, "repo")
|
|
os.makedirs(r)
|
|
for a in (["git", "init", "-q"], ["git", "config", "user.email", "decoy@felhom.invalid"],
|
|
["git", "config", "user.name", "decoy"], ["git", "config", "commit.gpgsign", "false"]):
|
|
run(r, a)
|
|
open(os.path.join(r, "README.md"), "w").write("scratch\n")
|
|
run(r, ["git", "add", "README.md"])
|
|
run(r, ["git", "commit", "-q", "-m", "base"])
|
|
return r
|
|
|
|
|
|
def commit(repo, path, text, msg):
|
|
full = os.path.join(repo, path)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
open(full, "w").write(text)
|
|
run(repo, ["git", "add", path])
|
|
run(repo, ["git", "commit", "-q", "-m", msg])
|
|
|
|
|
|
def gate(repo, env_extra=None):
|
|
env = dict(os.environ)
|
|
env["FELHOM_SCAN_RANGE"] = "-n 1 HEAD"
|
|
env.pop("GITHUB_ACTIONS", None)
|
|
env.pop("GITEA_ACTIONS", None)
|
|
env.update(env_extra or {})
|
|
return run(repo, [sys.executable, GATE, repo], env)
|
|
|
|
|
|
def check(name, cond, out):
|
|
global ran
|
|
ran += 1
|
|
if cond:
|
|
print(" ok %s" % name)
|
|
else:
|
|
fails.append("%s\n%s" % (name, out[-900:]))
|
|
|
|
|
|
def main():
|
|
base = tempfile.mkdtemp(prefix="secscan-decoy-")
|
|
try:
|
|
tok = fake_token()
|
|
repo = scratch_repo(base)
|
|
rc, out = gate(repo)
|
|
gitleaks_here = "gitleaks: NOT CHECKED" not in out
|
|
check("2 GENUINE: a clean commit — 0 hits, no WARN, exit 0",
|
|
rc == 0 and ("0 hit(s)" in out or not gitleaks_here)
|
|
and not [l for l in out.splitlines() if l.strip().startswith("WARN")], out)
|
|
|
|
if gitleaks_here:
|
|
commit(repo, "pkg/config.go", 'package pkg\n\nconst Token = "%s"\n' % tok, "plant")
|
|
rc, out = gate(repo)
|
|
warn = [l for l in out.splitlines() if l.strip().startswith("WARN github-pat@")]
|
|
check("1 FACT: a planted token is listed (rule + file), the value is never printed, exit 0",
|
|
rc == 0 and warn and "pkg/config.go:3" in warn[0] and tok not in out and "1 new hit(s)" in out, out)
|
|
key = warn[0].split()[1] if warn else "github-pat@none"
|
|
|
|
open(os.path.join(repo, ".security-accepted"), "w").write(
|
|
"gitleaks %s pkg/config.go # a decoy constant\n" % key)
|
|
rc, out = gate(repo)
|
|
check("3 ACCEPTED: the same line accepted by rule@hash + file", rc == 0 and "ACCEPTED " + key in out
|
|
and "0 new hit(s), 1 accepted" in out, out)
|
|
|
|
commit(repo, "pkg/config.go", 'package pkg\n\nconst Token = "%s"\nconst Other = "%s"\n'
|
|
% (tok, fake_token(1167)), "second")
|
|
rc, out = gate(repo)
|
|
check("4 FACT: a second secret on another line of an accepted file is still WARN",
|
|
rc == 0 and "pkg/config.go:4" in out and "WARN github-pat@" in out and tok not in out, out)
|
|
|
|
open(os.path.join(repo, ".security-accepted"), "w").write("gitleaks %s pkg/config.go\n" % key)
|
|
rc, out = gate(repo)
|
|
check("5 FACT: an accepted line with no reason is a WARN", rc == 0 and "has no reason" in out, out)
|
|
else:
|
|
print(" NOT CHECKED HERE cases 1, 3, 4, 5 — no gitleaks on this machine (the workspace and CI run them)")
|
|
|
|
# 6 — govulncheck through the JSON seam (no network, no Go needed).
|
|
mod = os.path.join(repo, "svc")
|
|
os.makedirs(mod, exist_ok=True)
|
|
open(os.path.join(mod, "go.mod"), "w").write("module example.invalid/svc\n\ngo 1.22\n")
|
|
run(repo, ["git", "add", "svc/go.mod"])
|
|
run(repo, ["git", "commit", "-q", "-m", "module"])
|
|
stream = [
|
|
{"config": {"scanner_name": "govulncheck", "scanner_version": "v1.8.0"}},
|
|
{"osv": {"id": "GO-2099-0001", "summary": "a decoy called vulnerability"}},
|
|
{"osv": {"id": "GO-2099-0002", "summary": "a decoy imported-only vulnerability"}},
|
|
{"finding": {"osv": "GO-2099-0001", "trace": [{"module": "example.invalid/dep", "function": "Parse"}]}},
|
|
{"finding": {"osv": "GO-2099-0002", "trace": [{"module": "example.invalid/dep2"}]}},
|
|
]
|
|
jf = os.path.join(base, "gv.json")
|
|
open(jf, "w").write("\n".join(json.dumps(o, indent=2) for o in stream))
|
|
open(os.path.join(repo, ".security-accepted"), "w").write("")
|
|
rc, out = gate(repo, {"FELHOM_SCAN_GOVULNCHECK_JSON": jf})
|
|
check("6a FACT: a CALLED vulnerability is listed by id; an imported-only one is counted, not listed",
|
|
rc == 0 and "WARN GO-2099-0001 called" in out and "GO-2099-0002" not in out
|
|
and "1 called vulnerability (1 imported" in out, out)
|
|
open(os.path.join(repo, ".security-accepted"), "w").write("govulncheck GO-2099-0001 svc # a decoy, accepted\n")
|
|
rc, out = gate(repo, {"FELHOM_SCAN_GOVULNCHECK_JSON": jf})
|
|
check("6b ACCEPTED: the same id accepted for that module", rc == 0 and "ACCEPTED GO-2099-0001" in out
|
|
and "WARN GO-2099" not in out, out)
|
|
|
|
# 7 — no gitleaks anywhere: NOT CHECKED out loud, exit 0.
|
|
empty = os.path.join(base, "empty")
|
|
os.makedirs(empty)
|
|
git_dir = os.path.dirname(shutil.which("git") or "/usr/bin/git")
|
|
only_git = os.path.join(base, "onlygit")
|
|
os.makedirs(only_git)
|
|
os.symlink(os.path.join(git_dir, "git"), os.path.join(only_git, "git"))
|
|
rc, out = gate(repo, {"FELHOM_SECURITY_TOOLS_DIR": empty, "FELHOM_SECURITY_BIN": "", "PATH": only_git,
|
|
"FELHOM_SCAN_FETCH": "0"})
|
|
check("7 FACT: no gitleaks and no govulncheck anywhere — NOT CHECKED, out loud, exit 0",
|
|
rc == 0 and "gitleaks: NOT CHECKED" in out and "govulncheck: NOT CHECKED" in out, out)
|
|
finally:
|
|
shutil.rmtree(base, ignore_errors=True)
|
|
if fails:
|
|
for f in fails:
|
|
print("FAIL: " + f)
|
|
return 1
|
|
print("test_security_scan_gate: all %d cases passed" % ran)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|