146 lines
7.0 KiB
Python
146 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""test_tenantsync_selfcheck.py — felhom-tenantsync v1.4.0 waits out the PBS proxy's permission cache (R-1295).
|
||
|
||
Runs the REAL script against stubbed `proxmox-backup-manager` / `proxmox-backup-client` / `sleep` on PATH (real bash,
|
||
real jq). The client stub refuses `snapshot list` (the self-check) the first FAILS times — the shape measured on a
|
||
throwaway PBS 4.2.5 under load (a 403 for 4.2–5.0 s after the grant) — and the sleep stub only counts, so the test is
|
||
fast. Every call is logged, so each case asserts what the script DID:
|
||
|
||
1. the check refused for ~4 s (8 tries) → the SAME token passes; one generate-token, no rollback, a log line per try;
|
||
2. refused for ever → bounded: two windows of 30 tries (the old regen-once path after the first), then the rollback
|
||
and `self_check_failed` — never a hang;
|
||
3. passing at once → one check, no sleep;
|
||
4. the transient ADMIN token's first namespace list refused for ~3 s → it waits; never a `namespace create` on a
|
||
refused list (measured: under load v1.3.0 then died on „missing Datastore.Modify").
|
||
|
||
RED-PROOF: the v1.3.0 script (one immediate re-check after a regen) fails case 1 — it rolls back.
|
||
Exit 0 all passed · 1 a case failed. Without bash or jq it says NOT CHECKED out loud and exits 0.
|
||
"""
|
||
import json
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
SCRIPT = os.environ.get("TENANTSYNC_SCRIPT") or os.path.join(HERE, "felhom-tenantsync.sh")
|
||
|
||
PBM = r'''#!/usr/bin/env bash
|
||
echo "pbm $*" >> "$CALLLOG"
|
||
case "$1 $2" in
|
||
"cert info") echo "Fingerprint (sha256): aa:bb:cc:dd" ;;
|
||
"datastore list") echo '[{"name":"felhom-offsite","path":"/nonexistent"}]' ;;
|
||
"user generate-token") echo '{"value": "FAKE-TOKEN-VALUE"}' ;;
|
||
"user list") echo '[{"userid":"felhom@pbs"}]' ;;
|
||
"user list-tokens") echo '[]' ;;
|
||
"prune-job list") echo '[{"id":"prune-x","store":"felhom-offsite","ns":"Tester4"}]' ;;
|
||
*) : ;;
|
||
esac
|
||
'''
|
||
PBC = r'''#!/usr/bin/env bash
|
||
echo "pbc $*" >> "$CALLLOG"
|
||
case "$1 $2" in
|
||
"namespace list") m=$(cat "$COUNTER.adm" 2>/dev/null || echo 0); echo $((m+1)) > "$COUNTER.adm"
|
||
[ "$m" -lt "${ADMIN_FAILS:-0}" ] && { echo "permission check failed" >&2; exit 1; }
|
||
echo '{"data":[{"ns":"Tester4"}]}' ;;
|
||
"snapshot list") n=$(cat "$COUNTER" 2>/dev/null || echo 0); echo $((n+1)) > "$COUNTER"
|
||
[ "$n" -lt "$FAILS" ] && { echo "permission check failed" >&2; exit 1; }
|
||
echo '[]' ;;
|
||
*) : ;;
|
||
esac
|
||
'''
|
||
SLEEP = '#!/usr/bin/env bash\necho "sleep $*" >> "$CALLLOG"\n'
|
||
|
||
fails = []
|
||
ran = 0
|
||
|
||
|
||
def check(name, cond, detail=""):
|
||
global ran
|
||
ran += 1
|
||
if cond:
|
||
print("ok %s" % name)
|
||
else:
|
||
fails.append("%s %s" % (name, detail))
|
||
|
||
|
||
def run(op, refuse, admin_refuse=0):
|
||
d = tempfile.mkdtemp(prefix="tenantsync-selfcheck-")
|
||
try:
|
||
b = os.path.join(d, "bin")
|
||
os.makedirs(b)
|
||
for name, body in (("proxmox-backup-manager", PBM), ("proxmox-backup-client", PBC), ("sleep", SLEEP)):
|
||
p = os.path.join(b, name)
|
||
open(p, "w").write(body)
|
||
os.chmod(p, 0o755)
|
||
calls = os.path.join(d, "calls.log")
|
||
open(calls, "w").close()
|
||
e = dict(os.environ, PATH=b + os.pathsep + os.environ["PATH"], CALLLOG=calls,
|
||
COUNTER=os.path.join(d, "n"), FAILS=str(refuse), ADMIN_FAILS=str(admin_refuse))
|
||
p = subprocess.run(["bash", SCRIPT], input=json.dumps({"op": op, "customer_id": "Tester4"}), env=e,
|
||
capture_output=True, text=True, timeout=60)
|
||
return p.returncode, p.stdout, p.stderr, open(calls).read().splitlines()
|
||
finally:
|
||
shutil.rmtree(d, ignore_errors=True)
|
||
|
||
|
||
def gens(calls):
|
||
return [c for c in calls if c.startswith("pbm user generate-token felhom@pbs")]
|
||
|
||
|
||
def checks(calls):
|
||
return [c for c in calls if c.startswith("pbc snapshot list")]
|
||
|
||
|
||
def main():
|
||
if not shutil.which("bash") or not shutil.which("jq"):
|
||
print("test_tenantsync_selfcheck: NOT CHECKED — bash or jq is missing here (the workspace run has both)")
|
||
return 0
|
||
|
||
# 1. the measured shape: refused for ~4 s, then the same token passes.
|
||
for op in ("reissue", "provision"):
|
||
rc, out, err, calls = run(op, 8)
|
||
status = (json.loads(out or "{}").get("status") if out.strip().startswith("{") else None)
|
||
check("%s: a self-check refused for ~4 s passes on the SAME token" % op, rc == 0 and status == "ok",
|
||
"rc=%s out=%s" % (rc, out.strip()[:160]))
|
||
check("%s: one token generated, never a regen" % op, len(gens(calls)) == 1, "generate-token calls: %d" % len(gens(calls)))
|
||
check("%s: nine checks, eight 0.5-s waits" % op, len(checks(calls)) == 9 and calls.count("sleep 0.5") == 8,
|
||
"checks=%d sleeps=%d" % (len(checks(calls)), calls.count("sleep 0.5")))
|
||
check("%s: a log line per refused try" % op, err.count("self-check try") == 8 and "passed on try 9/30" in err,
|
||
err[-300:])
|
||
check("%s: no rollback" % op, not any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-3:]))
|
||
check("%s: the secret never reaches stderr" % op, "FAKE-TOKEN-VALUE" not in err)
|
||
|
||
# 2. refused for ever → bounded, rolled back, typed.
|
||
rc, out, err, calls = run("reissue", 10 ** 6)
|
||
check("refused for ever: self_check_failed, rolled back", rc == 1 and '"code":"self_check_failed"' in out, out.strip()[:200])
|
||
check("refused for ever: bounded — two windows of 30 checks", len(checks(calls)) == 60, "checks=%d" % len(checks(calls)))
|
||
check("refused for ever: the token is deleted at the end", calls[-1].startswith("pbm acl update") or
|
||
any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-4:]), str(calls[-4:]))
|
||
|
||
# 4. the admin token's first namespace list refused for ~3 s → it waits, and never tries to CREATE the namespace.
|
||
rc, out, err, calls = run("reissue", 0, admin_refuse=6)
|
||
check("admin token refused for ~3 s: the op still succeeds", rc == 0 and '"status":"ok"' in out, out.strip()[:160])
|
||
check("admin token refused: never a namespace create on a refused list",
|
||
not any(c.startswith("pbc namespace create") for c in calls), str([c for c in calls if "namespace" in c]))
|
||
check("admin token refused: a log line per try", err.count("admin token try") == 6 and "admin token ready on try 7/30" in err, err[-300:])
|
||
|
||
# 3. passing at once → no wait.
|
||
rc, out, err, calls = run("reissue", 0)
|
||
check("passing at once: one check, no sleep", rc == 0 and len(checks(calls)) == 1 and not any(c.startswith("sleep") for c in calls),
|
||
"checks=%d" % len(checks(calls)))
|
||
|
||
if fails:
|
||
for f in fails:
|
||
print("FAIL %s" % f)
|
||
print("test_tenantsync_selfcheck: %d of %d check(s) FAILED" % (len(fails), ran))
|
||
return 1
|
||
print("test_tenantsync_selfcheck: all %d checks passed" % ran)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|