#!/usr/bin/env python3 # -*- coding: utf-8 -*- """test_tenantsync_selfcheck.py — felhom-tenantsync v1.4.0 waits out the PBS proxy's permission cache (R-1295). Runs the REAL script against stubbed `proxmox-backup-manager` / `proxmox-backup-client` / `sleep` on PATH (real bash, real jq). The client stub refuses `snapshot list` (the self-check) the first FAILS times — the shape measured on a throwaway PBS 4.2.5 under load (a 403 for 4.2–5.0 s after the grant) — and the sleep stub only counts, so the test is fast. Every call is logged, so each case asserts what the script DID: 1. the check refused for ~4 s (8 tries) → the SAME token passes; one generate-token, no rollback, a log line per try; 2. refused for ever → bounded: two windows of 30 tries (the old regen-once path after the first), then the rollback and `self_check_failed` — never a hang; 3. passing at once → one check, no sleep; 4. the transient ADMIN token's first namespace list refused for ~3 s → it waits; never a `namespace create` on a refused list (measured: under load v1.3.0 then died on „missing Datastore.Modify"). RED-PROOF: the v1.3.0 script (one immediate re-check after a regen) fails case 1 — it rolls back. Exit 0 all passed · 1 a case failed. Without bash or jq it says NOT CHECKED out loud and exits 0. """ import json import os import shutil import subprocess import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) SCRIPT = os.environ.get("TENANTSYNC_SCRIPT") or os.path.join(HERE, "felhom-tenantsync.sh") PBM = r'''#!/usr/bin/env bash echo "pbm $*" >> "$CALLLOG" case "$1 $2" in "cert info") echo "Fingerprint (sha256): aa:bb:cc:dd" ;; "datastore list") echo '[{"name":"felhom-offsite","path":"/nonexistent"}]' ;; "user generate-token") echo '{"value": "FAKE-TOKEN-VALUE"}' ;; "user list") echo '[{"userid":"felhom@pbs"}]' ;; "user list-tokens") echo '[]' ;; "prune-job list") echo '[{"id":"prune-x","store":"felhom-offsite","ns":"Tester4"}]' ;; *) : ;; esac ''' PBC = r'''#!/usr/bin/env bash echo "pbc $*" >> "$CALLLOG" case "$1 $2" in "namespace list") m=$(cat "$COUNTER.adm" 2>/dev/null || echo 0); echo $((m+1)) > "$COUNTER.adm" [ "$m" -lt "${ADMIN_FAILS:-0}" ] && { echo "permission check failed" >&2; exit 1; } echo '{"data":[{"ns":"Tester4"}]}' ;; "snapshot list") n=$(cat "$COUNTER" 2>/dev/null || echo 0); echo $((n+1)) > "$COUNTER" [ "$n" -lt "$FAILS" ] && { echo "permission check failed" >&2; exit 1; } echo '[]' ;; *) : ;; esac ''' SLEEP = '#!/usr/bin/env bash\necho "sleep $*" >> "$CALLLOG"\n' fails = [] ran = 0 def check(name, cond, detail=""): global ran ran += 1 if cond: print("ok %s" % name) else: fails.append("%s %s" % (name, detail)) def run(op, refuse, admin_refuse=0): d = tempfile.mkdtemp(prefix="tenantsync-selfcheck-") try: b = os.path.join(d, "bin") os.makedirs(b) for name, body in (("proxmox-backup-manager", PBM), ("proxmox-backup-client", PBC), ("sleep", SLEEP)): p = os.path.join(b, name) open(p, "w").write(body) os.chmod(p, 0o755) calls = os.path.join(d, "calls.log") open(calls, "w").close() e = dict(os.environ, PATH=b + os.pathsep + os.environ["PATH"], CALLLOG=calls, COUNTER=os.path.join(d, "n"), FAILS=str(refuse), ADMIN_FAILS=str(admin_refuse)) p = subprocess.run(["bash", SCRIPT], input=json.dumps({"op": op, "customer_id": "Tester4"}), env=e, capture_output=True, text=True, timeout=60) return p.returncode, p.stdout, p.stderr, open(calls).read().splitlines() finally: shutil.rmtree(d, ignore_errors=True) def gens(calls): return [c for c in calls if c.startswith("pbm user generate-token felhom@pbs")] def checks(calls): return [c for c in calls if c.startswith("pbc snapshot list")] def main(): if not shutil.which("bash") or not shutil.which("jq"): print("test_tenantsync_selfcheck: NOT CHECKED — bash or jq is missing here (the workspace run has both)") return 0 # 1. the measured shape: refused for ~4 s, then the same token passes. for op in ("reissue", "provision"): rc, out, err, calls = run(op, 8) status = (json.loads(out or "{}").get("status") if out.strip().startswith("{") else None) check("%s: a self-check refused for ~4 s passes on the SAME token" % op, rc == 0 and status == "ok", "rc=%s out=%s" % (rc, out.strip()[:160])) check("%s: one token generated, never a regen" % op, len(gens(calls)) == 1, "generate-token calls: %d" % len(gens(calls))) check("%s: nine checks, eight 0.5-s waits" % op, len(checks(calls)) == 9 and calls.count("sleep 0.5") == 8, "checks=%d sleeps=%d" % (len(checks(calls)), calls.count("sleep 0.5"))) check("%s: a log line per refused try" % op, err.count("self-check try") == 8 and "passed on try 9/30" in err, err[-300:]) check("%s: no rollback" % op, not any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-3:])) check("%s: the secret never reaches stderr" % op, "FAKE-TOKEN-VALUE" not in err) # 2. refused for ever → bounded, rolled back, typed. rc, out, err, calls = run("reissue", 10 ** 6) check("refused for ever: self_check_failed, rolled back", rc == 1 and '"code":"self_check_failed"' in out, out.strip()[:200]) check("refused for ever: bounded — two windows of 30 checks", len(checks(calls)) == 60, "checks=%d" % len(checks(calls))) check("refused for ever: the token is deleted at the end", calls[-1].startswith("pbm acl update") or any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-4:]), str(calls[-4:])) # 4. the admin token's first namespace list refused for ~3 s → it waits, and never tries to CREATE the namespace. rc, out, err, calls = run("reissue", 0, admin_refuse=6) check("admin token refused for ~3 s: the op still succeeds", rc == 0 and '"status":"ok"' in out, out.strip()[:160]) check("admin token refused: never a namespace create on a refused list", not any(c.startswith("pbc namespace create") for c in calls), str([c for c in calls if "namespace" in c])) check("admin token refused: a log line per try", err.count("admin token try") == 6 and "admin token ready on try 7/30" in err, err[-300:]) # 3. passing at once → no wait. rc, out, err, calls = run("reissue", 0) check("passing at once: one check, no sleep", rc == 0 and len(checks(calls)) == 1 and not any(c.startswith("sleep") for c in calls), "checks=%d" % len(checks(calls))) if fails: for f in fails: print("FAIL %s" % f) print("test_tenantsync_selfcheck: %d of %d check(s) FAILED" % (len(fails), ran)) return 1 print("test_tenantsync_selfcheck: all %d checks passed" % ran) return 0 if __name__ == "__main__": sys.exit(main())