R-1295: the PBS DR set-up waits out PBS's 5-s permission cache. Cause measured on a throwaway PBS 4.2.5 under load (v1.3.0 failed 11/30, v1.4.0 0/30): tenantsync v1.4.0 re-checks the same token (and the admin token) up to 15 s; hub v0.172.0 retries a self_check_failed rollback 3x, 7 s apart, and a final failure mails the operator (pbsdr_provision_failed, operator-only). Red-proved. Commit only; ep0 unchanged.
gates / gates (push) Successful in 13m24s
gates / gates (push) Successful in 13m24s
This commit is contained in:
admin committed
1 parent
c69f53055d
commit
722013da90
13 files changed
+553
-5
No files matched your search
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""test_tenantsync_selfcheck.py — felhom-tenantsync v1.4.0 waits out the PBS proxy's permission cache (R-1295).
|
||||
|
||||
Runs the REAL script against stubbed `proxmox-backup-manager` / `proxmox-backup-client` / `sleep` on PATH (real bash,
|
||||
real jq). The client stub refuses `snapshot list` (the self-check) the first FAILS times — the shape measured on a
|
||||
throwaway PBS 4.2.5 under load (a 403 for 4.2–5.0 s after the grant) — and the sleep stub only counts, so the test is
|
||||
fast. Every call is logged, so each case asserts what the script DID:
|
||||
|
||||
1. the check refused for ~4 s (8 tries) → the SAME token passes; one generate-token, no rollback, a log line per try;
|
||||
2. refused for ever → bounded: two windows of 30 tries (the old regen-once path after the first), then the rollback
|
||||
and `self_check_failed` — never a hang;
|
||||
3. passing at once → one check, no sleep;
|
||||
4. the transient ADMIN token's first namespace list refused for ~3 s → it waits; never a `namespace create` on a
|
||||
refused list (measured: under load v1.3.0 then died on „missing Datastore.Modify").
|
||||
|
||||
RED-PROOF: the v1.3.0 script (one immediate re-check after a regen) fails case 1 — it rolls back.
|
||||
Exit 0 all passed · 1 a case failed. Without bash or jq it says NOT CHECKED out loud and exits 0.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.environ.get("TENANTSYNC_SCRIPT") or os.path.join(HERE, "felhom-tenantsync.sh")
|
||||
|
||||
PBM = r'''#!/usr/bin/env bash
|
||||
echo "pbm $*" >> "$CALLLOG"
|
||||
case "$1 $2" in
|
||||
"cert info") echo "Fingerprint (sha256): aa:bb:cc:dd" ;;
|
||||
"datastore list") echo '[{"name":"felhom-offsite","path":"/nonexistent"}]' ;;
|
||||
"user generate-token") echo '{"value": "FAKE-TOKEN-VALUE"}' ;;
|
||||
"user list") echo '[{"userid":"felhom@pbs"}]' ;;
|
||||
"user list-tokens") echo '[]' ;;
|
||||
"prune-job list") echo '[{"id":"prune-x","store":"felhom-offsite","ns":"Tester4"}]' ;;
|
||||
*) : ;;
|
||||
esac
|
||||
'''
|
||||
PBC = r'''#!/usr/bin/env bash
|
||||
echo "pbc $*" >> "$CALLLOG"
|
||||
case "$1 $2" in
|
||||
"namespace list") m=$(cat "$COUNTER.adm" 2>/dev/null || echo 0); echo $((m+1)) > "$COUNTER.adm"
|
||||
[ "$m" -lt "${ADMIN_FAILS:-0}" ] && { echo "permission check failed" >&2; exit 1; }
|
||||
echo '{"data":[{"ns":"Tester4"}]}' ;;
|
||||
"snapshot list") n=$(cat "$COUNTER" 2>/dev/null || echo 0); echo $((n+1)) > "$COUNTER"
|
||||
[ "$n" -lt "$FAILS" ] && { echo "permission check failed" >&2; exit 1; }
|
||||
echo '[]' ;;
|
||||
*) : ;;
|
||||
esac
|
||||
'''
|
||||
SLEEP = '#!/usr/bin/env bash\necho "sleep $*" >> "$CALLLOG"\n'
|
||||
|
||||
fails = []
|
||||
ran = 0
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
global ran
|
||||
ran += 1
|
||||
if cond:
|
||||
print("ok %s" % name)
|
||||
else:
|
||||
fails.append("%s %s" % (name, detail))
|
||||
|
||||
|
||||
def run(op, refuse, admin_refuse=0):
|
||||
d = tempfile.mkdtemp(prefix="tenantsync-selfcheck-")
|
||||
try:
|
||||
b = os.path.join(d, "bin")
|
||||
os.makedirs(b)
|
||||
for name, body in (("proxmox-backup-manager", PBM), ("proxmox-backup-client", PBC), ("sleep", SLEEP)):
|
||||
p = os.path.join(b, name)
|
||||
open(p, "w").write(body)
|
||||
os.chmod(p, 0o755)
|
||||
calls = os.path.join(d, "calls.log")
|
||||
open(calls, "w").close()
|
||||
e = dict(os.environ, PATH=b + os.pathsep + os.environ["PATH"], CALLLOG=calls,
|
||||
COUNTER=os.path.join(d, "n"), FAILS=str(refuse), ADMIN_FAILS=str(admin_refuse))
|
||||
p = subprocess.run(["bash", SCRIPT], input=json.dumps({"op": op, "customer_id": "Tester4"}), env=e,
|
||||
capture_output=True, text=True, timeout=60)
|
||||
return p.returncode, p.stdout, p.stderr, open(calls).read().splitlines()
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
def gens(calls):
|
||||
return [c for c in calls if c.startswith("pbm user generate-token felhom@pbs")]
|
||||
|
||||
|
||||
def checks(calls):
|
||||
return [c for c in calls if c.startswith("pbc snapshot list")]
|
||||
|
||||
|
||||
def main():
|
||||
if not shutil.which("bash") or not shutil.which("jq"):
|
||||
print("test_tenantsync_selfcheck: NOT CHECKED — bash or jq is missing here (the workspace run has both)")
|
||||
return 0
|
||||
|
||||
# 1. the measured shape: refused for ~4 s, then the same token passes.
|
||||
for op in ("reissue", "provision"):
|
||||
rc, out, err, calls = run(op, 8)
|
||||
status = (json.loads(out or "{}").get("status") if out.strip().startswith("{") else None)
|
||||
check("%s: a self-check refused for ~4 s passes on the SAME token" % op, rc == 0 and status == "ok",
|
||||
"rc=%s out=%s" % (rc, out.strip()[:160]))
|
||||
check("%s: one token generated, never a regen" % op, len(gens(calls)) == 1, "generate-token calls: %d" % len(gens(calls)))
|
||||
check("%s: nine checks, eight 0.5-s waits" % op, len(checks(calls)) == 9 and calls.count("sleep 0.5") == 8,
|
||||
"checks=%d sleeps=%d" % (len(checks(calls)), calls.count("sleep 0.5")))
|
||||
check("%s: a log line per refused try" % op, err.count("self-check try") == 8 and "passed on try 9/30" in err,
|
||||
err[-300:])
|
||||
check("%s: no rollback" % op, not any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-3:]))
|
||||
check("%s: the secret never reaches stderr" % op, "FAKE-TOKEN-VALUE" not in err)
|
||||
|
||||
# 2. refused for ever → bounded, rolled back, typed.
|
||||
rc, out, err, calls = run("reissue", 10 ** 6)
|
||||
check("refused for ever: self_check_failed, rolled back", rc == 1 and '"code":"self_check_failed"' in out, out.strip()[:200])
|
||||
check("refused for ever: bounded — two windows of 30 checks", len(checks(calls)) == 60, "checks=%d" % len(checks(calls)))
|
||||
check("refused for ever: the token is deleted at the end", calls[-1].startswith("pbm acl update") or
|
||||
any(c.startswith("pbm user delete-token felhom@pbs Tester4") for c in calls[-4:]), str(calls[-4:]))
|
||||
|
||||
# 4. the admin token's first namespace list refused for ~3 s → it waits, and never tries to CREATE the namespace.
|
||||
rc, out, err, calls = run("reissue", 0, admin_refuse=6)
|
||||
check("admin token refused for ~3 s: the op still succeeds", rc == 0 and '"status":"ok"' in out, out.strip()[:160])
|
||||
check("admin token refused: never a namespace create on a refused list",
|
||||
not any(c.startswith("pbc namespace create") for c in calls), str([c for c in calls if "namespace" in c]))
|
||||
check("admin token refused: a log line per try", err.count("admin token try") == 6 and "admin token ready on try 7/30" in err, err[-300:])
|
||||
|
||||
# 3. passing at once → no wait.
|
||||
rc, out, err, calls = run("reissue", 0)
|
||||
check("passing at once: one check, no sleep", rc == 0 and len(checks(calls)) == 1 and not any(c.startswith("sleep") for c in calls),
|
||||
"checks=%d" % len(checks(calls)))
|
||||
|
||||
if fails:
|
||||
for f in fails:
|
||||
print("FAIL %s" % f)
|
||||
print("test_tenantsync_selfcheck: %d of %d check(s) FAILED" % (len(fails), ran))
|
||||
return 1
|
||||
print("test_tenantsync_selfcheck: all %d checks passed" % ran)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user