hub v0.161.0: passphrase keys only until the box uses its own (box-config route; R-1159, D19); ep0 namespaces without a prune job alarmed, tenantsync v1.3.0 creates the job (R-1165)

2026-10-19 15:55:45 +02:00
parent 224fe41e06
commit b6a911c3f9
16 changed files with 802 additions and 35 deletions
@@ -1,7 +1,16 @@
 #!/usr/bin/env bash
-# felhom-tenantsync v1.2.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier
+# felhom-tenantsync v1.3.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier
 # SLICE 1; spike SPIKE-pbs-tier-provisioning-2026-07-10 §3).
 #
+# v1.3.0 (2026-10-19, R-1165): a household's namespace gets its PRUNE JOB with it. provision/reissue ensure
+#   `prune-<id>` (keep-weekly 2, 03:30, ns-scoped, max-depth 0 — the shape of the demo-hp/demo-felhom jobs, STATUS
+#   D18) unless a job on felhom-offsite already covers that namespace (idempotent: a second run changes nothing);
+#   a failed create is logged and never fails the provision (the hub's alarm below names the namespace). deprovision
+#   removes the namespace's jobs before the namespace. `usage` adds `unpruned` — the top-level namespaces no job
+#   covers (read-only: the namespace folders + `prune-job list`), or `unpruned_checked:false` when the list cannot be
+#   read. Box tokens stay write-only: pruning is server-side, so a namespace without a job grows for ever.
+#   Pinned by scripts/tenantsync-prune-harness.sh.
+#
 # v1.2.0 (2026-07-17, hub R-5 PBS visibility): + read-only `usage` op — the felhom-offsite datastore's
 #   total/used/avail (bytes, via `df` on the datastore path). No customer_id, no admin token, NO
 #   mutation. Structural twin of `fingerprint`. Backward-compatible: an older hub never sends it, and a
@@ -28,7 +37,8 @@
 #       This is the DELIBERATE, gated data-destruction the slice-1 note reserved — the operator RESET
 #       confirm (typed customer-id + separate escrow-custody ack) is the human decision it demanded.
 #   {"op":"fingerprint"}                     → {"status":"ok","fingerprint":"<PBS cert sha256>"}
-#   {"op":"usage"}                           → {"status":"ok","total":<bytes>,"used":<bytes>,"avail":<bytes>}
+#   {"op":"usage"}                           → {"status":"ok","total":<bytes>,"used":<bytes>,"avail":<bytes>,
+#                                                "unpruned":[<ns>…],"unpruned_checked":true}
 #       READ-ONLY datastore fill (df on the felhom-offsite path). No customer_id, no admin token, no mutation.
 #
 # Secret hygiene (load-bearing):
@@ -85,7 +95,21 @@ if [ "$OP" = "usage" ]; then
  # df -B1: 1-byte blocks. --output pins the columns; tail -1 skips the header row.
  read -r TOTAL USED AVAIL < <(df -B1 --output=size,used,avail "$DS_PATH" | tail -1)
  [ -n "$TOTAL" ] && [ -n "$USED" ] && [ -n "$AVAIL" ] || err_json internal "could not read datastore usage"
-  printf '{"status":"ok","total":%s,"used":%s,"avail":%s}\n' "$TOTAL" "$USED" "$AVAIL"
+  # v1.3.0 (R-1165): the top-level namespaces no prune job covers. A job covers a namespace when it names it, or when
+  # it sits on the root (no ns) without a max-depth of 0. Read-only; a list that cannot be read says so.
+  NSES='[]'
+  if [ -d "$DS_PATH/ns" ]; then
+    NSES=$(find "$DS_PATH/ns" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | jq -R . | jq -cs .)
+  fi
+  if JOBS=$(proxmox-backup-manager prune-job list --output-format json 2>/dev/null) && jq -e 'type == "array"' >/dev/null 2>&1 <<<"$JOBS"; then
+    jq -cn --argjson t "$TOTAL" --argjson u "$USED" --argjson a "$AVAIL" --argjson ns "$NSES" --argjson jobs "$JOBS" --arg ds "$DS" '
+      ($jobs | map(select(.store == $ds))) as $j
+      | ($j | any(((.ns // "") == "") and ((."max-depth" // 7) > 0))) as $root
+      | {"status":"ok","total":$t,"used":$u,"avail":$a,"unpruned_checked":true,
+         "unpruned": (if $root then [] else [$ns[] as $n | select(($j | any((.ns // "") == $n)) | not) | $n] | sort end)}'
+  else
+    printf '{"status":"ok","total":%s,"used":%s,"avail":%s,"unpruned_checked":false}\n' "$TOTAL" "$USED" "$AVAIL"
+  fi
  exit 0
 fi

@@ -126,6 +150,11 @@ if [ "$OP" = "deprovision" ]; then
  # residual namespace ACLs (belt-and-suspenders — the user grant is not token-scoped).
  proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
  proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" --delete >&2 2>/dev/null || true
+  # v1.3.0 (R-1165): the namespace's prune job(s) go first — a job left on a deleted namespace fails every night.
+  for job in $(proxmox-backup-manager prune-job list --output-format json 2>/dev/null \
+      | jq -r --arg ns "$CID" --arg ds "$DS" '.[] | select(.store == $ds and (.ns // "") == $ns) | .id'); do
+    proxmox-backup-manager prune-job remove "$job" >&2 && log "deprovision: prune job $job removed"
+  done
  ns_existed=false
  if PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
      --output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
@@ -155,6 +184,23 @@ if ! PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADM
  log "created namespace $CID"
 fi

+# 3b. v1.3.0 (R-1165): the namespace's prune job — keep-weekly 2 at 03:30, the D18 shape. Idempotent: any job on this
+#     datastore that already names the namespace counts (whatever its id). A PBS job id is at most 32 characters, so a
+#     long customer id gets a shortened, hashed one. A failure is logged, never fatal (the hub's usage alarm names it).
+JOB_ID="prune-$CID"
+if [ "${#JOB_ID}" -gt 32 ]; then
+  JOB_ID="prune-${CID:0:20}-$(printf '%s' "$CID" | sha256sum | cut -c1-5)"
+fi
+if proxmox-backup-manager prune-job list --output-format json \
+    | jq -e --arg ns "$CID" --arg ds "$DS" 'any(.[]; .store == $ds and (.ns // "") == $ns)' >/dev/null; then
+  log "prune job for namespace $CID already present"
+elif proxmox-backup-manager prune-job create "$JOB_ID" --store "$DS" --ns "$CID" --max-depth 0 --keep-weekly 2 \
+    --schedule 03:30 --comment "R-1165: one copy per ISO week, 2 weeks, server-side (box tokens are write-only)" >&2; then
+  log "created prune job $JOB_ID (keep-weekly 2, 03:30) for namespace $CID"
+else
+  log "WARNING: could not create prune job $JOB_ID for namespace $CID — its copies are not pruned until one exists"
+fi
+
 token_exists() {
  proxmox-backup-manager user list-tokens "$PBS_USER" --output-format json \
    | jq -e --arg t "$TOKEN_ID" 'any(.[]; .tokenid == $t)' >/dev/null
@@ -23,6 +23,7 @@ echo "pbm $*" >> "$CALLLOG"
 case "$1 $2" in
  "cert info")        echo "Fingerprint (sha256): aa:bb:cc:dd" ;;
  "datastore list")   printf '[{"name":"scratch","path":"/srv/pbs-scratch"},{"name":"felhom-offsite","path":"%s"}]\n' "$DS_PATH" ;;
+  "prune-job list")   echo '[]' ;;   # read-only (v1.3.0 usage reads it for `unpruned`)
  "user generate-token") echo "MUTATION: $*" >> "$MUTLOG"; echo '{"value":"FAKE-ADMIN-TOKEN"}' ;;
  "user delete-token")   echo "MUTATION: $*" >> "$MUTLOG" ;;
  "acl update")          echo "MUTATION: $*" >> "$MUTLOG" ;;
@@ -54,7 +55,7 @@ fail() { echo "FAIL: $*"; RC=1; }
 # --- Test 1: usage op ---
 OUT=$(echo '{"op":"usage"}' | PATH="$STUB:$PATH" bash "$SCRIPT" 2>/dev/null); EC=$?
 [ "$EC" -eq 0 ] || fail "usage exit $EC (want 0)"
-EXP='{"status":"ok","total":41943040,"used":8388608,"avail":33554432}'
+EXP='{"status":"ok","total":41943040,"used":8388608,"avail":33554432,"unpruned_checked":true,"unpruned":[]}'
 [ "$OUT" = "$EXP" ] || fail "usage stdout = $OUT (want $EXP)"
 if [ -s "$MUTLOG" ]; then fail "usage ran mutations: $(cat "$MUTLOG")"; else echo "PASS: usage emits totals, exit 0, ZERO mutations"; fi

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""test_tenantsync_prune.py — felhom-tenantsync v1.3.0 gives every household namespace its prune job (R-1165).
+
+Runs the REAL script against stubbed `proxmox-backup-manager` / `proxmox-backup-client` / `df` on PATH (real bash, real
+jq). The stubs keep the prune jobs in a JSON file and LOG every call, so each case asserts what the script DID:
+
+  1. provision of a new household → exactly one `prune-job create prune-<id> --store felhom-offsite --ns <id>
+     --max-depth 0 --keep-weekly 2 --schedule 03:30`, and the provision still succeeds;
+  2. a second run (reissue) with the job present → no create (idempotent); a job under ANOTHER id that already names
+     the namespace counts too;
+  3. a 31-character customer id → a job id of at most 32 characters (PBS's limit);
+  4. a create that fails → logged, the provision still succeeds (the alarm names the namespace);
+  5. deprovision → the namespace's job removed, another household's kept;
+  6. usage → `unpruned` = the namespaces no job covers; a root job without max-depth 0 covers all; an unreadable job
+     list → `unpruned_checked: false`, never an empty list that reads as „all covered".
+
+Exit 0 all passed · 1 a case failed. Without bash or jq it says NOT CHECKED out loud and exits 0 (the CI runner may
+lack jq; the workspace pre-push run has both).
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPT = os.path.join(HERE, "felhom-tenantsync.sh")
+
+PBM = r'''#!/usr/bin/env bash
+echo "pbm $*" >> "$CALLLOG"
+case "$1 $2" in
+  "cert info")            echo "Fingerprint (sha256): aa:bb:cc:dd" ;;
+  "datastore list")       printf '[{"name":"felhom-offsite","path":"%s"}]\n' "$DS_PATH" ;;
+  "user generate-token")  echo '{"value": "FAKE-TOKEN-VALUE"}' ;;
+  "user list")            echo '[{"userid":"felhom@pbs"}]' ;;
+  "user list-tokens")     echo '[]' ;;
+  "prune-job list")       [ -n "${JOBS_FAIL:-}" ] && exit 1; cat "$JOBS" ;;
+  "prune-job create")     [ -n "${CREATE_FAIL:-}" ] && exit 1
+                          id="$3"; shift 3; ns=""
+                          while [ $# -gt 0 ]; do [ "$1" = "--ns" ] && ns="$2"; shift; done
+                          jq -c --arg id "$id" --arg ns "$ns" '. + [{"id":$id,"store":"felhom-offsite","ns":$ns,"keep-weekly":2,"schedule":"03:30"}]' "$JOBS" > "$JOBS.n" && mv "$JOBS.n" "$JOBS" ;;
+  "prune-job remove")     jq -c --arg id "$3" 'map(select(.id != $id))' "$JOBS" > "$JOBS.n" && mv "$JOBS.n" "$JOBS" ;;
+  *)                      : ;;
+esac
+'''
+PBC = r'''#!/usr/bin/env bash
+echo "pbc $*" >> "$CALLLOG"
+case "$1 $2" in
+  "namespace list")   echo '{"data":[]}' ;;
+  *)                  : ;;
+esac
+'''
+DF = '#!/usr/bin/env bash\necho "1B-blocks Used Avail"\necho "41943040 8388608 33554432"\n'
+
+fails = []
+ran = 0
+
+
+class Box(object):
+    def __init__(self):
+        self.dir = tempfile.mkdtemp(prefix="tenantsync-prune-")
+        self.bin = os.path.join(self.dir, "bin")
+        os.makedirs(self.bin)
+        for name, body in (("proxmox-backup-manager", PBM), ("proxmox-backup-client", PBC), ("df", DF)):
+            p = os.path.join(self.bin, name)
+            open(p, "w").write(body)
+            os.chmod(p, 0o755)
+        self.ds = os.path.join(self.dir, "ds")
+        os.makedirs(os.path.join(self.ds, "ns"))
+        self.jobs = os.path.join(self.dir, "jobs.json")
+        self.calls = os.path.join(self.dir, "calls.log")
+        self.set_jobs([])
+
+    def set_jobs(self, jobs):
+        open(self.jobs, "w").write(json.dumps(jobs))
+
+    def get_jobs(self):
+        return json.load(open(self.jobs))
+
+    def run(self, payload, **env):
+        open(self.calls, "w").close()
+        e = dict(os.environ, PATH=self.bin + os.pathsep + os.environ["PATH"], DS_PATH=self.ds, JOBS=self.jobs,
+                 CALLLOG=self.calls)
+        e.update(env)
+        p = subprocess.run(["bash", SCRIPT], input=json.dumps(payload), env=e, capture_output=True, text=True)
+        return p.returncode, p.stdout, p.stderr, open(self.calls).read().splitlines()
+
+    def close(self):
+        shutil.rmtree(self.dir, ignore_errors=True)
+
+
+def check(name, cond, detail=""):
+    global ran
+    ran += 1
+    if cond:
+        print("ok   %s" % name)
+    else:
+        fails.append("%s %s" % (name, detail))
+
+
+def creates(calls):
+    return [c for c in calls if c.startswith("pbm prune-job create")]
+
+
+def main():
+    if not shutil.which("bash") or not shutil.which("jq"):
+        print("test_tenantsync_prune: NOT CHECKED — bash or jq is missing here (the workspace run has both)")
+        return 0
+
+    b = Box()
+    try:
+        rc, out, err, calls = b.run({"op": "provision", "customer_id": "house-1"})
+        cr = creates(calls)
+        check("provision/creates-the-job", rc == 0 and len(cr) == 1 and cr[0] == (
+            "pbm prune-job create prune-house-1 --store felhom-offsite --ns house-1 --max-depth 0 --keep-weekly 2 "
+            "--schedule 03:30 --comment R-1165: one copy per ISO week, 2 weeks, server-side (box tokens are write-only)"),
+            "rc=%d creates=%s err=%s" % (rc, cr, err[-300:]))
+        check("provision/still-answers-ok", '"status":"ok"' in out and "token_secret" in out, out[-200:])
+
+        rc, out, err, calls = b.run({"op": "reissue", "customer_id": "house-1"})
+        check("reissue/idempotent", rc == 0 and not creates(calls) and len(b.get_jobs()) == 1,
+              "rc=%d creates=%s jobs=%s" % (rc, creates(calls), b.get_jobs()))
+
+        b.set_jobs([{"id": "hand-made", "store": "felhom-offsite", "ns": "house-2", "keep-weekly": 2}])
+        rc, out, err, calls = b.run({"op": "provision", "customer_id": "house-2"})
+        check("provision/a-job-under-another-id-counts", rc == 0 and not creates(calls), "creates=%s" % creates(calls))
+
+        long_id = "a" * 31
+        rc, out, err, calls = b.run({"op": "provision", "customer_id": long_id})
+        cr = creates(calls)
+        jid = cr[0].split()[3] if cr else ""
+        check("provision/long-id-fits-32", rc == 0 and 0 < len(jid) <= 32 and jid.startswith("prune-"), "id=%r" % jid)
+
+        rc, out, err, calls = b.run({"op": "provision", "customer_id": "house-3"}, CREATE_FAIL="1")
+        check("provision/create-failure-is-not-fatal", rc == 0 and '"status":"ok"' in out and "could not create prune job" in err,
+              "rc=%d out=%s err=%s" % (rc, out[-120:], err[-200:]))
+
+        b.set_jobs([{"id": "prune-house-1", "store": "felhom-offsite", "ns": "house-1"},
+                    {"id": "prune-house-9", "store": "felhom-offsite", "ns": "house-9"}])
+        rc, out, err, calls = b.run({"op": "deprovision", "customer_id": "house-1"})
+        ids = [j["id"] for j in b.get_jobs()]
+        check("deprovision/removes-only-its-job", rc == 0 and ids == ["prune-house-9"], "rc=%d ids=%s" % (rc, ids))
+
+        for n in ("demo-hp", "tester-1", "Tester-2"):
+            os.makedirs(os.path.join(b.ds, "ns", n))
+        b.set_jobs([{"id": "prune-demo-hp", "store": "felhom-offsite", "ns": "demo-hp", "max-depth": 0},
+                    {"id": "other-store", "store": "elsewhere", "ns": "tester-1"}])
+        rc, out, err, calls = b.run({"op": "usage"})
+        u = json.loads(out) if out.strip().startswith("{") else {}
+        check("usage/names-the-unpruned", rc == 0 and u.get("unpruned_checked") is True and u.get("unpruned") == ["Tester-2", "tester-1"]
+              and u.get("total") == 41943040, out)
+        check("usage/no-mutation", not [c for c in calls if any(k in c for k in ("create", "remove", "generate-token", "acl"))], calls)
+
+        b.set_jobs([{"id": "root", "store": "felhom-offsite", "keep-weekly": 2}])
+        rc, out, err, calls = b.run({"op": "usage"})
+        u = json.loads(out) if out.strip().startswith("{") else {}
+        check("usage/a-root-job-covers-all", u.get("unpruned") == [] and u.get("unpruned_checked") is True, out)
+
+        b.set_jobs([{"id": "root0", "store": "felhom-offsite", "max-depth": 0}])
+        rc, out, err, calls = b.run({"op": "usage"})
+        u = json.loads(out) if out.strip().startswith("{") else {}
+        check("usage/a-root-job-with-depth-0-covers-none", u.get("unpruned") == ["Tester-2", "demo-hp", "tester-1"], out)
+
+        rc, out, err, calls = b.run({"op": "usage"}, JOBS_FAIL="1")
+        u = json.loads(out) if out.strip().startswith("{") else {}
+        check("usage/unreadable-jobs-say-so", rc == 0 and u.get("unpruned_checked") is False and "unpruned" not in u, out)
+    finally:
+        b.close()
+
+    if fails:
+        for f in fails:
+            print("FAIL: %s" % f)
+        return 1
+    print("test_tenantsync_prune: all %d cases passed" % ran)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())