297 lines
18 KiB
Bash
297 lines
18 KiB
Bash
#!/usr/bin/env bash
|
||
# felhom-tenantsync v1.4.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier
|
||
# SLICE 1; spike SPIKE-pbs-tier-provisioning-2026-07-10 §3).
|
||
#
|
||
# v1.4.0 (2026-10-26, R-1295): the self-check WAITS OUT the proxy's permission cache before it gives up. PBS caches
|
||
# permissions for up to 5 s and an ACL-only change does not clear that cache; a request from another client between
|
||
# the token create and its grant leaves „new token, old ACLs" for up to ~5 s (reproduced on a throwaway PBS 4.2.5 with
|
||
# a second client listing every 50 ms: 9 of 20 first checks failed, the window 4.2–5.0 s). The old one-shot retry came
|
||
# within a second and failed inside the same window (Tester4, 2026-10-26 09:31:15Z: rolled back, the box got no
|
||
# whole-guest off-site tier). Now: the SAME token is re-checked every 0.5 s for up to 15 s, a log line per failed try;
|
||
# only then the old regenerate-once-and-rollback path. The transient admin token waits the same way before its first
|
||
# namespace list (it met the same cache under load). Pinned by scripts/test_tenantsync_selfcheck.py.
|
||
#
|
||
# v1.3.0 (2026-10-19, R-1165): a household's namespace gets its PRUNE JOB with it. provision/reissue ensure
|
||
# `prune-<id>` (keep-weekly 2, 03:30, ns-scoped, max-depth 0 — the shape of the demo-hp/demo-felhom jobs, STATUS
|
||
# D18) unless a job on felhom-offsite already covers that namespace (idempotent: a second run changes nothing);
|
||
# a failed create is logged and never fails the provision (the hub's alarm below names the namespace). deprovision
|
||
# removes the namespace's jobs before the namespace. `usage` adds `unpruned` — the top-level namespaces no job
|
||
# covers (read-only: the namespace folders + `prune-job list`), or `unpruned_checked:false` when the list cannot be
|
||
# read. Box tokens stay write-only: pruning is server-side, so a namespace without a job grows for ever.
|
||
# Pinned by scripts/tenantsync-prune-harness.sh.
|
||
#
|
||
# v1.2.0 (2026-07-17, hub R-5 PBS visibility): + read-only `usage` op — the felhom-offsite datastore's
|
||
# total/used/avail (bytes, via `df` on the datastore path). No customer_id, no admin token, NO
|
||
# mutation. Structural twin of `fingerprint`. Backward-compatible: an older hub never sends it, and a
|
||
# hub that does against an OLD script gets the existing `bad_request "unknown op"` (graceful).
|
||
#
|
||
# Runs as the SSH forced command for the hub's SECOND `felhom-peersync` key (via sudo — its own
|
||
# single sudoers line; the peersync script/key are untouched: one script, one job). JSON on stdin,
|
||
# JSON on stdout. Ops:
|
||
#
|
||
# {"op":"provision","customer_id":"<id>"} → ensure namespace <id> (idempotent) → CREATE token
|
||
# felhom@pbs!<id> (an EXISTING token is a hard error, code "token_exists" — re-issue is the
|
||
# explicit path) → dual-grant DatastoreBackup on /datastore/felhom-offsite/<id> to BOTH the
|
||
# user and the token (PBS privsep = intersection) → self-check: list the namespace AS the
|
||
# new token (one regen retry per the spike's transient-403 note; still failing → rollback)
|
||
# → {"status":"ok","token_id","token_secret","fingerprint","datastore","namespace"}
|
||
# {"op":"reissue","customer_id":"<id>"} → delete-token (its ACLs purge with it — spike) →
|
||
# recreate → re-grant BOTH → self-check → same ok-shape with the FRESH secret.
|
||
# {"op":"deprovision","customer_id":"<id>"} → the customer-RESET teardown (v0.61.0, hub-side
|
||
# ack-gated). Delete the token (ACLs purge with it) → delete the residual namespace ACLs →
|
||
# DESTROY the namespace AND all its backup groups (`namespace delete --delete-groups true`).
|
||
# IDEMPOTENT: a missing token / missing namespace is success, not an error (a re-run after a
|
||
# partial reset converges). The shared felhom@pbs USER is NEVER touched (other tenants ride it).
|
||
# → {"status":"ok","namespace","datastore","deleted":<bool ns existed>}.
|
||
# This is the DELIBERATE, gated data-destruction the slice-1 note reserved — the operator RESET
|
||
# confirm (typed customer-id + separate escrow-custody ack) is the human decision it demanded.
|
||
# {"op":"fingerprint"} → {"status":"ok","fingerprint":"<PBS cert sha256>"}
|
||
# {"op":"usage"} → {"status":"ok","total":<bytes>,"used":<bytes>,"avail":<bytes>,
|
||
# "unpruned":[<ns>…],"unpruned_checked":true}
|
||
# READ-ONLY datastore fill (df on the felhom-offsite path). No customer_id, no admin token, no mutation.
|
||
#
|
||
# Secret hygiene (load-bearing):
|
||
# - The token secret exists ONLY in memory and in the final stdout JSON — never a file, never
|
||
# stderr (the hub embeds remote stderr in error logs), never an argument.
|
||
# - The transient root@pam admin token (namespace ops are client-side) is held in memory and
|
||
# deleted on EVERY exit path (trap). A leftover from a crashed run is deleted at entry.
|
||
# - All tool stdout is redirected to stderr — the response JSON is the ONLY stdout bytes.
|
||
#
|
||
# Ordering facts this script encodes (all live-proven in the spike — do not "simplify"):
|
||
# - the token must exist BEFORE its ACL grant (PBS validates the auth-id);
|
||
# - user delete-token PURGES the token's ACLs → re-issue must re-grant;
|
||
# - `user generate-token` has no --output-format → the "value" line is sed-parsed;
|
||
# - proxmox-backup-client needs PBS_FINGERPRINT even against localhost.
|
||
set -euo pipefail
|
||
|
||
DS=felhom-offsite
|
||
PBS_USER=felhom@pbs
|
||
ADMIN_TOKEN_NAME=tenantsync-admin
|
||
REPO_HOST=localhost
|
||
|
||
err_json() { # code, message → error JSON on stdout, exit 1
|
||
printf '{"status":"error","code":"%s","error":"%s"}\n' "$1" "$2"
|
||
exit 1
|
||
}
|
||
|
||
command -v jq >/dev/null || err_json internal "jq is required"
|
||
command -v proxmox-backup-manager >/dev/null || err_json internal "proxmox-backup-manager is required"
|
||
command -v proxmox-backup-client >/dev/null || err_json internal "proxmox-backup-client is required"
|
||
|
||
# 1. Read stdin capped at 64 KiB; validate the envelope BEFORE touching anything.
|
||
payload=$(head -c 65536)
|
||
[ -n "$payload" ] || err_json bad_request "empty payload"
|
||
jq -e 'type == "object" and (.op | type) == "string"' >/dev/null 2>&1 <<<"$payload" \
|
||
|| err_json bad_request "payload must be a JSON object with a string op"
|
||
OP=$(jq -r '.op' <<<"$payload")
|
||
|
||
export PBS_FINGERPRINT
|
||
PBS_FINGERPRINT=$(proxmox-backup-manager cert info | awk '/Fingerprint/{print $3}' | head -1)
|
||
[ -n "$PBS_FINGERPRINT" ] || err_json internal "could not read the PBS cert fingerprint"
|
||
|
||
if [ "$OP" = "fingerprint" ]; then
|
||
printf '{"status":"ok","fingerprint":"%s"}\n' "$PBS_FINGERPRINT"
|
||
exit 0
|
||
fi
|
||
|
||
# usage (v1.2.0) — READ-ONLY box-level datastore fill. No customer_id, no admin token, NO mutation
|
||
# (a pure `df` on the datastore path). The hub's PBS-DR checker polls this on its 15-min throttle.
|
||
# Short-circuits BEFORE the admin-token generation below, exactly like fingerprint.
|
||
if [ "$OP" = "usage" ]; then
|
||
DS_PATH=$(proxmox-backup-manager datastore list --output-format json \
|
||
| jq -r --arg ds "$DS" '.[] | select(.name == $ds) | .path')
|
||
[ -n "$DS_PATH" ] || err_json internal "datastore $DS not found"
|
||
# df -B1: 1-byte blocks. --output pins the columns; tail -1 skips the header row.
|
||
read -r TOTAL USED AVAIL < <(df -B1 --output=size,used,avail "$DS_PATH" | tail -1)
|
||
[ -n "$TOTAL" ] && [ -n "$USED" ] && [ -n "$AVAIL" ] || err_json internal "could not read datastore usage"
|
||
# v1.3.0 (R-1165): the top-level namespaces no prune job covers. A job covers a namespace when it names it, or when
|
||
# it sits on the root (no ns) without a max-depth of 0. Read-only; a list that cannot be read says so.
|
||
NSES='[]'
|
||
if [ -d "$DS_PATH/ns" ]; then
|
||
NSES=$(find "$DS_PATH/ns" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | jq -R . | jq -cs .)
|
||
fi
|
||
if JOBS=$(proxmox-backup-manager prune-job list --output-format json 2>/dev/null) && jq -e 'type == "array"' >/dev/null 2>&1 <<<"$JOBS"; then
|
||
jq -cn --argjson t "$TOTAL" --argjson u "$USED" --argjson a "$AVAIL" --argjson ns "$NSES" --argjson jobs "$JOBS" --arg ds "$DS" '
|
||
($jobs | map(select(.store == $ds))) as $j
|
||
| ($j | any(((.ns // "") == "") and ((."max-depth" // 7) > 0))) as $root
|
||
| {"status":"ok","total":$t,"used":$u,"avail":$a,"unpruned_checked":true,
|
||
"unpruned": (if $root then [] else [$ns[] as $n | select(($j | any((.ns // "") == $n)) | not) | $n] | sort end)}'
|
||
else
|
||
printf '{"status":"ok","total":%s,"used":%s,"avail":%s,"unpruned_checked":false}\n' "$TOTAL" "$USED" "$AVAIL"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
case "$OP" in provision|reissue|deprovision) ;; *) err_json bad_request "unknown op" ;; esac
|
||
|
||
# customer_id → the namespace AND the token name. Conservative charset (PBS ns + token grammar,
|
||
# no leading dash/dot so it can never parse as an option).
|
||
CID=$(jq -r '.customer_id // ""' <<<"$payload")
|
||
[[ "$CID" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}$ ]] \
|
||
|| err_json bad_request "customer_id must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}\$"
|
||
TOKEN_ID="$PBS_USER!$CID"
|
||
|
||
log() { echo "felhom-tenantsync: $*" >&2; }
|
||
|
||
# 2. Transient admin token for the client-side namespace ops. Deleted on every exit; a leftover
|
||
# from a crashed run is cleared first (generate-token fails on an existing name).
|
||
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
||
ADM=$(proxmox-backup-manager user generate-token root@pam "$ADMIN_TOKEN_NAME" \
|
||
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p')
|
||
[ -n "$ADM" ] || err_json internal "admin token generation failed"
|
||
cleanup_admin() {
|
||
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
||
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
||
--auth-id "root@pam!$ADMIN_TOKEN_NAME" --delete >&2 2>/dev/null || true
|
||
}
|
||
trap cleanup_admin EXIT
|
||
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
||
--auth-id "root@pam!$ADMIN_TOKEN_NAME" >&2
|
||
|
||
ADMIN_REPO="root@pam!$ADMIN_TOKEN_NAME@$REPO_HOST:$DS"
|
||
|
||
# R-1295 (v1.4.0): the transient admin token meets the SAME permission cache as the tenant token — measured on the
|
||
# throwaway PBS under load: its first namespace list refused, then a wrong `namespace create` died on Datastore.Modify.
|
||
# Wait (bounded, a log line per try) until it can list before any namespace op reads its answer.
|
||
SELF_CHECK_TRIES=30
|
||
admin_ready() {
|
||
PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" --output-format json >/dev/null 2>&1
|
||
}
|
||
for i in $(seq 1 "$SELF_CHECK_TRIES"); do
|
||
admin_ready && { [ "$i" -gt 1 ] && log "admin token ready on try $i/$SELF_CHECK_TRIES (R-1295)"; break; }
|
||
log "admin token try $i/$SELF_CHECK_TRIES refused — the proxy may still hold the old permissions (up to 5 s); R-1295"
|
||
[ "$i" -lt "$SELF_CHECK_TRIES" ] && sleep 0.5 || err_json internal "the transient admin token was refused for 15 s (R-1295)"
|
||
done
|
||
|
||
# 2b. deprovision (v0.61.0 customer-RESET teardown): destroy this ONE tenant's token + namespace +
|
||
# backup groups. Every step is idempotent (missing = already gone = ok). The shared felhom@pbs
|
||
# user survives (co-tenants). Returns before the provision/reissue create-path below.
|
||
if [ "$OP" = "deprovision" ]; then
|
||
# token (its ACLs purge with it — spike); ignore "no such token".
|
||
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
||
# residual namespace ACLs (belt-and-suspenders — the user grant is not token-scoped).
|
||
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
||
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" --delete >&2 2>/dev/null || true
|
||
# v1.3.0 (R-1165): the namespace's prune job(s) go first — a job left on a deleted namespace fails every night.
|
||
for job in $(proxmox-backup-manager prune-job list --output-format json 2>/dev/null \
|
||
| jq -r --arg ns "$CID" --arg ds "$DS" '.[] | select(.store == $ds and (.ns // "") == $ns) | .id'); do
|
||
proxmox-backup-manager prune-job remove "$job" >&2 && log "deprovision: prune job $job removed"
|
||
done
|
||
ns_existed=false
|
||
if PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
||
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
||
ns_existed=true
|
||
# --delete-groups true destroys every backup group under the namespace (the deliberate data kill).
|
||
PBS_PASSWORD="$ADM" proxmox-backup-client namespace delete "$CID" --repository "$ADMIN_REPO" --delete-groups true >&2
|
||
log "deprovision: namespace $CID destroyed (all backup groups deleted)"
|
||
else
|
||
log "deprovision: namespace $CID absent — already gone"
|
||
fi
|
||
jq -cn --arg ns "$CID" --arg ds "$DS" --argjson del "$ns_existed" \
|
||
'{"status":"ok","namespace":$ns,"datastore":$ds,"deleted":$del}'
|
||
exit 0
|
||
fi
|
||
|
||
# 3. Ensure the shared user + the namespace (both idempotent).
|
||
if ! proxmox-backup-manager user list --output-format json | jq -e --arg u "$PBS_USER" \
|
||
'any(.[]; .userid == $u)' >/dev/null; then
|
||
proxmox-backup-manager user create "$PBS_USER" --comment 'offsite tenancy' >&2
|
||
log "created user $PBS_USER"
|
||
fi
|
||
# NOTE: the CLIENT wraps json output as {"data":[...]} (live-proven on ep0); the MANAGER commands
|
||
# return bare arrays. `(.data // .)` handles both.
|
||
if ! PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
||
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
||
PBS_PASSWORD="$ADM" proxmox-backup-client namespace create "$CID" --repository "$ADMIN_REPO" >&2
|
||
log "created namespace $CID"
|
||
fi
|
||
|
||
# 3b. v1.3.0 (R-1165): the namespace's prune job — keep-weekly 2 at 03:30, the D18 shape. Idempotent: any job on this
|
||
# datastore that already names the namespace counts (whatever its id). A PBS job id is at most 32 characters, so a
|
||
# long customer id gets a shortened, hashed one. A failure is logged, never fatal (the hub's usage alarm names it).
|
||
JOB_ID="prune-$CID"
|
||
if [ "${#JOB_ID}" -gt 32 ]; then
|
||
JOB_ID="prune-${CID:0:20}-$(printf '%s' "$CID" | sha256sum | cut -c1-5)"
|
||
fi
|
||
if proxmox-backup-manager prune-job list --output-format json \
|
||
| jq -e --arg ns "$CID" --arg ds "$DS" 'any(.[]; .store == $ds and (.ns // "") == $ns)' >/dev/null; then
|
||
log "prune job for namespace $CID already present"
|
||
elif proxmox-backup-manager prune-job create "$JOB_ID" --store "$DS" --ns "$CID" --max-depth 0 --keep-weekly 2 \
|
||
--schedule 03:30 --comment "R-1165: one copy per ISO week, 2 weeks, server-side (box tokens are write-only)" >&2; then
|
||
log "created prune job $JOB_ID (keep-weekly 2, 03:30) for namespace $CID"
|
||
else
|
||
log "WARNING: could not create prune job $JOB_ID for namespace $CID — its copies are not pruned until one exists"
|
||
fi
|
||
|
||
token_exists() {
|
||
proxmox-backup-manager user list-tokens "$PBS_USER" --output-format json \
|
||
| jq -e --arg t "$TOKEN_ID" 'any(.[]; .tokenid == $t)' >/dev/null
|
||
}
|
||
|
||
# gen_token → SECRET on stdout of this function only (command substitution), nothing persisted.
|
||
gen_token() {
|
||
proxmox-backup-manager user generate-token "$PBS_USER" "$CID" \
|
||
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p'
|
||
}
|
||
|
||
grant_both() { # dual-grant on the NAMESPACE ACL path (never /ns/<ns> — §4a gotcha)
|
||
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" >&2
|
||
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" >&2
|
||
}
|
||
|
||
self_check() { # own-namespace list AS the new token; secret via env, never argv/stderr
|
||
PBS_PASSWORD="$1" proxmox-backup-client snapshot list --ns "$CID" \
|
||
--repository "$TOKEN_ID@$REPO_HOST:$DS" >/dev/null
|
||
}
|
||
|
||
# R-1295 (v1.4.0): re-check the SAME token for up to SELF_CHECK_TRIES × 0.5 s (3 × the proxy's 5-s permission cache).
|
||
wait_self_check() {
|
||
local i
|
||
for i in $(seq 1 "$SELF_CHECK_TRIES"); do
|
||
if self_check "$1" 2>/dev/null; then
|
||
[ "$i" -gt 1 ] && log "self-check passed on try $i/$SELF_CHECK_TRIES (the proxy's permission cache lagged; R-1295)"
|
||
return 0
|
||
fi
|
||
log "self-check try $i/$SELF_CHECK_TRIES refused — the proxy may still hold the old permissions (up to 5 s); R-1295"
|
||
[ "$i" -lt "$SELF_CHECK_TRIES" ] && sleep 0.5
|
||
done
|
||
return 1
|
||
}
|
||
|
||
rollback_token() {
|
||
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
||
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup \
|
||
--auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
||
}
|
||
|
||
if [ "$OP" = "provision" ]; then
|
||
if token_exists; then
|
||
err_json token_exists "token $TOKEN_ID already exists — use the reissue op (explicit re-key)"
|
||
fi
|
||
else # reissue: delete-token purges its ACLs; recreate + re-grant below
|
||
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
||
log "reissue: old token deleted (ACLs purged with it)"
|
||
fi
|
||
|
||
SECRET=$(gen_token)
|
||
[ -n "$SECRET" ] || err_json internal "token generation returned no value"
|
||
grant_both
|
||
|
||
if ! wait_self_check "$SECRET"; then
|
||
# The spike's transient-403 note: one delete+regen+re-grant retry, then rollback + fail.
|
||
log "self-check failed — regenerating once (spike transient-403 note)"
|
||
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
||
SECRET=$(gen_token)
|
||
[ -n "$SECRET" ] || { rollback_token; err_json internal "token regeneration returned no value"; }
|
||
grant_both
|
||
if ! wait_self_check "$SECRET"; then
|
||
rollback_token
|
||
err_json self_check_failed "own-namespace list as $TOKEN_ID failed twice — rolled back"
|
||
fi
|
||
fi
|
||
|
||
log "$OP ok: ns=$CID token=$TOKEN_ID (secret rides stdout only)"
|
||
jq -cn --arg tid "$TOKEN_ID" --arg sec "$SECRET" --arg fp "$PBS_FINGERPRINT" \
|
||
--arg ds "$DS" --arg ns "$CID" \
|
||
'{"status":"ok","token_id":$tid,"token_secret":$sec,"fingerprint":$fp,"datastore":$ds,"namespace":$ns}'
|