261 lines
15 KiB
Bash
261 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
# felhom-tenantsync v1.3.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier
|
|
# SLICE 1; spike SPIKE-pbs-tier-provisioning-2026-07-10 §3).
|
|
#
|
|
# v1.3.0 (2026-10-19, R-1165): a household's namespace gets its PRUNE JOB with it. provision/reissue ensure
|
|
# `prune-<id>` (keep-weekly 2, 03:30, ns-scoped, max-depth 0 — the shape of the demo-hp/demo-felhom jobs, STATUS
|
|
# D18) unless a job on felhom-offsite already covers that namespace (idempotent: a second run changes nothing);
|
|
# a failed create is logged and never fails the provision (the hub's alarm below names the namespace). deprovision
|
|
# removes the namespace's jobs before the namespace. `usage` adds `unpruned` — the top-level namespaces no job
|
|
# covers (read-only: the namespace folders + `prune-job list`), or `unpruned_checked:false` when the list cannot be
|
|
# read. Box tokens stay write-only: pruning is server-side, so a namespace without a job grows for ever.
|
|
# Pinned by scripts/tenantsync-prune-harness.sh.
|
|
#
|
|
# v1.2.0 (2026-07-17, hub R-5 PBS visibility): + read-only `usage` op — the felhom-offsite datastore's
|
|
# total/used/avail (bytes, via `df` on the datastore path). No customer_id, no admin token, NO
|
|
# mutation. Structural twin of `fingerprint`. Backward-compatible: an older hub never sends it, and a
|
|
# hub that does against an OLD script gets the existing `bad_request "unknown op"` (graceful).
|
|
#
|
|
# Runs as the SSH forced command for the hub's SECOND `felhom-peersync` key (via sudo — its own
|
|
# single sudoers line; the peersync script/key are untouched: one script, one job). JSON on stdin,
|
|
# JSON on stdout. Ops:
|
|
#
|
|
# {"op":"provision","customer_id":"<id>"} → ensure namespace <id> (idempotent) → CREATE token
|
|
# felhom@pbs!<id> (an EXISTING token is a hard error, code "token_exists" — re-issue is the
|
|
# explicit path) → dual-grant DatastoreBackup on /datastore/felhom-offsite/<id> to BOTH the
|
|
# user and the token (PBS privsep = intersection) → self-check: list the namespace AS the
|
|
# new token (one regen retry per the spike's transient-403 note; still failing → rollback)
|
|
# → {"status":"ok","token_id","token_secret","fingerprint","datastore","namespace"}
|
|
# {"op":"reissue","customer_id":"<id>"} → delete-token (its ACLs purge with it — spike) →
|
|
# recreate → re-grant BOTH → self-check → same ok-shape with the FRESH secret.
|
|
# {"op":"deprovision","customer_id":"<id>"} → the customer-RESET teardown (v0.61.0, hub-side
|
|
# ack-gated). Delete the token (ACLs purge with it) → delete the residual namespace ACLs →
|
|
# DESTROY the namespace AND all its backup groups (`namespace delete --delete-groups true`).
|
|
# IDEMPOTENT: a missing token / missing namespace is success, not an error (a re-run after a
|
|
# partial reset converges). The shared felhom@pbs USER is NEVER touched (other tenants ride it).
|
|
# → {"status":"ok","namespace","datastore","deleted":<bool ns existed>}.
|
|
# This is the DELIBERATE, gated data-destruction the slice-1 note reserved — the operator RESET
|
|
# confirm (typed customer-id + separate escrow-custody ack) is the human decision it demanded.
|
|
# {"op":"fingerprint"} → {"status":"ok","fingerprint":"<PBS cert sha256>"}
|
|
# {"op":"usage"} → {"status":"ok","total":<bytes>,"used":<bytes>,"avail":<bytes>,
|
|
# "unpruned":[<ns>…],"unpruned_checked":true}
|
|
# READ-ONLY datastore fill (df on the felhom-offsite path). No customer_id, no admin token, no mutation.
|
|
#
|
|
# Secret hygiene (load-bearing):
|
|
# - The token secret exists ONLY in memory and in the final stdout JSON — never a file, never
|
|
# stderr (the hub embeds remote stderr in error logs), never an argument.
|
|
# - The transient root@pam admin token (namespace ops are client-side) is held in memory and
|
|
# deleted on EVERY exit path (trap). A leftover from a crashed run is deleted at entry.
|
|
# - All tool stdout is redirected to stderr — the response JSON is the ONLY stdout bytes.
|
|
#
|
|
# Ordering facts this script encodes (all live-proven in the spike — do not "simplify"):
|
|
# - the token must exist BEFORE its ACL grant (PBS validates the auth-id);
|
|
# - user delete-token PURGES the token's ACLs → re-issue must re-grant;
|
|
# - `user generate-token` has no --output-format → the "value" line is sed-parsed;
|
|
# - proxmox-backup-client needs PBS_FINGERPRINT even against localhost.
|
|
set -euo pipefail
|
|
|
|
DS=felhom-offsite
|
|
PBS_USER=felhom@pbs
|
|
ADMIN_TOKEN_NAME=tenantsync-admin
|
|
REPO_HOST=localhost
|
|
|
|
err_json() { # code, message → error JSON on stdout, exit 1
|
|
printf '{"status":"error","code":"%s","error":"%s"}\n' "$1" "$2"
|
|
exit 1
|
|
}
|
|
|
|
command -v jq >/dev/null || err_json internal "jq is required"
|
|
command -v proxmox-backup-manager >/dev/null || err_json internal "proxmox-backup-manager is required"
|
|
command -v proxmox-backup-client >/dev/null || err_json internal "proxmox-backup-client is required"
|
|
|
|
# 1. Read stdin capped at 64 KiB; validate the envelope BEFORE touching anything.
|
|
payload=$(head -c 65536)
|
|
[ -n "$payload" ] || err_json bad_request "empty payload"
|
|
jq -e 'type == "object" and (.op | type) == "string"' >/dev/null 2>&1 <<<"$payload" \
|
|
|| err_json bad_request "payload must be a JSON object with a string op"
|
|
OP=$(jq -r '.op' <<<"$payload")
|
|
|
|
export PBS_FINGERPRINT
|
|
PBS_FINGERPRINT=$(proxmox-backup-manager cert info | awk '/Fingerprint/{print $3}' | head -1)
|
|
[ -n "$PBS_FINGERPRINT" ] || err_json internal "could not read the PBS cert fingerprint"
|
|
|
|
if [ "$OP" = "fingerprint" ]; then
|
|
printf '{"status":"ok","fingerprint":"%s"}\n' "$PBS_FINGERPRINT"
|
|
exit 0
|
|
fi
|
|
|
|
# usage (v1.2.0) — READ-ONLY box-level datastore fill. No customer_id, no admin token, NO mutation
|
|
# (a pure `df` on the datastore path). The hub's PBS-DR checker polls this on its 15-min throttle.
|
|
# Short-circuits BEFORE the admin-token generation below, exactly like fingerprint.
|
|
if [ "$OP" = "usage" ]; then
|
|
DS_PATH=$(proxmox-backup-manager datastore list --output-format json \
|
|
| jq -r --arg ds "$DS" '.[] | select(.name == $ds) | .path')
|
|
[ -n "$DS_PATH" ] || err_json internal "datastore $DS not found"
|
|
# df -B1: 1-byte blocks. --output pins the columns; tail -1 skips the header row.
|
|
read -r TOTAL USED AVAIL < <(df -B1 --output=size,used,avail "$DS_PATH" | tail -1)
|
|
[ -n "$TOTAL" ] && [ -n "$USED" ] && [ -n "$AVAIL" ] || err_json internal "could not read datastore usage"
|
|
# v1.3.0 (R-1165): the top-level namespaces no prune job covers. A job covers a namespace when it names it, or when
|
|
# it sits on the root (no ns) without a max-depth of 0. Read-only; a list that cannot be read says so.
|
|
NSES='[]'
|
|
if [ -d "$DS_PATH/ns" ]; then
|
|
NSES=$(find "$DS_PATH/ns" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | jq -R . | jq -cs .)
|
|
fi
|
|
if JOBS=$(proxmox-backup-manager prune-job list --output-format json 2>/dev/null) && jq -e 'type == "array"' >/dev/null 2>&1 <<<"$JOBS"; then
|
|
jq -cn --argjson t "$TOTAL" --argjson u "$USED" --argjson a "$AVAIL" --argjson ns "$NSES" --argjson jobs "$JOBS" --arg ds "$DS" '
|
|
($jobs | map(select(.store == $ds))) as $j
|
|
| ($j | any(((.ns // "") == "") and ((."max-depth" // 7) > 0))) as $root
|
|
| {"status":"ok","total":$t,"used":$u,"avail":$a,"unpruned_checked":true,
|
|
"unpruned": (if $root then [] else [$ns[] as $n | select(($j | any((.ns // "") == $n)) | not) | $n] | sort end)}'
|
|
else
|
|
printf '{"status":"ok","total":%s,"used":%s,"avail":%s,"unpruned_checked":false}\n' "$TOTAL" "$USED" "$AVAIL"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
case "$OP" in provision|reissue|deprovision) ;; *) err_json bad_request "unknown op" ;; esac
|
|
|
|
# customer_id → the namespace AND the token name. Conservative charset (PBS ns + token grammar,
|
|
# no leading dash/dot so it can never parse as an option).
|
|
CID=$(jq -r '.customer_id // ""' <<<"$payload")
|
|
[[ "$CID" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}$ ]] \
|
|
|| err_json bad_request "customer_id must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}\$"
|
|
TOKEN_ID="$PBS_USER!$CID"
|
|
|
|
log() { echo "felhom-tenantsync: $*" >&2; }
|
|
|
|
# 2. Transient admin token for the client-side namespace ops. Deleted on every exit; a leftover
|
|
# from a crashed run is cleared first (generate-token fails on an existing name).
|
|
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
|
ADM=$(proxmox-backup-manager user generate-token root@pam "$ADMIN_TOKEN_NAME" \
|
|
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p')
|
|
[ -n "$ADM" ] || err_json internal "admin token generation failed"
|
|
cleanup_admin() {
|
|
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
|
--auth-id "root@pam!$ADMIN_TOKEN_NAME" --delete >&2 2>/dev/null || true
|
|
}
|
|
trap cleanup_admin EXIT
|
|
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
|
--auth-id "root@pam!$ADMIN_TOKEN_NAME" >&2
|
|
|
|
ADMIN_REPO="root@pam!$ADMIN_TOKEN_NAME@$REPO_HOST:$DS"
|
|
|
|
# 2b. deprovision (v0.61.0 customer-RESET teardown): destroy this ONE tenant's token + namespace +
|
|
# backup groups. Every step is idempotent (missing = already gone = ok). The shared felhom@pbs
|
|
# user survives (co-tenants). Returns before the provision/reissue create-path below.
|
|
if [ "$OP" = "deprovision" ]; then
|
|
# token (its ACLs purge with it — spike); ignore "no such token".
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
# residual namespace ACLs (belt-and-suspenders — the user grant is not token-scoped).
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" --delete >&2 2>/dev/null || true
|
|
# v1.3.0 (R-1165): the namespace's prune job(s) go first — a job left on a deleted namespace fails every night.
|
|
for job in $(proxmox-backup-manager prune-job list --output-format json 2>/dev/null \
|
|
| jq -r --arg ns "$CID" --arg ds "$DS" '.[] | select(.store == $ds and (.ns // "") == $ns) | .id'); do
|
|
proxmox-backup-manager prune-job remove "$job" >&2 && log "deprovision: prune job $job removed"
|
|
done
|
|
ns_existed=false
|
|
if PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
|
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
|
ns_existed=true
|
|
# --delete-groups true destroys every backup group under the namespace (the deliberate data kill).
|
|
PBS_PASSWORD="$ADM" proxmox-backup-client namespace delete "$CID" --repository "$ADMIN_REPO" --delete-groups true >&2
|
|
log "deprovision: namespace $CID destroyed (all backup groups deleted)"
|
|
else
|
|
log "deprovision: namespace $CID absent — already gone"
|
|
fi
|
|
jq -cn --arg ns "$CID" --arg ds "$DS" --argjson del "$ns_existed" \
|
|
'{"status":"ok","namespace":$ns,"datastore":$ds,"deleted":$del}'
|
|
exit 0
|
|
fi
|
|
|
|
# 3. Ensure the shared user + the namespace (both idempotent).
|
|
if ! proxmox-backup-manager user list --output-format json | jq -e --arg u "$PBS_USER" \
|
|
'any(.[]; .userid == $u)' >/dev/null; then
|
|
proxmox-backup-manager user create "$PBS_USER" --comment 'offsite tenancy' >&2
|
|
log "created user $PBS_USER"
|
|
fi
|
|
# NOTE: the CLIENT wraps json output as {"data":[...]} (live-proven on ep0); the MANAGER commands
|
|
# return bare arrays. `(.data // .)` handles both.
|
|
if ! PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
|
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
|
PBS_PASSWORD="$ADM" proxmox-backup-client namespace create "$CID" --repository "$ADMIN_REPO" >&2
|
|
log "created namespace $CID"
|
|
fi
|
|
|
|
# 3b. v1.3.0 (R-1165): the namespace's prune job — keep-weekly 2 at 03:30, the D18 shape. Idempotent: any job on this
|
|
# datastore that already names the namespace counts (whatever its id). A PBS job id is at most 32 characters, so a
|
|
# long customer id gets a shortened, hashed one. A failure is logged, never fatal (the hub's usage alarm names it).
|
|
JOB_ID="prune-$CID"
|
|
if [ "${#JOB_ID}" -gt 32 ]; then
|
|
JOB_ID="prune-${CID:0:20}-$(printf '%s' "$CID" | sha256sum | cut -c1-5)"
|
|
fi
|
|
if proxmox-backup-manager prune-job list --output-format json \
|
|
| jq -e --arg ns "$CID" --arg ds "$DS" 'any(.[]; .store == $ds and (.ns // "") == $ns)' >/dev/null; then
|
|
log "prune job for namespace $CID already present"
|
|
elif proxmox-backup-manager prune-job create "$JOB_ID" --store "$DS" --ns "$CID" --max-depth 0 --keep-weekly 2 \
|
|
--schedule 03:30 --comment "R-1165: one copy per ISO week, 2 weeks, server-side (box tokens are write-only)" >&2; then
|
|
log "created prune job $JOB_ID (keep-weekly 2, 03:30) for namespace $CID"
|
|
else
|
|
log "WARNING: could not create prune job $JOB_ID for namespace $CID — its copies are not pruned until one exists"
|
|
fi
|
|
|
|
token_exists() {
|
|
proxmox-backup-manager user list-tokens "$PBS_USER" --output-format json \
|
|
| jq -e --arg t "$TOKEN_ID" 'any(.[]; .tokenid == $t)' >/dev/null
|
|
}
|
|
|
|
# gen_token → SECRET on stdout of this function only (command substitution), nothing persisted.
|
|
gen_token() {
|
|
proxmox-backup-manager user generate-token "$PBS_USER" "$CID" \
|
|
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p'
|
|
}
|
|
|
|
grant_both() { # dual-grant on the NAMESPACE ACL path (never /ns/<ns> — §4a gotcha)
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" >&2
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" >&2
|
|
}
|
|
|
|
self_check() { # own-namespace list AS the new token; secret via env, never argv/stderr
|
|
PBS_PASSWORD="$1" proxmox-backup-client snapshot list --ns "$CID" \
|
|
--repository "$TOKEN_ID@$REPO_HOST:$DS" >/dev/null
|
|
}
|
|
|
|
rollback_token() {
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup \
|
|
--auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
|
}
|
|
|
|
if [ "$OP" = "provision" ]; then
|
|
if token_exists; then
|
|
err_json token_exists "token $TOKEN_ID already exists — use the reissue op (explicit re-key)"
|
|
fi
|
|
else # reissue: delete-token purges its ACLs; recreate + re-grant below
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
log "reissue: old token deleted (ACLs purged with it)"
|
|
fi
|
|
|
|
SECRET=$(gen_token)
|
|
[ -n "$SECRET" ] || err_json internal "token generation returned no value"
|
|
grant_both
|
|
|
|
if ! self_check "$SECRET"; then
|
|
# The spike's transient-403 note: one delete+regen+re-grant retry, then rollback + fail.
|
|
log "self-check failed — regenerating once (spike transient-403 note)"
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
SECRET=$(gen_token)
|
|
[ -n "$SECRET" ] || { rollback_token; err_json internal "token regeneration returned no value"; }
|
|
grant_both
|
|
if ! self_check "$SECRET"; then
|
|
rollback_token
|
|
err_json self_check_failed "own-namespace list as $TOKEN_ID failed twice — rolled back"
|
|
fi
|
|
fi
|
|
|
|
log "$OP ok: ns=$CID token=$TOKEN_ID (secret rides stdout only)"
|
|
jq -cn --arg tid "$TOKEN_ID" --arg sec "$SECRET" --arg fp "$PBS_FINGERPRINT" \
|
|
--arg ds "$DS" --arg ns "$CID" \
|
|
'{"status":"ok","token_id":$tid,"token_secret":$sec,"fingerprint":$fp,"datastore":$ds,"namespace":$ns}'
|