admin
4009401f46
One operator action returns a customer to pre-first-install: all operational state dies (offsite repo, PBS namespace+backups, DR recipe, one-time secret, claim state, retained escrow custody); identity + basic config + provenance + events survive. Sits between host delete and customer Delete. - store/customer_reset.go: customer_resets journal, live inventory, ack-gated purge (never touches identity/provenance/events), DeleteClaim. - claim.ResetToUnclaimed: delete claim row -> fresh code next onboarding. - offsite.Deprovision (idempotent) + OffsiteIdentifier + ClearProvisionedDescriptor. - tenantsync.Deprovision + felhom-tenantsync.sh deprovision op (destroys ns + backup groups + token; shared user untouched; idempotent). - web/customer_reset.go: GET reset -> inventory JSON; POST -> orchestration (external teardown FIRST, DB purge LAST; refuse-while-hosts; typed-id + separate escrow ack). Amber RESET card distinct from red Danger-zone Delete. - Red-proofs: ack-gate + partial-failure resumability (both proven red); store ack-gating + journal round-trip; offsite idempotency + descriptor clear; RESET-card render. Green: build + vet + test.
194 lines
10 KiB
Bash
194 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
# felhom-tenantsync v1.1.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier
|
|
# SLICE 1; spike SPIKE-pbs-tier-provisioning-2026-07-10 §3).
|
|
#
|
|
# Runs as the SSH forced command for the hub's SECOND `felhom-peersync` key (via sudo — its own
|
|
# single sudoers line; the peersync script/key are untouched: one script, one job). JSON on stdin,
|
|
# JSON on stdout. Ops:
|
|
#
|
|
# {"op":"provision","customer_id":"<id>"} → ensure namespace <id> (idempotent) → CREATE token
|
|
# felhom@pbs!<id> (an EXISTING token is a hard error, code "token_exists" — re-issue is the
|
|
# explicit path) → dual-grant DatastoreBackup on /datastore/felhom-offsite/<id> to BOTH the
|
|
# user and the token (PBS privsep = intersection) → self-check: list the namespace AS the
|
|
# new token (one regen retry per the spike's transient-403 note; still failing → rollback)
|
|
# → {"status":"ok","token_id","token_secret","fingerprint","datastore","namespace"}
|
|
# {"op":"reissue","customer_id":"<id>"} → delete-token (its ACLs purge with it — spike) →
|
|
# recreate → re-grant BOTH → self-check → same ok-shape with the FRESH secret.
|
|
# {"op":"deprovision","customer_id":"<id>"} → the customer-RESET teardown (v0.61.0, hub-side
|
|
# ack-gated). Delete the token (ACLs purge with it) → delete the residual namespace ACLs →
|
|
# DESTROY the namespace AND all its backup groups (`namespace delete --delete-groups true`).
|
|
# IDEMPOTENT: a missing token / missing namespace is success, not an error (a re-run after a
|
|
# partial reset converges). The shared felhom@pbs USER is NEVER touched (other tenants ride it).
|
|
# → {"status":"ok","namespace","datastore","deleted":<bool ns existed>}.
|
|
# This is the DELIBERATE, gated data-destruction the slice-1 note reserved — the operator RESET
|
|
# confirm (typed customer-id + separate escrow-custody ack) is the human decision it demanded.
|
|
# {"op":"fingerprint"} → {"status":"ok","fingerprint":"<PBS cert sha256>"}
|
|
#
|
|
# Secret hygiene (load-bearing):
|
|
# - The token secret exists ONLY in memory and in the final stdout JSON — never a file, never
|
|
# stderr (the hub embeds remote stderr in error logs), never an argument.
|
|
# - The transient root@pam admin token (namespace ops are client-side) is held in memory and
|
|
# deleted on EVERY exit path (trap). A leftover from a crashed run is deleted at entry.
|
|
# - All tool stdout is redirected to stderr — the response JSON is the ONLY stdout bytes.
|
|
#
|
|
# Ordering facts this script encodes (all live-proven in the spike — do not "simplify"):
|
|
# - the token must exist BEFORE its ACL grant (PBS validates the auth-id);
|
|
# - user delete-token PURGES the token's ACLs → re-issue must re-grant;
|
|
# - `user generate-token` has no --output-format → the "value" line is sed-parsed;
|
|
# - proxmox-backup-client needs PBS_FINGERPRINT even against localhost.
|
|
set -euo pipefail
|
|
|
|
DS=felhom-offsite
|
|
PBS_USER=felhom@pbs
|
|
ADMIN_TOKEN_NAME=tenantsync-admin
|
|
REPO_HOST=localhost
|
|
|
|
err_json() { # code, message → error JSON on stdout, exit 1
|
|
printf '{"status":"error","code":"%s","error":"%s"}\n' "$1" "$2"
|
|
exit 1
|
|
}
|
|
|
|
command -v jq >/dev/null || err_json internal "jq is required"
|
|
command -v proxmox-backup-manager >/dev/null || err_json internal "proxmox-backup-manager is required"
|
|
command -v proxmox-backup-client >/dev/null || err_json internal "proxmox-backup-client is required"
|
|
|
|
# 1. Read stdin capped at 64 KiB; validate the envelope BEFORE touching anything.
|
|
payload=$(head -c 65536)
|
|
[ -n "$payload" ] || err_json bad_request "empty payload"
|
|
jq -e 'type == "object" and (.op | type) == "string"' >/dev/null 2>&1 <<<"$payload" \
|
|
|| err_json bad_request "payload must be a JSON object with a string op"
|
|
OP=$(jq -r '.op' <<<"$payload")
|
|
|
|
export PBS_FINGERPRINT
|
|
PBS_FINGERPRINT=$(proxmox-backup-manager cert info | awk '/Fingerprint/{print $3}' | head -1)
|
|
[ -n "$PBS_FINGERPRINT" ] || err_json internal "could not read the PBS cert fingerprint"
|
|
|
|
if [ "$OP" = "fingerprint" ]; then
|
|
printf '{"status":"ok","fingerprint":"%s"}\n' "$PBS_FINGERPRINT"
|
|
exit 0
|
|
fi
|
|
|
|
case "$OP" in provision|reissue|deprovision) ;; *) err_json bad_request "unknown op" ;; esac
|
|
|
|
# customer_id → the namespace AND the token name. Conservative charset (PBS ns + token grammar,
|
|
# no leading dash/dot so it can never parse as an option).
|
|
CID=$(jq -r '.customer_id // ""' <<<"$payload")
|
|
[[ "$CID" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}$ ]] \
|
|
|| err_json bad_request "customer_id must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}\$"
|
|
TOKEN_ID="$PBS_USER!$CID"
|
|
|
|
log() { echo "felhom-tenantsync: $*" >&2; }
|
|
|
|
# 2. Transient admin token for the client-side namespace ops. Deleted on every exit; a leftover
|
|
# from a crashed run is cleared first (generate-token fails on an existing name).
|
|
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
|
ADM=$(proxmox-backup-manager user generate-token root@pam "$ADMIN_TOKEN_NAME" \
|
|
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p')
|
|
[ -n "$ADM" ] || err_json internal "admin token generation failed"
|
|
cleanup_admin() {
|
|
proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
|
--auth-id "root@pam!$ADMIN_TOKEN_NAME" --delete >&2 2>/dev/null || true
|
|
}
|
|
trap cleanup_admin EXIT
|
|
proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \
|
|
--auth-id "root@pam!$ADMIN_TOKEN_NAME" >&2
|
|
|
|
ADMIN_REPO="root@pam!$ADMIN_TOKEN_NAME@$REPO_HOST:$DS"
|
|
|
|
# 2b. deprovision (v0.61.0 customer-RESET teardown): destroy this ONE tenant's token + namespace +
|
|
# backup groups. Every step is idempotent (missing = already gone = ok). The shared felhom@pbs
|
|
# user survives (co-tenants). Returns before the provision/reissue create-path below.
|
|
if [ "$OP" = "deprovision" ]; then
|
|
# token (its ACLs purge with it — spike); ignore "no such token".
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
# residual namespace ACLs (belt-and-suspenders — the user grant is not token-scoped).
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" --delete >&2 2>/dev/null || true
|
|
ns_existed=false
|
|
if PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
|
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
|
ns_existed=true
|
|
# --delete-groups true destroys every backup group under the namespace (the deliberate data kill).
|
|
PBS_PASSWORD="$ADM" proxmox-backup-client namespace delete "$CID" --repository "$ADMIN_REPO" --delete-groups true >&2
|
|
log "deprovision: namespace $CID destroyed (all backup groups deleted)"
|
|
else
|
|
log "deprovision: namespace $CID absent — already gone"
|
|
fi
|
|
jq -cn --arg ns "$CID" --arg ds "$DS" --argjson del "$ns_existed" \
|
|
'{"status":"ok","namespace":$ns,"datastore":$ds,"deleted":$del}'
|
|
exit 0
|
|
fi
|
|
|
|
# 3. Ensure the shared user + the namespace (both idempotent).
|
|
if ! proxmox-backup-manager user list --output-format json | jq -e --arg u "$PBS_USER" \
|
|
'any(.[]; .userid == $u)' >/dev/null; then
|
|
proxmox-backup-manager user create "$PBS_USER" --comment 'offsite tenancy' >&2
|
|
log "created user $PBS_USER"
|
|
fi
|
|
# NOTE: the CLIENT wraps json output as {"data":[...]} (live-proven on ep0); the MANAGER commands
|
|
# return bare arrays. `(.data // .)` handles both.
|
|
if ! PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \
|
|
--output-format json | jq -e --arg ns "$CID" '(.data // .) | any(.[]; .ns == $ns)' >/dev/null; then
|
|
PBS_PASSWORD="$ADM" proxmox-backup-client namespace create "$CID" --repository "$ADMIN_REPO" >&2
|
|
log "created namespace $CID"
|
|
fi
|
|
|
|
token_exists() {
|
|
proxmox-backup-manager user list-tokens "$PBS_USER" --output-format json \
|
|
| jq -e --arg t "$TOKEN_ID" 'any(.[]; .tokenid == $t)' >/dev/null
|
|
}
|
|
|
|
# gen_token → SECRET on stdout of this function only (command substitution), nothing persisted.
|
|
gen_token() {
|
|
proxmox-backup-manager user generate-token "$PBS_USER" "$CID" \
|
|
| sed -n 's/.*"value": "\([^"]*\)".*/\1/p'
|
|
}
|
|
|
|
grant_both() { # dual-grant on the NAMESPACE ACL path (never /ns/<ns> — §4a gotcha)
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" >&2
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" >&2
|
|
}
|
|
|
|
self_check() { # own-namespace list AS the new token; secret via env, never argv/stderr
|
|
PBS_PASSWORD="$1" proxmox-backup-client snapshot list --ns "$CID" \
|
|
--repository "$TOKEN_ID@$REPO_HOST:$DS" >/dev/null
|
|
}
|
|
|
|
rollback_token() {
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup \
|
|
--auth-id "$PBS_USER" --delete >&2 2>/dev/null || true
|
|
}
|
|
|
|
if [ "$OP" = "provision" ]; then
|
|
if token_exists; then
|
|
err_json token_exists "token $TOKEN_ID already exists — use the reissue op (explicit re-key)"
|
|
fi
|
|
else # reissue: delete-token purges its ACLs; recreate + re-grant below
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
log "reissue: old token deleted (ACLs purged with it)"
|
|
fi
|
|
|
|
SECRET=$(gen_token)
|
|
[ -n "$SECRET" ] || err_json internal "token generation returned no value"
|
|
grant_both
|
|
|
|
if ! self_check "$SECRET"; then
|
|
# The spike's transient-403 note: one delete+regen+re-grant retry, then rollback + fail.
|
|
log "self-check failed — regenerating once (spike transient-403 note)"
|
|
proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true
|
|
SECRET=$(gen_token)
|
|
[ -n "$SECRET" ] || { rollback_token; err_json internal "token regeneration returned no value"; }
|
|
grant_both
|
|
if ! self_check "$SECRET"; then
|
|
rollback_token
|
|
err_json self_check_failed "own-namespace list as $TOKEN_ID failed twice — rolled back"
|
|
fi
|
|
fi
|
|
|
|
log "$OP ok: ns=$CID token=$TOKEN_ID (secret rides stdout only)"
|
|
jq -cn --arg tid "$TOKEN_ID" --arg sec "$SECRET" --arg fp "$PBS_FINGERPRINT" \
|
|
--arg ds "$DS" --arg ns "$CID" \
|
|
'{"status":"ok","token_id":$tid,"token_secret":$sec,"fingerprint":$fp,"datastore":$ds,"namespace":$ns}'
|