admin
cea8502f0b
gates / gates (push) Successful in 32s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
66 lines
3.8 KiB
Bash
Executable File
66 lines
3.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# felhom-hub-db-backup — push the hub's newest nightly snapshot to ep0's PBS, encrypted (R-173, decision A).
|
|
# Runs on DooPlex as root from felhom-hub-db-backup.timer (02:30; the hub writes the snapshot at 02:00).
|
|
# Runbook: documentation/runbooks/RUNBOOK-hub-db-offsite-backup.md Step 4. Pinned by test_hub_db_backup.py.
|
|
#
|
|
# Refuses to push — and so never writes the success signal — when: no snapshot exists; the newest is older than
|
|
# MAX_AGE_H (the hub stopped snapshotting: pushing yesterday's copy again would read as success); the copied bytes
|
|
# differ in size from the pod's file; PRAGMA integrity_check is not "ok"; the copy holds no hosts. The success
|
|
# timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
|
|
set -eu
|
|
CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup}
|
|
STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup}
|
|
TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
|
MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-26}
|
|
NOW=${FELHOM_HUBBK_NOW:-$(date +%s)}
|
|
. "$CONF/env" # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
|
|
|
|
log() { echo "felhom-hub-db-backup: $*"; }
|
|
die() { echo "felhom-hub-db-backup: FAILED: $*" >&2; exit 1; }
|
|
|
|
umask 077
|
|
STAGE="$STATE/stage"
|
|
mkdir -p "$STAGE"; chmod 700 "$STATE" "$STAGE"
|
|
rm -f "$STAGE"/*
|
|
trap 'if [ -f "$STAGE/hub.db" ]; then shred -u "$STAGE/hub.db" 2>/dev/null || rm -f "$STAGE/hub.db"; fi' EXIT
|
|
|
|
SNAP=$(kubectl -n felhom-system exec deploy/hub -- sh -c 'ls -1 /data/snapshots/hub-*.db 2>/dev/null | tail -n 1') || die "listing snapshots in the hub pod"
|
|
[ -n "$SNAP" ] || die "no snapshot in the hub pod's /data/snapshots"
|
|
NAME=${SNAP##*/}
|
|
STAMP=${NAME#hub-}; STAMP=${STAMP%.db} # 20261005T020000Z
|
|
case "$STAMP" in [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z) ;; *) die "unexpected snapshot name $NAME" ;; esac
|
|
ISO=$(echo "$STAMP" | sed -E 's/^(....)(..)(..)T(..)(..)(..)Z$/\1-\2-\3T\4:\5:\6Z/')
|
|
SNAP_EPOCH=$(date -u -d "$ISO" +%s) || die "cannot parse snapshot time $ISO"
|
|
AGE=$((NOW - SNAP_EPOCH))
|
|
[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest snapshot $NAME is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h) — the hub stopped snapshotting"
|
|
log "snapshot $NAME, $((AGE / 60)) min old"
|
|
|
|
WANT=$(kubectl -n felhom-system exec deploy/hub -- sh -c "wc -c < '$SNAP'" | tr -d ' \r\n') || die "sizing $NAME"
|
|
kubectl -n felhom-system exec deploy/hub -- cat "$SNAP" > "$STAGE/hub.db" || die "copying $NAME out of the pod"
|
|
GOT=$(wc -c < "$STAGE/hub.db" | tr -d ' ')
|
|
[ "$GOT" = "$WANT" ] || die "copy is $GOT bytes, the pod's file is $WANT"
|
|
|
|
IC=$(sqlite3 -readonly "$STAGE/hub.db" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
|
|
[ "$IC" = "ok" ] || die "integrity_check: $IC"
|
|
HOSTS=$(sqlite3 -readonly "$STAGE/hub.db" 'SELECT COUNT(*) FROM hosts;' 2>/dev/null) || die "cannot count hosts"
|
|
[ "${HOSTS:-0}" -gt 0 ] || die "the copy holds no hosts"
|
|
log "checked: $GOT bytes, integrity ok, $HOSTS host(s)"
|
|
|
|
START=$(date +%s)
|
|
PBS_PASSWORD_FILE="$CONF/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
|
|
proxmox-backup-client backup hubdb.pxar:"$STAGE" --ns operator --backup-type host --backup-id dooplex-hub \
|
|
--keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
|
|
|| die "proxmox-backup-client backup"
|
|
log "pushed $NAME to ep0 (ns operator) in $(( $(date +%s) - START )) s"
|
|
|
|
TMP="$TEXTFILE_DIR/felhom_hub_db_backup.prom.$$"
|
|
{
|
|
echo "# HELP felhom_hub_db_backup_last_success_timestamp_seconds Last successful push of the hub DB snapshot to ep0 (R-173)."
|
|
echo "# TYPE felhom_hub_db_backup_last_success_timestamp_seconds gauge"
|
|
echo "felhom_hub_db_backup_last_success_timestamp_seconds $(date +%s)"
|
|
echo "felhom_hub_db_backup_last_success_bytes $GOT"
|
|
} > "$TMP"
|
|
chmod 644 "$TMP"
|
|
mv "$TMP" "$TEXTFILE_DIR/felhom_hub_db_backup.prom"
|
|
log "success signal written"
|