#!/bin/sh # felhom-hub-db-restore-test — restore the newest hub DB copy from ep0 with the READ-ONLY token and check it (R-173). # Runs on DooPlex as root from felhom-hub-db-restore-test.timer (Sun 04:30). Runbook Step 5. Pinned by # test_hub_db_backup.py. # # The success timestamp is written ONLY when: the newest copy on ep0 is at most MAX_AGE_H old; it restores and # decrypts; PRAGMA integrity_check is "ok"; it holds at least one host; and NO console password is stored readable # (every non-empty host_recovery.secret starts with "enc:v1:", the hub's seal, 05 §16.2). set -eu CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup} STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup} TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector} MAX_AGE_H=${FELHOM_HUBBK_RESTORE_MAX_AGE_H:-50} NOW=${FELHOM_HUBBK_NOW:-$(date +%s)} . "$CONF/env" # PBS_REPOSITORY_RESTORE, PBS_FINGERPRINT log() { echo "felhom-hub-db-restore-test: $*"; } die() { echo "felhom-hub-db-restore-test: FAILED: $*" >&2; exit 1; } umask 077 mkdir -p "$STATE"; chmod 700 "$STATE" T=$(mktemp -d "$STATE/restore.XXXXXX") trap 'find "$T" -type f -exec shred -u {} + 2>/dev/null; rm -rf "$T"' EXIT export PBS_PASSWORD_FILE="$CONF/token-restore" PBS_FINGERPRINT LIST=$(proxmox-backup-client snapshot list host/dooplex-hub --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \ || die "listing snapshots on ep0" NEWEST=$(printf '%s' "$LIST" | python3 -c ' import json, sys s = [x for x in json.load(sys.stdin) if x.get("backup-type") == "host" and x.get("backup-id") == "dooplex-hub"] if s: n = max(s, key=lambda x: x["backup-time"]) print(n["backup-time"]) ') || die "reading the snapshot list" [ -n "$NEWEST" ] || die "no hub DB copy on ep0" AGE=$((NOW - NEWEST)) [ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest copy on ep0 is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h)" SNAPSHOT="host/dooplex-hub/$(date -u -d "@$NEWEST" +%Y-%m-%dT%H:%M:%SZ)" log "restoring $SNAPSHOT" proxmox-backup-client restore "$SNAPSHOT" hubdb.pxar "$T/out" --ns operator \ --keyfile "$CONF/enc.key" --repository "$PBS_REPOSITORY_RESTORE" || die "restore of $SNAPSHOT" DB="$T/out/hub.db" [ -s "$DB" ] || die "the restored archive holds no hub.db" IC=$(sqlite3 -readonly "$DB" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true [ "$IC" = "ok" ] || die "integrity_check: $IC" HOSTS=$(sqlite3 -readonly "$DB" 'SELECT COUNT(*) FROM hosts;' 2>/dev/null) || die "cannot count hosts" [ "${HOSTS:-0}" -gt 0 ] || die "the restored copy holds no hosts" PLAIN=$(sqlite3 -readonly "$DB" "SELECT COUNT(*) FROM host_recovery WHERE COALESCE(secret,'') <> '' AND secret NOT LIKE 'enc:v1:%';" 2>/dev/null) \ || die "cannot read host_recovery" [ "$PLAIN" -eq 0 ] || die "$PLAIN console password(s) stored readable" SEALED=$(sqlite3 -readonly "$DB" "SELECT COUNT(*) FROM host_recovery WHERE secret LIKE 'enc:v1:%';") log "checked: integrity ok, $HOSTS host(s), $SEALED sealed console password(s), 0 readable" TMP="$TEXTFILE_DIR/felhom_hub_db_restore.prom.$$" { echo "# HELP felhom_hub_db_restore_test_last_success_timestamp_seconds Last successful restore test of the hub DB copy on ep0 (R-173)." echo "# TYPE felhom_hub_db_restore_test_last_success_timestamp_seconds gauge" echo "felhom_hub_db_restore_test_last_success_timestamp_seconds $(date +%s)" } > "$TMP" chmod 644 "$TMP" mv "$TMP" "$TEXTFILE_DIR/felhom_hub_db_restore.prom" log "success signal written"