admin
4cee21acf7
gates / gates (push) Successful in 5m32s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
90 lines
5.7 KiB
Bash
Executable File
90 lines
5.7 KiB
Bash
Executable File
#!/bin/sh
|
|
# felhom-dooplex-offsite-restore-test — restore the newest Gitea + secrets copy from ep0 with the READ-ONLY token and
|
|
# check it (R-232 (h)). Runs on DooPlex as root from felhom-dooplex-offsite-restore-test.timer (Sun 05:30).
|
|
# Pinned by test_dooplex_offsite.py.
|
|
#
|
|
# The success timestamp is written ONLY when: the newest copy on ep0 is at most MAX_AGE_H old; it restores and
|
|
# decrypts; every file matches MANIFEST.sha256; the repository count matches REPOS; `git fsck` passes on EVERY
|
|
# repository; `pg_restore --list` reads gitea.dump; app.ini is there; at least one secrets file is there.
|
|
set -eu
|
|
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
|
|
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
|
|
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
|
|
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
|
MAX_AGE_H=${FELHOM_DXOFF_RESTORE_MAX_AGE_H:-50}
|
|
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
|
|
. "$CONF/env" # PBS_REPOSITORY_RESTORE, PBS_FINGERPRINT
|
|
|
|
log() { echo "felhom-dooplex-offsite-restore-test: $*"; }
|
|
die() { echo "felhom-dooplex-offsite-restore-test: FAILED: $*" >&2; exit 1; }
|
|
|
|
umask 077
|
|
mkdir -p "$STATE"; chmod 700 "$STATE"
|
|
T=$(mktemp -d "$STATE/restore.XXXXXX")
|
|
trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql "$T"/out/vaultwarden/db.sqlite3 "$T"/out/vaultwarden/rsa_key.pem; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
|
|
export PBS_PASSWORD_FILE="$TOKENS/token-restore" PBS_FINGERPRINT
|
|
|
|
LIST=$(proxmox-backup-client snapshot list host/dooplex-gitea --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
|
|
|| die "listing snapshots on ep0"
|
|
NEWEST=$(printf '%s' "$LIST" | python3 -c '
|
|
import json, sys
|
|
s = [x for x in json.load(sys.stdin) if x.get("backup-type") == "host" and x.get("backup-id") == "dooplex-gitea"]
|
|
if s:
|
|
print(max(x["backup-time"] for x in s))
|
|
') || die "reading the snapshot list"
|
|
[ -n "$NEWEST" ] || die "no Gitea copy on ep0"
|
|
AGE=$((NOW - NEWEST))
|
|
[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest copy on ep0 is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h)"
|
|
SNAPSHOT="host/dooplex-gitea/$(date -u -d "@$NEWEST" +%Y-%m-%dT%H:%M:%SZ)"
|
|
log "restoring $SNAPSHOT"
|
|
|
|
proxmox-backup-client restore "$SNAPSHOT" dooplex.pxar "$T/out" --ns operator \
|
|
--keyfile "$CONF/enc.key" --repository "$PBS_REPOSITORY_RESTORE" || die "restore of $SNAPSHOT"
|
|
O="$T/out"
|
|
[ -s "$O/MANIFEST.sha256" ] || die "the copy holds no MANIFEST.sha256"
|
|
(cd "$O" && sha256sum -c MANIFEST.sha256 >/dev/null) || die "a file does not match MANIFEST.sha256"
|
|
FILES=$(wc -l < "$O/MANIFEST.sha256" | tr -d ' ')
|
|
|
|
WANT=$(cat "$O/REPOS" 2>/dev/null) || die "the copy holds no REPOS count"
|
|
REPOS=$(find "$O/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | sort)
|
|
GOT=$(printf '%s\n' "$REPOS" | grep -c '\.git$') || GOT=0
|
|
[ "$GOT" -gt 0 ] && [ "$GOT" = "$WANT" ] || die "the copy holds $GOT repositories, REPOS says $WANT"
|
|
# The repositories' own `config` files came from the Gitea pod — untrusted input to a root git. So git never reads
|
|
# them: each repository is checked through a fresh scratch repository with a known config, holding a copy of its
|
|
# HEAD and refs, with the copy's objects as its object store. No system or global config either.
|
|
export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null
|
|
for r in $REPOS; do
|
|
S="$T/fsck.git"; rm -rf "$S"
|
|
git init -q --bare "$S" || die "git init for the fsck scratch repository"
|
|
cp "$r/HEAD" "$S/HEAD"; [ -f "$r/packed-refs" ] && cp "$r/packed-refs" "$S/packed-refs"
|
|
[ -d "$r/refs" ] && cp -R "$r/refs/." "$S/refs/"
|
|
GIT_OBJECT_DIRECTORY="$r/objects" git --git-dir="$S" -c core.hooksPath=/dev/null -c core.fsmonitor=false \
|
|
fsck --no-progress --no-dangling >/dev/null 2>"$T/fsck.err" \
|
|
|| die "git fsck ${r#"$O"/gitea/git/repositories/}: $(head -n 3 "$T/fsck.err")"
|
|
done
|
|
rm -rf "$T/fsck.git"
|
|
[ -s "$O/gitea/gitea/conf/app.ini" ] || die "app.ini missing"
|
|
pg_restore --list "$O/db/gitea.dump" >/dev/null || die "pg_restore cannot read gitea.dump"
|
|
ls "$O"/secrets/*.gpg >/dev/null 2>&1 || die "no secrets file in the copy"
|
|
# Vaultwarden (R-923): rows, never contents. Copies made before 2026-10-09 midday have no vaultwarden/ — refused, since
|
|
# the newest copy is the one tested and every copy since then carries it.
|
|
VDB="$O/vaultwarden/db.sqlite3"
|
|
[ -s "$VDB" ] || die "no Vaultwarden database in the copy"
|
|
VIC=$(sqlite3 -readonly "$VDB" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
|
|
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
|
|
VUSERS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
|
|
VITEMS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM ciphers;' 2>/dev/null) || die "cannot count Vaultwarden items"
|
|
[ "${VUSERS:-0}" -gt 0 ] && [ "$VUSERS" = "$(cat "$O/vaultwarden/USERS" 2>/dev/null)" ] || die "Vaultwarden users: $VUSERS, USERS says $(cat "$O/vaultwarden/USERS" 2>/dev/null)"
|
|
[ "${VITEMS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no item"
|
|
log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s), Vaultwarden $VUSERS user(s) / $VITEMS item(s)"
|
|
|
|
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom.$$"
|
|
{
|
|
echo "# HELP felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds Last successful restore test of the Gitea + secrets copy on ep0 (R-232)."
|
|
echo "# TYPE felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds gauge"
|
|
echo "felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds $(date +%s)"
|
|
} > "$TMP"
|
|
chmod 644 "$TMP"
|
|
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom"
|
|
log "success signal written"
|