admin
4cee21acf7
gates / gates (push) Successful in 5m32s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
141 lines
9.0 KiB
Bash
Executable File
141 lines
9.0 KiB
Bash
Executable File
#!/bin/sh
|
|
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config), Vaultwarden (R-923) and DooPlex's nightly secrets export to
|
|
# ep0's PBS, encrypted on DooPlex (R-232 (b)). Runs on DooPlex as root from felhom-dooplex-offsite.timer (00:20).
|
|
# Plan: documentation/audits/dooplex-survival-2026-10-09/PLAN.md. Restore: documentation/runbooks/gitea-restore.md.
|
|
# Pinned by test_dooplex_offsite.py.
|
|
#
|
|
# Order is the consistency argument (PLAN.md): the database dump is taken FIRST (the newest complete 6-hourly dump,
|
|
# PostgreSQL's own snapshot), the repositories AFTER it, so every commit the database names is in the copy.
|
|
#
|
|
# Refuses to push — and so never writes the success signal — when: no complete dump exists or the newest is older than
|
|
# DUMP_MAX_AGE_H; the dump is empty; the file copy from the pod fails twice; the copy holds no repository or fewer
|
|
# repositories than the pod lists; app.ini is missing; no secrets export exists or it is older than SECRETS_MAX_AGE_H.
|
|
# The success timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
|
|
set -eu
|
|
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
|
|
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
|
|
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
|
|
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
|
DUMPS=${FELHOM_DXOFF_DUMPS:-/mnt/5_hdd/backup/postgresql/dumps}
|
|
SECRETS=${FELHOM_DXOFF_SECRETS:-/mnt/5_hdd/backup/secrets/exports}
|
|
DUMP_MAX_AGE_H=${FELHOM_DXOFF_DUMP_MAX_AGE_H:-7}
|
|
SECRETS_MAX_AGE_H=${FELHOM_DXOFF_SECRETS_MAX_AGE_H:-30}
|
|
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
|
|
RETRY_SLEEP=${FELHOM_DXOFF_RETRY_SLEEP:-30}
|
|
. "$CONF/env" # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
|
|
|
|
log() { echo "felhom-dooplex-offsite: $*"; }
|
|
die() { echo "felhom-dooplex-offsite: FAILED: $*" >&2; exit 1; }
|
|
K() { kubectl -n gitea-system exec deploy/gitea -c gitea -- "$@"; }
|
|
V() { kubectl -n vaultwarden-system exec deploy/vaultwarden -- "$@"; }
|
|
|
|
umask 077
|
|
STAGE="$STATE/stage"
|
|
mkdir -p "$STATE"; chmod 700 "$STATE"
|
|
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets" "$STAGE/root/vaultwarden"
|
|
cleanup() {
|
|
for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql" \
|
|
"$STAGE/root/vaultwarden/db.sqlite3" "$STAGE/root/vaultwarden/rsa_key.pem"; do
|
|
[ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
|
|
done
|
|
rm -rf "$STAGE"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# 1. the database — the newest COMPLETE dump (its folder carries SUCCESS), taken before the files
|
|
DUMPDIR=""
|
|
for d in $(ls -1d "$DUMPS"/[0-9]*-[0-9]* 2>/dev/null | sort -r); do
|
|
if [ -f "$d/SUCCESS" ] && [ -s "$d/gitea.dump" ]; then DUMPDIR=$d; break; fi
|
|
done
|
|
[ -n "$DUMPDIR" ] || die "no complete gitea.dump under $DUMPS"
|
|
DAGE=$((NOW - $(stat -c %Y "$DUMPDIR/SUCCESS")))
|
|
[ "$DAGE" -le $((DUMP_MAX_AGE_H * 3600)) ] || die "newest complete dump ${DUMPDIR##*/} is $((DAGE / 3600)) h old (limit ${DUMP_MAX_AGE_H} h) — the dump CronJob stopped"
|
|
cp "$DUMPDIR/gitea.dump" "$STAGE/root/db/gitea.dump"
|
|
[ -f "$DUMPDIR/globals.sql" ] && cp "$DUMPDIR/globals.sql" "$STAGE/root/db/globals.sql"
|
|
echo "${DUMPDIR##*/}" > "$STAGE/root/db/DUMP-FOLDER"
|
|
log "database: ${DUMPDIR##*/}, $(wc -c < "$STAGE/root/db/gitea.dump" | tr -d ' ') bytes, $((DAGE / 60)) min old"
|
|
|
|
# 2. the files — after the dump. Not the registry (packages: rebuilt from the code), logs, indexers, queues, tmp.
|
|
PATHS="git/repositories git/lfs gitea/conf/app.ini gitea/attachments gitea/avatars gitea/repo-avatars gitea/jwt"
|
|
n=0
|
|
until K tar -cf - -C /data $PATHS > "$STAGE/files.tar"; do
|
|
n=$((n + 1)); [ "$n" -lt 2 ] || die "copying Gitea's files out of the pod failed twice"
|
|
log "file copy failed once (a file moved under a push?) — retrying in ${RETRY_SLEEP} s"; sleep "$RETRY_SLEEP"
|
|
done
|
|
# The pod's archive is untrusted input to a root process: refuse any symlink or hardlink in it (a bare repository holds
|
|
# none), and extract without the pod's owners.
|
|
LINKS=$(tar -tvf "$STAGE/files.tar" | grep -c '^[lh]') || LINKS=0
|
|
[ "$LINKS" -eq 0 ] || die "the pod's archive holds $LINKS link(s) — refused"
|
|
tar -xof "$STAGE/files.tar" -C "$STAGE/root/gitea" || die "unpacking the file copy"
|
|
rm -f "$STAGE/files.tar"
|
|
[ -s "$STAGE/root/gitea/gitea/conf/app.ini" ] && [ ! -L "$STAGE/root/gitea/gitea/conf/app.ini" ] || die "app.ini missing from the copy"
|
|
LISTING=$(K find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git') || die "listing repositories in the pod"
|
|
WANT=$(printf '%s\n' "$LISTING" | grep -c '\.git$') || WANT=0
|
|
GOT=$(find "$STAGE/root/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | wc -l | tr -d ' ')
|
|
[ "$GOT" -gt 0 ] || die "the copy holds no repository"
|
|
[ "$GOT" -ge "$WANT" ] || die "the copy holds $GOT repositories, the pod lists $WANT"
|
|
log "files: $GOT repositories, $(du -sm "$STAGE/root/gitea" | cut -f1) MB"
|
|
|
|
# 3. the secrets — the newest night's GPG files (already encrypted with DooPlex's restic passphrase)
|
|
NEWEST=$(ls -1 "$SECRETS"/secrets-*.yaml.gpg 2>/dev/null | sort | tail -n 1)
|
|
[ -n "$NEWEST" ] || die "no secrets export under $SECRETS"
|
|
STAMP=${NEWEST##*/secrets-}; STAMP=${STAMP%.yaml.gpg}
|
|
SAGE=$((NOW - $(stat -c %Y "$NEWEST")))
|
|
[ "$SAGE" -le $((SECRETS_MAX_AGE_H * 3600)) ] || die "newest secrets export is $((SAGE / 3600)) h old (limit ${SECRETS_MAX_AGE_H} h)"
|
|
cp "$SECRETS"/*-"$STAMP".*gpg "$STAGE/root/secrets/"
|
|
log "secrets: $(ls "$STAGE/root/secrets" | wc -l | tr -d ' ') file(s) of $STAMP"
|
|
|
|
# 3b. the password manager (Vaultwarden, R-923) — its own `vaultwarden backup` (SQLite VACUUM INTO, consistent while it
|
|
# runs) writes ONE file into /data; it is copied out and that one file removed. Plus rsa_key.pem and, when they exist,
|
|
# attachments/, sends/, config.json (Vaultwarden's backup guidance). The vault items are encrypted under each user's
|
|
# master password; this job never opens them. Refuses on: the backup command failing, an unexpected file name, a
|
|
# failed integrity_check, no user.
|
|
VOUT=$(V /vaultwarden backup 2>&1) || die "vaultwarden backup: $(printf '%s' "$VOUT" | tail -n 1)"
|
|
VFILE=$(printf '%s\n' "$VOUT" | sed -n "s#^Backup to '\(.*\)' was successful.*#\1#p" | tail -n 1)
|
|
case "$VFILE" in data/db_[0-9]*_[0-9]*.sqlite3) ;; *) die "vaultwarden backup: unexpected output (no backup file named)" ;; esac
|
|
VPATH="/$VFILE"
|
|
VRC=0; V cat "$VPATH" > "$STAGE/root/vaultwarden/db.sqlite3" || VRC=$?
|
|
V rm -f "$VPATH" || log "WARNING: could not remove $VPATH from Vaultwarden's /data"
|
|
[ "$VRC" -eq 0 ] || die "copying $VPATH out of the Vaultwarden pod"
|
|
VLIST=$(V sh -c 'cd /data && for p in rsa_key.pem rsa_key.pub.pem config.json attachments sends; do if [ -e "$p" ]; then echo "$p"; fi; done') \
|
|
|| die "listing Vaultwarden's files"
|
|
[ -n "$VLIST" ] && { V tar -cf - -C /data $VLIST > "$STAGE/vw.tar" || die "copying Vaultwarden's files"; }
|
|
if [ -s "$STAGE/vw.tar" ]; then
|
|
LINKS=$(tar -tvf "$STAGE/vw.tar" | grep -c '^[lh]') || LINKS=0
|
|
[ "$LINKS" -eq 0 ] || die "Vaultwarden's archive holds $LINKS link(s) — refused"
|
|
tar -xof "$STAGE/vw.tar" -C "$STAGE/root/vaultwarden" || die "unpacking Vaultwarden's files"
|
|
rm -f "$STAGE/vw.tar"
|
|
fi
|
|
VIC=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
|
|
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
|
|
VUSERS=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
|
|
[ "${VUSERS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no user"
|
|
echo "$VUSERS" > "$STAGE/root/vaultwarden/USERS"
|
|
log "vaultwarden: integrity ok, $VUSERS user(s), files: db.sqlite3 $(echo $VLIST)"
|
|
|
|
# 4. the manifest the restore test checks, then the push
|
|
echo "$GOT" > "$STAGE/root/REPOS"
|
|
(cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \
|
|
|| die "writing the manifest"
|
|
[ "$(wc -l < "$STAGE/root/MANIFEST.sha256")" -gt "$GOT" ] || die "the manifest is short"
|
|
START=$(date +%s)
|
|
PBS_PASSWORD_FILE="$TOKENS/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
|
|
proxmox-backup-client backup dooplex.pxar:"$STAGE/root" --ns operator --backup-type host --backup-id dooplex-gitea \
|
|
--keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
|
|
|| die "proxmox-backup-client backup"
|
|
BYTES=$(du -sb "$STAGE/root" | cut -f1)
|
|
log "pushed to ep0 (ns operator, host/dooplex-gitea) in $(( $(date +%s) - START )) s"
|
|
|
|
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite.prom.$$"
|
|
{
|
|
echo "# HELP felhom_dooplex_offsite_last_success_timestamp_seconds Last successful push of Gitea + DooPlex secrets to ep0 (R-232)."
|
|
echo "# TYPE felhom_dooplex_offsite_last_success_timestamp_seconds gauge"
|
|
echo "felhom_dooplex_offsite_last_success_timestamp_seconds $(date +%s)"
|
|
echo "felhom_dooplex_offsite_last_success_bytes $BYTES"
|
|
echo "felhom_dooplex_offsite_last_success_repositories $GOT"
|
|
echo "felhom_dooplex_offsite_last_success_vaultwarden_users $VUSERS"
|
|
} > "$TMP"
|
|
chmod 644 "$TMP"
|
|
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite.prom"
|
|
log "success signal written"
|