admin
1707c928a9
gates / gates (push) Successful in 5m4s
Part A plan + readings, Part E read-backs (R-861 a, R-518) in audits/dooplex-survival-2026-10-09/. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
110 lines
6.6 KiB
Bash
Executable File
110 lines
6.6 KiB
Bash
Executable File
#!/bin/sh
|
|
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config) and DooPlex's nightly secrets export to
|
|
# ep0's PBS, encrypted on DooPlex (R-232 (b)). Runs on DooPlex as root from felhom-dooplex-offsite.timer (00:20).
|
|
# Plan: documentation/audits/dooplex-survival-2026-10-09/PLAN.md. Restore: documentation/runbooks/gitea-restore.md.
|
|
# Pinned by test_dooplex_offsite.py.
|
|
#
|
|
# Order is the consistency argument (PLAN.md): the database dump is taken FIRST (the newest complete 6-hourly dump,
|
|
# PostgreSQL's own snapshot), the repositories AFTER it, so every commit the database names is in the copy.
|
|
#
|
|
# Refuses to push — and so never writes the success signal — when: no complete dump exists or the newest is older than
|
|
# DUMP_MAX_AGE_H; the dump is empty; the file copy from the pod fails twice; the copy holds no repository or fewer
|
|
# repositories than the pod lists; app.ini is missing; no secrets export exists or it is older than SECRETS_MAX_AGE_H.
|
|
# The success timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
|
|
set -eu
|
|
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
|
|
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
|
|
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
|
|
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
|
DUMPS=${FELHOM_DXOFF_DUMPS:-/mnt/5_hdd/backup/postgresql/dumps}
|
|
SECRETS=${FELHOM_DXOFF_SECRETS:-/mnt/5_hdd/backup/secrets/exports}
|
|
DUMP_MAX_AGE_H=${FELHOM_DXOFF_DUMP_MAX_AGE_H:-7}
|
|
SECRETS_MAX_AGE_H=${FELHOM_DXOFF_SECRETS_MAX_AGE_H:-30}
|
|
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
|
|
RETRY_SLEEP=${FELHOM_DXOFF_RETRY_SLEEP:-30}
|
|
. "$CONF/env" # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
|
|
|
|
log() { echo "felhom-dooplex-offsite: $*"; }
|
|
die() { echo "felhom-dooplex-offsite: FAILED: $*" >&2; exit 1; }
|
|
K() { kubectl -n gitea-system exec deploy/gitea -c gitea -- "$@"; }
|
|
|
|
umask 077
|
|
STAGE="$STATE/stage"
|
|
mkdir -p "$STATE"; chmod 700 "$STATE"
|
|
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets"
|
|
cleanup() {
|
|
for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql"; do
|
|
[ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
|
|
done
|
|
rm -rf "$STAGE"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# 1. the database — the newest COMPLETE dump (its folder carries SUCCESS), taken before the files
|
|
DUMPDIR=""
|
|
for d in $(ls -1d "$DUMPS"/[0-9]*-[0-9]* 2>/dev/null | sort -r); do
|
|
if [ -f "$d/SUCCESS" ] && [ -s "$d/gitea.dump" ]; then DUMPDIR=$d; break; fi
|
|
done
|
|
[ -n "$DUMPDIR" ] || die "no complete gitea.dump under $DUMPS"
|
|
DAGE=$((NOW - $(stat -c %Y "$DUMPDIR/SUCCESS")))
|
|
[ "$DAGE" -le $((DUMP_MAX_AGE_H * 3600)) ] || die "newest complete dump ${DUMPDIR##*/} is $((DAGE / 3600)) h old (limit ${DUMP_MAX_AGE_H} h) — the dump CronJob stopped"
|
|
cp "$DUMPDIR/gitea.dump" "$STAGE/root/db/gitea.dump"
|
|
[ -f "$DUMPDIR/globals.sql" ] && cp "$DUMPDIR/globals.sql" "$STAGE/root/db/globals.sql"
|
|
echo "${DUMPDIR##*/}" > "$STAGE/root/db/DUMP-FOLDER"
|
|
log "database: ${DUMPDIR##*/}, $(wc -c < "$STAGE/root/db/gitea.dump" | tr -d ' ') bytes, $((DAGE / 60)) min old"
|
|
|
|
# 2. the files — after the dump. Not the registry (packages: rebuilt from the code), logs, indexers, queues, tmp.
|
|
PATHS="git/repositories git/lfs gitea/conf/app.ini gitea/attachments gitea/avatars gitea/repo-avatars gitea/jwt"
|
|
n=0
|
|
until K tar -cf - -C /data $PATHS > "$STAGE/files.tar"; do
|
|
n=$((n + 1)); [ "$n" -lt 2 ] || die "copying Gitea's files out of the pod failed twice"
|
|
log "file copy failed once (a file moved under a push?) — retrying in ${RETRY_SLEEP} s"; sleep "$RETRY_SLEEP"
|
|
done
|
|
# The pod's archive is untrusted input to a root process: refuse any symlink or hardlink in it (a bare repository holds
|
|
# none), and extract without the pod's owners.
|
|
LINKS=$(tar -tvf "$STAGE/files.tar" | grep -c '^[lh]') || LINKS=0
|
|
[ "$LINKS" -eq 0 ] || die "the pod's archive holds $LINKS link(s) — refused"
|
|
tar -xof "$STAGE/files.tar" -C "$STAGE/root/gitea" || die "unpacking the file copy"
|
|
rm -f "$STAGE/files.tar"
|
|
[ -s "$STAGE/root/gitea/gitea/conf/app.ini" ] && [ ! -L "$STAGE/root/gitea/gitea/conf/app.ini" ] || die "app.ini missing from the copy"
|
|
LISTING=$(K find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git') || die "listing repositories in the pod"
|
|
WANT=$(printf '%s\n' "$LISTING" | grep -c '\.git$') || WANT=0
|
|
GOT=$(find "$STAGE/root/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | wc -l | tr -d ' ')
|
|
[ "$GOT" -gt 0 ] || die "the copy holds no repository"
|
|
[ "$GOT" -ge "$WANT" ] || die "the copy holds $GOT repositories, the pod lists $WANT"
|
|
log "files: $GOT repositories, $(du -sm "$STAGE/root/gitea" | cut -f1) MB"
|
|
|
|
# 3. the secrets — the newest night's GPG files (already encrypted with DooPlex's restic passphrase)
|
|
NEWEST=$(ls -1 "$SECRETS"/secrets-*.yaml.gpg 2>/dev/null | sort | tail -n 1)
|
|
[ -n "$NEWEST" ] || die "no secrets export under $SECRETS"
|
|
STAMP=${NEWEST##*/secrets-}; STAMP=${STAMP%.yaml.gpg}
|
|
SAGE=$((NOW - $(stat -c %Y "$NEWEST")))
|
|
[ "$SAGE" -le $((SECRETS_MAX_AGE_H * 3600)) ] || die "newest secrets export is $((SAGE / 3600)) h old (limit ${SECRETS_MAX_AGE_H} h)"
|
|
cp "$SECRETS"/*-"$STAMP".*gpg "$STAGE/root/secrets/"
|
|
log "secrets: $(ls "$STAGE/root/secrets" | wc -l | tr -d ' ') file(s) of $STAMP"
|
|
|
|
# 4. the manifest the restore test checks, then the push
|
|
echo "$GOT" > "$STAGE/root/REPOS"
|
|
(cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \
|
|
|| die "writing the manifest"
|
|
[ "$(wc -l < "$STAGE/root/MANIFEST.sha256")" -gt "$GOT" ] || die "the manifest is short"
|
|
START=$(date +%s)
|
|
PBS_PASSWORD_FILE="$TOKENS/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
|
|
proxmox-backup-client backup dooplex.pxar:"$STAGE/root" --ns operator --backup-type host --backup-id dooplex-gitea \
|
|
--keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
|
|
|| die "proxmox-backup-client backup"
|
|
BYTES=$(du -sb "$STAGE/root" | cut -f1)
|
|
log "pushed to ep0 (ns operator, host/dooplex-gitea) in $(( $(date +%s) - START )) s"
|
|
|
|
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite.prom.$$"
|
|
{
|
|
echo "# HELP felhom_dooplex_offsite_last_success_timestamp_seconds Last successful push of Gitea + DooPlex secrets to ep0 (R-232)."
|
|
echo "# TYPE felhom_dooplex_offsite_last_success_timestamp_seconds gauge"
|
|
echo "felhom_dooplex_offsite_last_success_timestamp_seconds $(date +%s)"
|
|
echo "felhom_dooplex_offsite_last_success_bytes $BYTES"
|
|
echo "felhom_dooplex_offsite_last_success_repositories $GOT"
|
|
} > "$TMP"
|
|
chmod 644 "$TMP"
|
|
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite.prom"
|
|
log "success signal written"
|