dooplex-offsite: nightly encrypted copy of Gitea + DooPlex secrets to ep0, restore test, failure mail (R-232 b/h)
gates / gates (push) Successful in 5m4s
gates / gates (push) Successful in 5m4s
Part A plan + readings, Part E read-backs (R-861 a, R-518) in audits/dooplex-survival-2026-10-09/. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# felhom-backup-failmail — mail admin@ that a Felhom backup unit on DooPlex failed (R-232 (a) route, reused).
|
||||
# Called by felhom-backup-failmail@<unit>.service, which the backup units name in OnFailure=.
|
||||
# Reuses notify_failure from /opt/backup/scripts/backup-config.sh (Resend; never prints the key; never fails).
|
||||
set -u
|
||||
UNIT=${1:?unit name}
|
||||
CONFIG=${FELHOM_FAILMAIL_CONFIG:-/opt/backup/scripts/backup-config.sh}
|
||||
# shellcheck source=/dev/null
|
||||
. "$CONFIG"
|
||||
notify_failure "systemd unit ${UNIT} failed on $(hostname) — see: journalctl -u ${UNIT}"
|
||||
@@ -0,0 +1,8 @@
|
||||
# Versioned in felhom.eu/scripts/dooplex-offsite/ (R-232); installed by install.sh.
|
||||
# Named by OnFailure= in felhom-dooplex-offsite*.service and felhom-hub-db-*.service; %i is the failed unit.
|
||||
[Unit]
|
||||
Description=Felhom: mail admin@ that %i failed (R-232)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-backup-failmail %i
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
#!/bin/sh
|
||||
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config) and DooPlex's nightly secrets export to
|
||||
# ep0's PBS, encrypted on DooPlex (R-232 (b)). Runs on DooPlex as root from felhom-dooplex-offsite.timer (00:20).
|
||||
# Plan: documentation/audits/dooplex-survival-2026-10-09/PLAN.md. Restore: documentation/runbooks/gitea-restore.md.
|
||||
# Pinned by test_dooplex_offsite.py.
|
||||
#
|
||||
# Order is the consistency argument (PLAN.md): the database dump is taken FIRST (the newest complete 6-hourly dump,
|
||||
# PostgreSQL's own snapshot), the repositories AFTER it, so every commit the database names is in the copy.
|
||||
#
|
||||
# Refuses to push — and so never writes the success signal — when: no complete dump exists or the newest is older than
|
||||
# DUMP_MAX_AGE_H; the dump is empty; the file copy from the pod fails twice; the copy holds no repository or fewer
|
||||
# repositories than the pod lists; app.ini is missing; no secrets export exists or it is older than SECRETS_MAX_AGE_H.
|
||||
# The success timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
|
||||
set -eu
|
||||
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
|
||||
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
|
||||
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
|
||||
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
||||
DUMPS=${FELHOM_DXOFF_DUMPS:-/mnt/5_hdd/backup/postgresql/dumps}
|
||||
SECRETS=${FELHOM_DXOFF_SECRETS:-/mnt/5_hdd/backup/secrets/exports}
|
||||
DUMP_MAX_AGE_H=${FELHOM_DXOFF_DUMP_MAX_AGE_H:-7}
|
||||
SECRETS_MAX_AGE_H=${FELHOM_DXOFF_SECRETS_MAX_AGE_H:-30}
|
||||
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
|
||||
RETRY_SLEEP=${FELHOM_DXOFF_RETRY_SLEEP:-30}
|
||||
. "$CONF/env" # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
|
||||
|
||||
log() { echo "felhom-dooplex-offsite: $*"; }
|
||||
die() { echo "felhom-dooplex-offsite: FAILED: $*" >&2; exit 1; }
|
||||
K() { kubectl -n gitea-system exec deploy/gitea -c gitea -- "$@"; }
|
||||
|
||||
umask 077
|
||||
STAGE="$STATE/stage"
|
||||
mkdir -p "$STATE"; chmod 700 "$STATE"
|
||||
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets"
|
||||
cleanup() {
|
||||
for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql"; do
|
||||
[ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
|
||||
done
|
||||
rm -rf "$STAGE"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# 1. the database — the newest COMPLETE dump (its folder carries SUCCESS), taken before the files
|
||||
DUMPDIR=""
|
||||
for d in $(ls -1d "$DUMPS"/[0-9]*-[0-9]* 2>/dev/null | sort -r); do
|
||||
if [ -f "$d/SUCCESS" ] && [ -s "$d/gitea.dump" ]; then DUMPDIR=$d; break; fi
|
||||
done
|
||||
[ -n "$DUMPDIR" ] || die "no complete gitea.dump under $DUMPS"
|
||||
DAGE=$((NOW - $(stat -c %Y "$DUMPDIR/SUCCESS")))
|
||||
[ "$DAGE" -le $((DUMP_MAX_AGE_H * 3600)) ] || die "newest complete dump ${DUMPDIR##*/} is $((DAGE / 3600)) h old (limit ${DUMP_MAX_AGE_H} h) — the dump CronJob stopped"
|
||||
cp "$DUMPDIR/gitea.dump" "$STAGE/root/db/gitea.dump"
|
||||
[ -f "$DUMPDIR/globals.sql" ] && cp "$DUMPDIR/globals.sql" "$STAGE/root/db/globals.sql"
|
||||
echo "${DUMPDIR##*/}" > "$STAGE/root/db/DUMP-FOLDER"
|
||||
log "database: ${DUMPDIR##*/}, $(wc -c < "$STAGE/root/db/gitea.dump" | tr -d ' ') bytes, $((DAGE / 60)) min old"
|
||||
|
||||
# 2. the files — after the dump. Not the registry (packages: rebuilt from the code), logs, indexers, queues, tmp.
|
||||
PATHS="git/repositories git/lfs gitea/conf/app.ini gitea/attachments gitea/avatars gitea/repo-avatars gitea/jwt"
|
||||
n=0
|
||||
until K tar -cf - -C /data $PATHS > "$STAGE/files.tar"; do
|
||||
n=$((n + 1)); [ "$n" -lt 2 ] || die "copying Gitea's files out of the pod failed twice"
|
||||
log "file copy failed once (a file moved under a push?) — retrying in ${RETRY_SLEEP} s"; sleep "$RETRY_SLEEP"
|
||||
done
|
||||
# The pod's archive is untrusted input to a root process: refuse any symlink or hardlink in it (a bare repository holds
|
||||
# none), and extract without the pod's owners.
|
||||
LINKS=$(tar -tvf "$STAGE/files.tar" | grep -c '^[lh]') || LINKS=0
|
||||
[ "$LINKS" -eq 0 ] || die "the pod's archive holds $LINKS link(s) — refused"
|
||||
tar -xof "$STAGE/files.tar" -C "$STAGE/root/gitea" || die "unpacking the file copy"
|
||||
rm -f "$STAGE/files.tar"
|
||||
[ -s "$STAGE/root/gitea/gitea/conf/app.ini" ] && [ ! -L "$STAGE/root/gitea/gitea/conf/app.ini" ] || die "app.ini missing from the copy"
|
||||
LISTING=$(K find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git') || die "listing repositories in the pod"
|
||||
WANT=$(printf '%s\n' "$LISTING" | grep -c '\.git$') || WANT=0
|
||||
GOT=$(find "$STAGE/root/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | wc -l | tr -d ' ')
|
||||
[ "$GOT" -gt 0 ] || die "the copy holds no repository"
|
||||
[ "$GOT" -ge "$WANT" ] || die "the copy holds $GOT repositories, the pod lists $WANT"
|
||||
log "files: $GOT repositories, $(du -sm "$STAGE/root/gitea" | cut -f1) MB"
|
||||
|
||||
# 3. the secrets — the newest night's GPG files (already encrypted with DooPlex's restic passphrase)
|
||||
NEWEST=$(ls -1 "$SECRETS"/secrets-*.yaml.gpg 2>/dev/null | sort | tail -n 1)
|
||||
[ -n "$NEWEST" ] || die "no secrets export under $SECRETS"
|
||||
STAMP=${NEWEST##*/secrets-}; STAMP=${STAMP%.yaml.gpg}
|
||||
SAGE=$((NOW - $(stat -c %Y "$NEWEST")))
|
||||
[ "$SAGE" -le $((SECRETS_MAX_AGE_H * 3600)) ] || die "newest secrets export is $((SAGE / 3600)) h old (limit ${SECRETS_MAX_AGE_H} h)"
|
||||
cp "$SECRETS"/*-"$STAMP".*gpg "$STAGE/root/secrets/"
|
||||
log "secrets: $(ls "$STAGE/root/secrets" | wc -l | tr -d ' ') file(s) of $STAMP"
|
||||
|
||||
# 4. the manifest the restore test checks, then the push
|
||||
echo "$GOT" > "$STAGE/root/REPOS"
|
||||
(cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \
|
||||
|| die "writing the manifest"
|
||||
[ "$(wc -l < "$STAGE/root/MANIFEST.sha256")" -gt "$GOT" ] || die "the manifest is short"
|
||||
START=$(date +%s)
|
||||
PBS_PASSWORD_FILE="$TOKENS/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
|
||||
proxmox-backup-client backup dooplex.pxar:"$STAGE/root" --ns operator --backup-type host --backup-id dooplex-gitea \
|
||||
--keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
|
||||
|| die "proxmox-backup-client backup"
|
||||
BYTES=$(du -sb "$STAGE/root" | cut -f1)
|
||||
log "pushed to ep0 (ns operator, host/dooplex-gitea) in $(( $(date +%s) - START )) s"
|
||||
|
||||
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite.prom.$$"
|
||||
{
|
||||
echo "# HELP felhom_dooplex_offsite_last_success_timestamp_seconds Last successful push of Gitea + DooPlex secrets to ep0 (R-232)."
|
||||
echo "# TYPE felhom_dooplex_offsite_last_success_timestamp_seconds gauge"
|
||||
echo "felhom_dooplex_offsite_last_success_timestamp_seconds $(date +%s)"
|
||||
echo "felhom_dooplex_offsite_last_success_bytes $BYTES"
|
||||
echo "felhom_dooplex_offsite_last_success_repositories $GOT"
|
||||
} > "$TMP"
|
||||
chmod 644 "$TMP"
|
||||
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite.prom"
|
||||
log "success signal written"
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/bin/sh
|
||||
# felhom-dooplex-offsite-restore-test — restore the newest Gitea + secrets copy from ep0 with the READ-ONLY token and
|
||||
# check it (R-232 (h)). Runs on DooPlex as root from felhom-dooplex-offsite-restore-test.timer (Sun 05:30).
|
||||
# Pinned by test_dooplex_offsite.py.
|
||||
#
|
||||
# The success timestamp is written ONLY when: the newest copy on ep0 is at most MAX_AGE_H old; it restores and
|
||||
# decrypts; every file matches MANIFEST.sha256; the repository count matches REPOS; `git fsck` passes on EVERY
|
||||
# repository; `pg_restore --list` reads gitea.dump; app.ini is there; at least one secrets file is there.
|
||||
set -eu
|
||||
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
|
||||
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
|
||||
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
|
||||
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
|
||||
MAX_AGE_H=${FELHOM_DXOFF_RESTORE_MAX_AGE_H:-50}
|
||||
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
|
||||
. "$CONF/env" # PBS_REPOSITORY_RESTORE, PBS_FINGERPRINT
|
||||
|
||||
log() { echo "felhom-dooplex-offsite-restore-test: $*"; }
|
||||
die() { echo "felhom-dooplex-offsite-restore-test: FAILED: $*" >&2; exit 1; }
|
||||
|
||||
umask 077
|
||||
mkdir -p "$STATE"; chmod 700 "$STATE"
|
||||
T=$(mktemp -d "$STATE/restore.XXXXXX")
|
||||
trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
|
||||
export PBS_PASSWORD_FILE="$TOKENS/token-restore" PBS_FINGERPRINT
|
||||
|
||||
LIST=$(proxmox-backup-client snapshot list host/dooplex-gitea --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
|
||||
|| die "listing snapshots on ep0"
|
||||
NEWEST=$(printf '%s' "$LIST" | python3 -c '
|
||||
import json, sys
|
||||
s = [x for x in json.load(sys.stdin) if x.get("backup-type") == "host" and x.get("backup-id") == "dooplex-gitea"]
|
||||
if s:
|
||||
print(max(x["backup-time"] for x in s))
|
||||
') || die "reading the snapshot list"
|
||||
[ -n "$NEWEST" ] || die "no Gitea copy on ep0"
|
||||
AGE=$((NOW - NEWEST))
|
||||
[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest copy on ep0 is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h)"
|
||||
SNAPSHOT="host/dooplex-gitea/$(date -u -d "@$NEWEST" +%Y-%m-%dT%H:%M:%SZ)"
|
||||
log "restoring $SNAPSHOT"
|
||||
|
||||
proxmox-backup-client restore "$SNAPSHOT" dooplex.pxar "$T/out" --ns operator \
|
||||
--keyfile "$CONF/enc.key" --repository "$PBS_REPOSITORY_RESTORE" || die "restore of $SNAPSHOT"
|
||||
O="$T/out"
|
||||
[ -s "$O/MANIFEST.sha256" ] || die "the copy holds no MANIFEST.sha256"
|
||||
(cd "$O" && sha256sum -c MANIFEST.sha256 >/dev/null) || die "a file does not match MANIFEST.sha256"
|
||||
FILES=$(wc -l < "$O/MANIFEST.sha256" | tr -d ' ')
|
||||
|
||||
WANT=$(cat "$O/REPOS" 2>/dev/null) || die "the copy holds no REPOS count"
|
||||
REPOS=$(find "$O/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | sort)
|
||||
GOT=$(printf '%s\n' "$REPOS" | grep -c '\.git$') || GOT=0
|
||||
[ "$GOT" -gt 0 ] && [ "$GOT" = "$WANT" ] || die "the copy holds $GOT repositories, REPOS says $WANT"
|
||||
# The repositories' own `config` files came from the Gitea pod — untrusted input to a root git. So git never reads
|
||||
# them: each repository is checked through a fresh scratch repository with a known config, holding a copy of its
|
||||
# HEAD and refs, with the copy's objects as its object store. No system or global config either.
|
||||
export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null
|
||||
for r in $REPOS; do
|
||||
S="$T/fsck.git"; rm -rf "$S"
|
||||
git init -q --bare "$S" || die "git init for the fsck scratch repository"
|
||||
cp "$r/HEAD" "$S/HEAD"; [ -f "$r/packed-refs" ] && cp "$r/packed-refs" "$S/packed-refs"
|
||||
[ -d "$r/refs" ] && cp -R "$r/refs/." "$S/refs/"
|
||||
GIT_OBJECT_DIRECTORY="$r/objects" git --git-dir="$S" -c core.hooksPath=/dev/null -c core.fsmonitor=false \
|
||||
fsck --no-progress --no-dangling >/dev/null 2>"$T/fsck.err" \
|
||||
|| die "git fsck ${r#"$O"/gitea/git/repositories/}: $(head -n 3 "$T/fsck.err")"
|
||||
done
|
||||
rm -rf "$T/fsck.git"
|
||||
[ -s "$O/gitea/gitea/conf/app.ini" ] || die "app.ini missing"
|
||||
pg_restore --list "$O/db/gitea.dump" >/dev/null || die "pg_restore cannot read gitea.dump"
|
||||
ls "$O"/secrets/*.gpg >/dev/null 2>&1 || die "no secrets file in the copy"
|
||||
log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s)"
|
||||
|
||||
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom.$$"
|
||||
{
|
||||
echo "# HELP felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds Last successful restore test of the Gitea + secrets copy on ep0 (R-232)."
|
||||
echo "# TYPE felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds gauge"
|
||||
echo "felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds $(date +%s)"
|
||||
} > "$TMP"
|
||||
chmod 644 "$TMP"
|
||||
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom"
|
||||
log "success signal written"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Versioned in felhom.eu/scripts/dooplex-offsite/ (R-232); installed by install.sh. Plan: audits/dooplex-survival-2026-10-09/PLAN.md.
|
||||
[Unit]
|
||||
Description=Felhom: restore-test the Gitea + secrets copy on ep0 (R-232)
|
||||
Wants=network-online.target felhom-ep0-pbs-tunnel.service
|
||||
After=network-online.target felhom-ep0-pbs-tunnel.service
|
||||
OnFailure=felhom-backup-failmail@%n.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-dooplex-offsite-restore-test
|
||||
Environment=HOME=/var/lib/felhom-dooplex-offsite KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||
UMask=0077
|
||||
TimeoutStartSec=90min
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
PrivateTmp=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ReadWritePaths=/var/lib/felhom-dooplex-offsite /var/lib/node_exporter/textfile_collector
|
||||
NoNewPrivileges=yes
|
||||
@@ -0,0 +1,11 @@
|
||||
# Versioned in felhom.eu/scripts/dooplex-offsite/ (R-232); installed by install.sh.
|
||||
[Unit]
|
||||
Description=Felhom: restore-test the Gitea + secrets copy on ep0 (R-232) — schedule
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 05:30:00
|
||||
Persistent=true
|
||||
RandomizedDelaySec=2min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,20 @@
|
||||
# Versioned in felhom.eu/scripts/dooplex-offsite/ (R-232); installed by install.sh. Plan: audits/dooplex-survival-2026-10-09/PLAN.md.
|
||||
[Unit]
|
||||
Description=Felhom: push Gitea + DooPlex secrets to ep0, encrypted (R-232)
|
||||
Wants=network-online.target felhom-ep0-pbs-tunnel.service
|
||||
After=network-online.target felhom-ep0-pbs-tunnel.service
|
||||
OnFailure=felhom-backup-failmail@%n.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-dooplex-offsite
|
||||
Environment=HOME=/var/lib/felhom-dooplex-offsite KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||
UMask=0077
|
||||
TimeoutStartSec=90min
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
PrivateTmp=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ReadWritePaths=/var/lib/felhom-dooplex-offsite /var/lib/node_exporter/textfile_collector
|
||||
NoNewPrivileges=yes
|
||||
@@ -0,0 +1,11 @@
|
||||
# Versioned in felhom.eu/scripts/dooplex-offsite/ (R-232); installed by install.sh.
|
||||
[Unit]
|
||||
Description=Felhom: push Gitea + DooPlex secrets to ep0 (R-232) — schedule
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 00:20:00
|
||||
Persistent=true
|
||||
RandomizedDelaySec=2min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# install.sh — install the Gitea + secrets off-site units on DooPlex (R-232). Root. Idempotent.
|
||||
# Installs the three scripts and five units, writes /etc/felhom-dooplex-offsite/env (no secrets) when absent, and does
|
||||
# NOT enable the timers — enable them by hand after the first manual run:
|
||||
# systemctl enable --now felhom-dooplex-offsite.timer felhom-dooplex-offsite-restore-test.timer
|
||||
# The tokens are the hub-DB ones (/etc/felhom-hub-backup/token-push, token-restore), read in place. enc.key is created
|
||||
# separately (proxmox-backup-client key create --kdf none), never by this script.
|
||||
set -eu
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
[ "$(id -u)" = 0 ] || { echo "install.sh: run as root" >&2; exit 1; }
|
||||
for s in felhom-dooplex-offsite felhom-dooplex-offsite-restore-test felhom-backup-failmail; do
|
||||
install -m 0755 "$HERE/$s" "/usr/local/sbin/$s"
|
||||
done
|
||||
for u in felhom-dooplex-offsite.service felhom-dooplex-offsite.timer felhom-dooplex-offsite-restore-test.service \
|
||||
felhom-dooplex-offsite-restore-test.timer felhom-backup-failmail@.service; do
|
||||
install -m 0644 "$HERE/$u" "/etc/systemd/system/$u"
|
||||
done
|
||||
install -d -m 0700 /etc/felhom-dooplex-offsite /var/lib/felhom-dooplex-offsite
|
||||
if [ ! -f /etc/felhom-dooplex-offsite/env ]; then
|
||||
umask 077
|
||||
grep -E '^(PBS_REPOSITORY_PUSH|PBS_REPOSITORY_RESTORE|PBS_FINGERPRINT)=' /etc/felhom-hub-backup/env > /etc/felhom-dooplex-offsite/env
|
||||
fi
|
||||
systemctl daemon-reload
|
||||
echo "install.sh: installed; timers NOT enabled (see the header)"
|
||||
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for felhom-dooplex-offsite, its restore test and felhom-backup-failmail (R-232).
|
||||
|
||||
No test reaches Gitea, k3s, PBS, ep0 or Resend: `kubectl`, `proxmox-backup-client` and `pg_restore` are fakes on PATH
|
||||
(the Gitea pod's /data is a temp dir; the PBS "server" is a temp dir; the fake pg_restore accepts a file that starts
|
||||
with PostgreSQL's custom-dump magic "PGDMP"). `git`, `tar`, `sha256sum`, `shred`, `find` are the real tools, so
|
||||
`git fsck` runs on real repositories. Each test asserts the CONSEQUENCE: whether a push happened and whether the
|
||||
success signal (the file the alarm reads) was written. Run: python3 scripts/dooplex-offsite/test_dooplex_offsite.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PUSH = os.path.join(HERE, "felhom-dooplex-offsite")
|
||||
RESTORE = os.path.join(HERE, "felhom-dooplex-offsite-restore-test")
|
||||
FAILMAIL = os.path.join(HERE, "felhom-backup-failmail")
|
||||
|
||||
FAKE_KUBECTL = r'''#!/usr/bin/env python3
|
||||
import os, subprocess, sys
|
||||
a = sys.argv[1:]
|
||||
pod = os.environ["FAKE_POD_DATA"]
|
||||
cmd = a[a.index("--") + 1:]
|
||||
if cmd[0] == "tar":
|
||||
cnt = os.path.join(os.environ["FAKE_STATE"], "tar-calls")
|
||||
n = int(open(cnt).read()) if os.path.exists(cnt) else 0
|
||||
open(cnt, "w").write(str(n + 1))
|
||||
if n < int(os.environ.get("FAKE_TAR_FAILS", "0")):
|
||||
sys.stderr.write("tar: file vanished\n"); sys.exit(1)
|
||||
c = ["tar" if x == "tar" else (pod if x == "/data" else x) for x in cmd]
|
||||
sys.exit(subprocess.call(c))
|
||||
if cmd[0] == "find":
|
||||
out = subprocess.run(["find", pod + cmd[1][len("/data"):]] + cmd[2:],
|
||||
capture_output=True, text=True)
|
||||
for line in out.stdout.splitlines():
|
||||
print("/data" + line[len(pod):])
|
||||
for extra in filter(None, os.environ.get("FAKE_EXTRA_REPOS", "").split(",")):
|
||||
print("/data/git/repositories/admin/" + extra)
|
||||
sys.exit(out.returncode)
|
||||
sys.exit(97)
|
||||
'''
|
||||
|
||||
FAKE_PBS = r'''#!/usr/bin/env python3
|
||||
import json, os, shutil, sys, time
|
||||
a = sys.argv[1:]
|
||||
srv = os.environ["FAKE_PBS_DIR"]
|
||||
open(os.path.join(srv, "calls.log"), "a").write(json.dumps({"argv": a, "pw": os.environ.get("PBS_PASSWORD_FILE", "")}) + "\n")
|
||||
base = os.path.join(srv, "snaps")
|
||||
if a[0] == "backup":
|
||||
if os.environ.get("FAKE_PBS_FAIL"): sys.exit(1)
|
||||
src = a[1].split(":", 1)[1]
|
||||
t = int(os.environ.get("FAKE_PBS_TIME", time.time()))
|
||||
shutil.copytree(src, os.path.join(base, str(t)))
|
||||
sys.exit(0)
|
||||
if a[0] == "snapshot" and a[1] == "list":
|
||||
out = [{"backup-type": "host", "backup-id": "dooplex-gitea", "backup-time": int(x)} for x in (os.listdir(base) if os.path.isdir(base) else [])]
|
||||
print(json.dumps(out)); sys.exit(0)
|
||||
if a[0] == "restore":
|
||||
import calendar
|
||||
t = calendar.timegm(time.strptime(a[1].split("/")[-1], "%Y-%m-%dT%H:%M:%SZ"))
|
||||
shutil.copytree(os.path.join(base, str(t)), a[3])
|
||||
sys.exit(0)
|
||||
sys.exit(98)
|
||||
'''
|
||||
|
||||
FAKE_PG_RESTORE = r'''#!/bin/sh
|
||||
[ "$1" = "--list" ] || exit 97
|
||||
head -c 5 "$2" | grep -q '^PGDMP' || { echo "pg_restore: error: input file does not appear to be a valid archive" >&2; exit 1; }
|
||||
'''
|
||||
|
||||
|
||||
def git(*a, cwd=None):
|
||||
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t")
|
||||
subprocess.run(["git", *a], cwd=cwd, check=True, capture_output=True, env=env)
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.t = tempfile.mkdtemp()
|
||||
j = lambda *p: os.path.join(self.t, *p)
|
||||
self.pod, self.pbs, self.state, self.text = j("pod"), j("pbs"), j("state"), j("textfile")
|
||||
self.conf, self.tokens, self.dumps, self.secrets, self.bin = j("conf"), j("tokens"), j("dumps"), j("secrets"), j("bin")
|
||||
for d in (self.pbs, self.state, self.text, self.conf, self.tokens, self.dumps, self.secrets, self.bin):
|
||||
os.makedirs(d)
|
||||
# the Gitea pod: two real repositories with a commit each, app.ini, the other paths
|
||||
for name in ("felhom.eu", "felhom-agent"):
|
||||
bare = j("pod", "git", "repositories", "admin", name + ".git")
|
||||
os.makedirs(os.path.dirname(bare), exist_ok=True)
|
||||
git("init", "-q", "--bare", bare)
|
||||
work = j("work-" + name)
|
||||
git("init", "-q", work)
|
||||
open(os.path.join(work, "README"), "w").write(name + "\n")
|
||||
git("add", "README", cwd=work); git("commit", "-q", "-m", "c", cwd=work)
|
||||
git("push", "-q", bare, "HEAD:refs/heads/main", cwd=work)
|
||||
for p in ("git/lfs", "gitea/conf", "gitea/attachments", "gitea/avatars", "gitea/repo-avatars", "gitea/jwt", "gitea/packages"):
|
||||
os.makedirs(j("pod", p), exist_ok=True)
|
||||
open(j("pod", "gitea/conf/app.ini"), "w").write("[database]\nDB_TYPE = postgres\n")
|
||||
open(j("pod", "gitea/packages/blob"), "w").write("registry - must not be copied\n")
|
||||
# a complete dump (00:00 local) and the nightly secrets export
|
||||
self.now = int(time.time())
|
||||
self.add_dump("20261008-220001", self.now - 1200)
|
||||
self.add_secrets("20261008_031011", self.now - 21 * 3600)
|
||||
for n in ("token-push", "token-restore"):
|
||||
open(os.path.join(self.tokens, n), "w").write("x")
|
||||
open(os.path.join(self.conf, "enc.key"), "w").write("{}")
|
||||
open(os.path.join(self.conf, "env"), "w").write(
|
||||
"PBS_REPOSITORY_PUSH='u!push@h:1:s'\nPBS_REPOSITORY_RESTORE='u!restore@h:1:s'\nPBS_FINGERPRINT='aa'\n")
|
||||
for name, body in (("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS), ("pg_restore", FAKE_PG_RESTORE)):
|
||||
p = os.path.join(self.bin, name)
|
||||
open(p, "w").write(body); os.chmod(p, 0o755)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.t, ignore_errors=True)
|
||||
|
||||
def add_dump(self, name, mtime, complete=True, magic=b"PGDMP"):
|
||||
d = os.path.join(self.dumps, name); os.makedirs(d)
|
||||
open(os.path.join(d, "gitea.dump"), "wb").write(magic + b"\x01dump")
|
||||
open(os.path.join(d, "globals.sql"), "w").write("-- roles\n")
|
||||
if complete:
|
||||
open(os.path.join(d, "SUCCESS"), "w").close(); os.utime(os.path.join(d, "SUCCESS"), (mtime, mtime))
|
||||
|
||||
def add_secrets(self, stamp, mtime):
|
||||
for k, ext in (("secrets", "yaml.gpg"), ("configmaps", "yaml.gpg"), ("by-namespace", "tar.gz.gpg")):
|
||||
p = os.path.join(self.secrets, "%s-%s.%s" % (k, stamp, ext))
|
||||
open(p, "wb").write(b"gpg-" + k.encode()); os.utime(p, (mtime, mtime))
|
||||
|
||||
def env(self, **kw):
|
||||
e = dict(os.environ, PATH=self.bin + ":" + os.environ["PATH"], FAKE_POD_DATA=self.pod, FAKE_PBS_DIR=self.pbs,
|
||||
FAKE_STATE=self.t, FELHOM_DXOFF_CONF=self.conf, FELHOM_DXOFF_TOKENS=self.tokens,
|
||||
FELHOM_DXOFF_STATE=self.state, FELHOM_DXOFF_TEXTFILE_DIR=self.text, FELHOM_DXOFF_DUMPS=self.dumps,
|
||||
FELHOM_DXOFF_SECRETS=self.secrets)
|
||||
e.update({k: str(v) for k, v in kw.items()})
|
||||
return e
|
||||
|
||||
def push(self, **kw):
|
||||
return subprocess.run([PUSH], env=self.env(**kw), capture_output=True, text=True)
|
||||
|
||||
def restore(self, **kw):
|
||||
return subprocess.run([RESTORE], env=self.env(**kw), capture_output=True, text=True)
|
||||
|
||||
def pushed(self):
|
||||
d = os.path.join(self.pbs, "snaps")
|
||||
return sorted(os.listdir(d)) if os.path.isdir(d) else []
|
||||
|
||||
def signal(self, name="felhom_dooplex_offsite.prom"):
|
||||
return os.path.exists(os.path.join(self.text, name))
|
||||
|
||||
|
||||
class Push(Base):
|
||||
def test_happy_path_pushes_everything_but_the_registry_and_writes_the_signal(self):
|
||||
r = self.push()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertEqual(len(self.pushed()), 1)
|
||||
snap = os.path.join(self.pbs, "snaps", self.pushed()[0])
|
||||
self.assertTrue(os.path.isfile(os.path.join(snap, "gitea/git/repositories/admin/felhom.eu.git/HEAD")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(snap, "gitea/gitea/conf/app.ini")))
|
||||
self.assertFalse(os.path.exists(os.path.join(snap, "gitea/gitea/packages")), "the registry must stay out")
|
||||
self.assertEqual(open(os.path.join(snap, "db/DUMP-FOLDER")).read().strip(), "20261008-220001")
|
||||
self.assertEqual(len(os.listdir(os.path.join(snap, "secrets"))), 3)
|
||||
self.assertEqual(open(os.path.join(snap, "REPOS")).read().strip(), "2")
|
||||
self.assertTrue(self.signal())
|
||||
call = json.loads(open(os.path.join(self.pbs, "calls.log")).readline())
|
||||
self.assertIn("--crypt-mode", call["argv"]); self.assertIn("encrypt", call["argv"])
|
||||
self.assertTrue(call["pw"].endswith("token-push"))
|
||||
self.assertFalse(os.path.exists(os.path.join(self.state, "stage")), "the stage must be removed")
|
||||
|
||||
def test_newest_incomplete_dump_is_skipped_for_the_complete_one(self):
|
||||
self.add_dump("20261009-040001", self.now - 60, complete=False)
|
||||
r = self.push()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
snap = os.path.join(self.pbs, "snaps", self.pushed()[0])
|
||||
self.assertEqual(open(os.path.join(snap, "db/DUMP-FOLDER")).read().strip(), "20261008-220001")
|
||||
|
||||
def test_stale_dump_refuses(self):
|
||||
shutil.rmtree(self.dumps); os.makedirs(self.dumps)
|
||||
self.add_dump("20261008-040001", self.now - 8 * 3600)
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("dump CronJob stopped", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_no_complete_dump_refuses(self):
|
||||
shutil.rmtree(self.dumps); os.makedirs(self.dumps)
|
||||
self.add_dump("20261009-040001", self.now, complete=False)
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_one_failed_file_copy_is_retried(self):
|
||||
r = self.push(FAKE_TAR_FAILS=1, FELHOM_DXOFF_RETRY_SLEEP=0)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertTrue(self.signal())
|
||||
|
||||
def test_two_failed_file_copies_refuse(self):
|
||||
r = self.push(FAKE_TAR_FAILS=2, FELHOM_DXOFF_RETRY_SLEEP=0)
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("failed twice", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_fewer_repositories_than_the_pod_lists_refuses(self):
|
||||
r = self.push(FAKE_EXTRA_REPOS="ghost.git")
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("the pod lists 3", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_missing_app_ini_refuses(self):
|
||||
os.remove(os.path.join(self.pod, "gitea/conf/app.ini"))
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_stale_secrets_refuse(self):
|
||||
shutil.rmtree(self.secrets); os.makedirs(self.secrets)
|
||||
self.add_secrets("20261007_031011", self.now - 31 * 3600)
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("secrets export", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_a_link_in_the_pods_archive_refuses(self):
|
||||
os.symlink("/etc/passwd", os.path.join(self.pod, "gitea/avatars/evil"))
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("link(s)", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_failed_push_writes_no_signal(self):
|
||||
r = self.push(FAKE_PBS_FAIL=1)
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertFalse(self.signal())
|
||||
|
||||
|
||||
class RestoreTest(Base):
|
||||
def pushed_copy(self, **kw):
|
||||
r = self.push(FAKE_PBS_TIME=self.now - 3600, **kw)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
return os.path.join(self.pbs, "snaps", self.pushed()[0])
|
||||
|
||||
def test_happy_path_writes_the_signal_with_the_readonly_token(self):
|
||||
self.pushed_copy()
|
||||
r = self.restore()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("2 repositories pass git fsck", r.stdout)
|
||||
self.assertTrue(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
calls = [json.loads(x) for x in open(os.path.join(self.pbs, "calls.log"))]
|
||||
self.assertTrue(all(c["pw"].endswith("token-restore") for c in calls if c["argv"][0] != "backup"))
|
||||
|
||||
def test_a_changed_file_fails_the_manifest(self):
|
||||
snap = self.pushed_copy()
|
||||
open(os.path.join(snap, "secrets", sorted(os.listdir(os.path.join(snap, "secrets")))[0]), "ab").write(b"x")
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("MANIFEST", r.stderr)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
def test_a_broken_repository_fails_git_fsck(self):
|
||||
snap = self.pushed_copy()
|
||||
repo = os.path.join(snap, "gitea/git/repositories/admin/felhom-agent.git")
|
||||
objs = [os.path.join(dp, f) for dp, _, fs in os.walk(os.path.join(repo, "objects")) for f in fs
|
||||
if len(os.path.basename(dp)) == 2]
|
||||
os.remove(objs[0])
|
||||
# keep the manifest honest about the removal, so ONLY git fsck can catch it
|
||||
man = os.path.join(snap, "MANIFEST.sha256")
|
||||
rel = "./" + os.path.relpath(objs[0], snap)
|
||||
kept = [l for l in open(man).readlines() if not l.rstrip().endswith(rel)]
|
||||
open(man, "w").writelines(kept)
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("git fsck", r.stderr)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
def test_git_never_reads_a_repositorys_own_config(self):
|
||||
# The pod's `config` files are untrusted input to a root git. A config git would refuse to open
|
||||
# (repositoryformatversion 99) proves git never read it: the old `git -C <repo> fsck` failed here (red-proof).
|
||||
snap = self.pushed_copy()
|
||||
cfg = os.path.join(snap, "gitea/git/repositories/admin/felhom.eu.git/config")
|
||||
open(cfg, "w").write("[core]\n\trepositoryformatversion = 99\n\tbare = true\n\tfsmonitor = touch /nonexistent/x\n")
|
||||
man = os.path.join(snap, "MANIFEST.sha256")
|
||||
kept = [l for l in open(man).readlines() if not l.rstrip().endswith("felhom.eu.git/config")]
|
||||
open(man, "w").writelines(kept)
|
||||
r = self.restore()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_an_unreadable_dump_fails(self):
|
||||
shutil.rmtree(self.dumps); os.makedirs(self.dumps)
|
||||
self.add_dump("20261008-220001", self.now - 1200, magic=b"XXXXX")
|
||||
self.pushed_copy()
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("pg_restore", r.stderr)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
def test_an_old_copy_fails(self):
|
||||
r = self.push(FAKE_PBS_TIME=self.now - 51 * 3600)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("h old", r.stderr)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
def test_no_copy_fails(self):
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
|
||||
class FailMail(unittest.TestCase):
|
||||
def test_failmail_calls_notify_failure_with_the_unit_name(self):
|
||||
t = tempfile.mkdtemp()
|
||||
try:
|
||||
cfg = os.path.join(t, "cfg.sh"); out = os.path.join(t, "out")
|
||||
open(cfg, "w").write('notify_failure() { echo "$1" > %s; return 0; }\n' % out)
|
||||
r = subprocess.run([FAILMAIL, "felhom-dooplex-offsite.service"], env=dict(os.environ, FELHOM_FAILMAIL_CONFIG=cfg),
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("felhom-dooplex-offsite.service failed", open(out).read())
|
||||
finally:
|
||||
shutil.rmtree(t)
|
||||
|
||||
def test_every_backup_unit_names_the_failmail(self):
|
||||
units = [os.path.join(HERE, u) for u in ("felhom-dooplex-offsite.service", "felhom-dooplex-offsite-restore-test.service")]
|
||||
units += [os.path.join(HERE, "..", "hub-db-backup", u) for u in ("felhom-hub-db-backup.service", "felhom-hub-db-restore-test.service")]
|
||||
for u in units:
|
||||
self.assertIn("OnFailure=felhom-backup-failmail@%n.service", open(u).read(), u)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user