#!/bin/sh
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config), Vaultwarden (R-923) and DooPlex's nightly secrets export to
# ep0's PBS, encrypted on DooPlex (R-232 (b)). Runs on DooPlex as root from felhom-dooplex-offsite.timer (00:20).
# Plan: documentation/audits/dooplex-survival-2026-10-09/PLAN.md. Restore: documentation/runbooks/gitea-restore.md.
# Pinned by test_dooplex_offsite.py.
#
# Order is the consistency argument (PLAN.md): the database dump is taken FIRST (the newest complete 6-hourly dump,
# PostgreSQL's own snapshot), the repositories AFTER it, so every commit the database names is in the copy.
#
# Refuses to push — and so never writes the success signal — when: no complete dump exists or the newest is older than
# DUMP_MAX_AGE_H; the dump is empty; the file copy from the pod fails twice; the copy holds no repository or fewer
# repositories than the pod lists; app.ini is missing; no secrets export exists or it is older than SECRETS_MAX_AGE_H.
# The success timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
set -eu
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
DUMPS=${FELHOM_DXOFF_DUMPS:-/mnt/5_hdd/backup/postgresql/dumps}
SECRETS=${FELHOM_DXOFF_SECRETS:-/mnt/5_hdd/backup/secrets/exports}
SIGNING=${FELHOM_DXOFF_SIGNING:-/mnt/5_hdd/felhom.eu}
SIGNING_KEYS="felhom-op-operational felhom-rec-recovery felhom_op_ed25519"
DUMP_MAX_AGE_H=${FELHOM_DXOFF_DUMP_MAX_AGE_H:-7}
SECRETS_MAX_AGE_H=${FELHOM_DXOFF_SECRETS_MAX_AGE_H:-30}
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
RETRY_SLEEP=${FELHOM_DXOFF_RETRY_SLEEP:-30}
. "$CONF/env"   # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)

log() { echo "felhom-dooplex-offsite: $*"; }
die() { echo "felhom-dooplex-offsite: FAILED: $*" >&2; exit 1; }
K() { kubectl -n gitea-system exec deploy/gitea -c gitea -- "$@"; }
V() { kubectl -n vaultwarden-system exec deploy/vaultwarden -- "$@"; }

umask 077
STAGE="$STATE/stage"
mkdir -p "$STATE"; chmod 700 "$STATE"
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets" "$STAGE/root/vaultwarden" "$STAGE/root/signing"
cleanup() {
  for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql" \
           "$STAGE/root/vaultwarden/db.sqlite3" "$STAGE/root/vaultwarden/rsa_key.pem" \
           "$STAGE/root/signing/felhom-op-operational" "$STAGE/root/signing/felhom-rec-recovery" "$STAGE/root/signing/felhom_op_ed25519"; do
    [ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
  done
  rm -rf "$STAGE"
}
trap cleanup EXIT

# 1. the database — the newest COMPLETE dump (its folder carries SUCCESS), taken before the files
DUMPDIR=""
for d in $(ls -1d "$DUMPS"/[0-9]*-[0-9]* 2>/dev/null | sort -r); do
  if [ -f "$d/SUCCESS" ] && [ -s "$d/gitea.dump" ]; then DUMPDIR=$d; break; fi
done
[ -n "$DUMPDIR" ] || die "no complete gitea.dump under $DUMPS"
DAGE=$((NOW - $(stat -c %Y "$DUMPDIR/SUCCESS")))
[ "$DAGE" -le $((DUMP_MAX_AGE_H * 3600)) ] || die "newest complete dump ${DUMPDIR##*/} is $((DAGE / 3600)) h old (limit ${DUMP_MAX_AGE_H} h) — the dump CronJob stopped"
cp "$DUMPDIR/gitea.dump" "$STAGE/root/db/gitea.dump"
[ -f "$DUMPDIR/globals.sql" ] && cp "$DUMPDIR/globals.sql" "$STAGE/root/db/globals.sql"
echo "${DUMPDIR##*/}" > "$STAGE/root/db/DUMP-FOLDER"
log "database: ${DUMPDIR##*/}, $(wc -c < "$STAGE/root/db/gitea.dump" | tr -d ' ') bytes, $((DAGE / 60)) min old"

# 2. the files — after the dump. Not the registry (packages: rebuilt from the code), logs, indexers, queues, tmp.
PATHS="git/repositories git/lfs gitea/conf/app.ini gitea/attachments gitea/avatars gitea/repo-avatars gitea/jwt"
n=0
until K tar -cf - -C /data $PATHS > "$STAGE/files.tar"; do
  n=$((n + 1)); [ "$n" -lt 2 ] || die "copying Gitea's files out of the pod failed twice"
  log "file copy failed once (a file moved under a push?) — retrying in ${RETRY_SLEEP} s"; sleep "$RETRY_SLEEP"
done
# The pod's archive is untrusted input to a root process: refuse any symlink or hardlink in it (a bare repository holds
# none), and extract without the pod's owners.
LINKS=$(tar -tvf "$STAGE/files.tar" | grep -c '^[lh]') || LINKS=0
[ "$LINKS" -eq 0 ] || die "the pod's archive holds $LINKS link(s) — refused"
tar -xof "$STAGE/files.tar" -C "$STAGE/root/gitea" || die "unpacking the file copy"
rm -f "$STAGE/files.tar"
[ -s "$STAGE/root/gitea/gitea/conf/app.ini" ] && [ ! -L "$STAGE/root/gitea/gitea/conf/app.ini" ] || die "app.ini missing from the copy"
LISTING=$(K find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git') || die "listing repositories in the pod"
WANT=$(printf '%s\n' "$LISTING" | grep -c '\.git$') || WANT=0
GOT=$(find "$STAGE/root/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | wc -l | tr -d ' ')
[ "$GOT" -gt 0 ] || die "the copy holds no repository"
[ "$GOT" -ge "$WANT" ] || die "the copy holds $GOT repositories, the pod lists $WANT"
log "files: $GOT repositories, $(du -sm "$STAGE/root/gitea" | cut -f1) MB"

# 3. the secrets — the newest night's GPG files (already encrypted with DooPlex's restic passphrase)
NEWEST=$(ls -1 "$SECRETS"/secrets-*.yaml.gpg 2>/dev/null | sort | tail -n 1)
[ -n "$NEWEST" ] || die "no secrets export under $SECRETS"
STAMP=${NEWEST##*/secrets-}; STAMP=${STAMP%.yaml.gpg}
SAGE=$((NOW - $(stat -c %Y "$NEWEST")))
[ "$SAGE" -le $((SECRETS_MAX_AGE_H * 3600)) ] || die "newest secrets export is $((SAGE / 3600)) h old (limit ${SECRETS_MAX_AGE_H} h)"
cp "$SECRETS"/*-"$STAMP".*gpg "$STAGE/root/secrets/"
log "secrets: $(ls "$STAGE/root/secrets" | wc -l | tr -d ' ') file(s) of $STAMP"

# 3b. the password manager (Vaultwarden, R-923) — its own `vaultwarden backup` (SQLite VACUUM INTO, consistent while it
# runs) writes ONE file into /data; it is copied out and that one file removed. Plus rsa_key.pem and, when they exist,
# attachments/, sends/, config.json (Vaultwarden's backup guidance). The vault items are encrypted under each user's
# master password; this job never opens them. Refuses on: the backup command failing, an unexpected file name, a
# failed integrity_check, no user.
VOUT=$(V /vaultwarden backup 2>&1) || die "vaultwarden backup: $(printf '%s' "$VOUT" | tail -n 1)"
VFILE=$(printf '%s\n' "$VOUT" | sed -n "s#^Backup to '\(.*\)' was successful.*#\1#p" | tail -n 1)
case "$VFILE" in data/db_[0-9]*_[0-9]*.sqlite3) ;; *) die "vaultwarden backup: unexpected output (no backup file named)" ;; esac
VPATH="/$VFILE"
VRC=0; V cat "$VPATH" > "$STAGE/root/vaultwarden/db.sqlite3" || VRC=$?
V rm -f "$VPATH" || log "WARNING: could not remove $VPATH from Vaultwarden's /data"
[ "$VRC" -eq 0 ] || die "copying $VPATH out of the Vaultwarden pod"
VLIST=$(V sh -c 'cd /data && for p in rsa_key.pem rsa_key.pub.pem config.json attachments sends; do if [ -e "$p" ]; then echo "$p"; fi; done') \
  || die "listing Vaultwarden's files"
[ -n "$VLIST" ] && { V tar -cf - -C /data $VLIST > "$STAGE/vw.tar" || die "copying Vaultwarden's files"; }
if [ -s "$STAGE/vw.tar" ]; then
  LINKS=$(tar -tvf "$STAGE/vw.tar" | grep -c '^[lh]') || LINKS=0
  [ "$LINKS" -eq 0 ] || die "Vaultwarden's archive holds $LINKS link(s) — refused"
  tar -xof "$STAGE/vw.tar" -C "$STAGE/root/vaultwarden" || die "unpacking Vaultwarden's files"
  rm -f "$STAGE/vw.tar"
fi
VIC=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
VUSERS=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
[ "${VUSERS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no user"
echo "$VUSERS" > "$STAGE/root/vaultwarden/USERS"
log "vaultwarden: integrity ok, $VUSERS user(s), files: db.sqlite3 $(echo $VLIST)"

# 3c. the operator signing keys (R-924, operator ruling 2026-10-09: option A — on paper AND in this copy). Without them
# no box accepts a signed agent update, config bundle or OS step after a loss of DooPlex. Each private key and its .pub;
# refuses when one is missing or is a link (a missing key would otherwise go unnoticed until the day it is needed).
for k in $SIGNING_KEYS; do
  for f in "$k" "$k.pub"; do
    [ -f "$SIGNING/$f" ] && [ ! -L "$SIGNING/$f" ] || die "signing key file $f missing (or a link) in $SIGNING"
    cp "$SIGNING/$f" "$STAGE/root/signing/$f"
  done
done
log "signing: $(echo $SIGNING_KEYS | wc -w | tr -d ' ') key(s) with their public halves"

# 4. the manifest the restore test checks, then the push
echo "$GOT" > "$STAGE/root/REPOS"
(cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \
  || die "writing the manifest"
[ "$(wc -l < "$STAGE/root/MANIFEST.sha256")" -gt "$GOT" ] || die "the manifest is short"
START=$(date +%s)
PBS_PASSWORD_FILE="$TOKENS/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
  proxmox-backup-client backup dooplex.pxar:"$STAGE/root" --ns operator --backup-type host --backup-id dooplex-gitea \
  --keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
  || die "proxmox-backup-client backup"
BYTES=$(du -sb "$STAGE/root" | cut -f1)
log "pushed to ep0 (ns operator, host/dooplex-gitea) in $(( $(date +%s) - START )) s"

TMP="$TEXTFILE_DIR/felhom_dooplex_offsite.prom.$$"
{
  echo "# HELP felhom_dooplex_offsite_last_success_timestamp_seconds Last successful push of Gitea + DooPlex secrets to ep0 (R-232)."
  echo "# TYPE felhom_dooplex_offsite_last_success_timestamp_seconds gauge"
  echo "felhom_dooplex_offsite_last_success_timestamp_seconds $(date +%s)"
  echo "felhom_dooplex_offsite_last_success_bytes $BYTES"
  echo "felhom_dooplex_offsite_last_success_repositories $GOT"
  echo "felhom_dooplex_offsite_last_success_vaultwarden_users $VUSERS"
} > "$TMP"
chmod 644 "$TMP"
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite.prom"
log "success signal written"