#!/bin/sh
# felhom-dooplex-offsite-restore-test — restore the newest Gitea + secrets copy from ep0 with the READ-ONLY token and
# check it (R-232 (h)). Runs on DooPlex as root from felhom-dooplex-offsite-restore-test.timer (Sun 05:30).
# Pinned by test_dooplex_offsite.py.
#
# The success timestamp is written ONLY when: the newest copy on ep0 is at most MAX_AGE_H old; it restores and
# decrypts; every file matches MANIFEST.sha256; the repository count matches REPOS; `git fsck` passes on EVERY
# repository; `pg_restore --list` reads gitea.dump; app.ini is there; at least one secrets file is there.
set -eu
CONF=${FELHOM_DXOFF_CONF:-/etc/felhom-dooplex-offsite}
TOKENS=${FELHOM_DXOFF_TOKENS:-/etc/felhom-hub-backup}
STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
MAX_AGE_H=${FELHOM_DXOFF_RESTORE_MAX_AGE_H:-50}
NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
. "$CONF/env"   # PBS_REPOSITORY_RESTORE, PBS_FINGERPRINT

log() { echo "felhom-dooplex-offsite-restore-test: $*"; }
die() { echo "felhom-dooplex-offsite-restore-test: FAILED: $*" >&2; exit 1; }

umask 077
mkdir -p "$STATE"; chmod 700 "$STATE"
T=$(mktemp -d "$STATE/restore.XXXXXX")
trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql "$T"/out/vaultwarden/db.sqlite3 "$T"/out/vaultwarden/rsa_key.pem "$T"/out/signing/felhom-op-operational "$T"/out/signing/felhom-rec-recovery "$T"/out/signing/felhom_op_ed25519; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
export PBS_PASSWORD_FILE="$TOKENS/token-restore" PBS_FINGERPRINT

LIST=$(proxmox-backup-client snapshot list host/dooplex-gitea --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
  || die "listing snapshots on ep0"
NEWEST=$(printf '%s' "$LIST" | python3 -c '
import json, sys
s = [x for x in json.load(sys.stdin) if x.get("backup-type") == "host" and x.get("backup-id") == "dooplex-gitea"]
if s:
    print(max(x["backup-time"] for x in s))
') || die "reading the snapshot list"
[ -n "$NEWEST" ] || die "no Gitea copy on ep0"
AGE=$((NOW - NEWEST))
[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest copy on ep0 is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h)"
SNAPSHOT="host/dooplex-gitea/$(date -u -d "@$NEWEST" +%Y-%m-%dT%H:%M:%SZ)"
log "restoring $SNAPSHOT"

proxmox-backup-client restore "$SNAPSHOT" dooplex.pxar "$T/out" --ns operator \
  --keyfile "$CONF/enc.key" --repository "$PBS_REPOSITORY_RESTORE" || die "restore of $SNAPSHOT"
O="$T/out"
[ -s "$O/MANIFEST.sha256" ] || die "the copy holds no MANIFEST.sha256"
(cd "$O" && sha256sum -c MANIFEST.sha256 >/dev/null) || die "a file does not match MANIFEST.sha256"
FILES=$(wc -l < "$O/MANIFEST.sha256" | tr -d ' ')

WANT=$(cat "$O/REPOS" 2>/dev/null) || die "the copy holds no REPOS count"
REPOS=$(find "$O/gitea/git/repositories" -mindepth 2 -maxdepth 2 -type d -name '*.git' | sort)
GOT=$(printf '%s\n' "$REPOS" | grep -c '\.git$') || GOT=0
[ "$GOT" -gt 0 ] && [ "$GOT" = "$WANT" ] || die "the copy holds $GOT repositories, REPOS says $WANT"
# The repositories' own `config` files came from the Gitea pod — untrusted input to a root git. So git never reads
# them: each repository is checked through a fresh scratch repository with a known config, holding a copy of its
# HEAD and refs, with the copy's objects as its object store. No system or global config either.
export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null
for r in $REPOS; do
  S="$T/fsck.git"; rm -rf "$S"
  git init -q --bare "$S" || die "git init for the fsck scratch repository"
  cp "$r/HEAD" "$S/HEAD"; [ -f "$r/packed-refs" ] && cp "$r/packed-refs" "$S/packed-refs"
  [ -d "$r/refs" ] && cp -R "$r/refs/." "$S/refs/"
  GIT_OBJECT_DIRECTORY="$r/objects" git --git-dir="$S" -c core.hooksPath=/dev/null -c core.fsmonitor=false \
    fsck --no-progress --no-dangling >/dev/null 2>"$T/fsck.err" \
    || die "git fsck ${r#"$O"/gitea/git/repositories/}: $(head -n 3 "$T/fsck.err")"
done
rm -rf "$T/fsck.git"
[ -s "$O/gitea/gitea/conf/app.ini" ] || die "app.ini missing"
pg_restore --list "$O/db/gitea.dump" >/dev/null || die "pg_restore cannot read gitea.dump"
ls "$O"/secrets/*.gpg >/dev/null 2>&1 || die "no secrets file in the copy"
# Vaultwarden (R-923): rows, never contents. Copies made before 2026-10-09 midday have no vaultwarden/ — refused, since
# the newest copy is the one tested and every copy since then carries it.
VDB="$O/vaultwarden/db.sqlite3"
[ -s "$VDB" ] || die "no Vaultwarden database in the copy"
VIC=$(sqlite3 -readonly "$VDB" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
VUSERS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
VITEMS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM ciphers;' 2>/dev/null) || die "cannot count Vaultwarden items"
[ "${VUSERS:-0}" -gt 0 ] && [ "$VUSERS" = "$(cat "$O/vaultwarden/USERS" 2>/dev/null)" ] || die "Vaultwarden users: $VUSERS, USERS says $(cat "$O/vaultwarden/USERS" 2>/dev/null)"
[ "${VITEMS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no item"
# The signing keys (R-924): each private key must still derive its own public half — a usable key, never printed.
NSIGN=0
for k in felhom-op-operational felhom-rec-recovery felhom_op_ed25519; do
  [ -s "$O/signing/$k" ] && [ -s "$O/signing/$k.pub" ] || die "signing key $k missing from the copy"
  DER=$(ssh-keygen -y -P '' -f "$O/signing/$k" 2>/dev/null | cut -d' ' -f1,2) || die "signing key $k does not open"
  [ -n "$DER" ] && [ "$DER" = "$(cut -d' ' -f1,2 "$O/signing/$k.pub")" ] || die "signing key $k does not match its public half"
  NSIGN=$((NSIGN + 1))
done
log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s), Vaultwarden $VUSERS user(s) / $VITEMS item(s), $NSIGN signing key(s) match"

TMP="$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom.$$"
{
  echo "# HELP felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds Last successful restore test of the Gitea + secrets copy on ep0 (R-232)."
  echo "# TYPE felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds gauge"
  echo "felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds $(date +%s)"
} > "$TMP"
chmod 644 "$TMP"
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom"
log "success signal written"