scripts/hub-db-backup: DooPlex push (02:30) + weekly restore test (Sun 04:30) of the hub DB to ep0 (R-173, R-231); 15 tests, red-proofs P1-P9; Part A/B evidence

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

scripts/hub-db-backup/felhom-hub-db-backup

@@ -0,0 +1,65 @@
+#!/bin/sh
+# felhom-hub-db-backup — push the hub's newest nightly snapshot to ep0's PBS, encrypted (R-173, decision A).
+# Runs on DooPlex as root from felhom-hub-db-backup.timer (02:30; the hub writes the snapshot at 02:00).
+# Runbook: documentation/runbooks/RUNBOOK-hub-db-offsite-backup.md Step 4. Pinned by test_hub_db_backup.py.
+#
+# Refuses to push — and so never writes the success signal — when: no snapshot exists; the newest is older than
+# MAX_AGE_H (the hub stopped snapshotting: pushing yesterday's copy again would read as success); the copied bytes
+# differ in size from the pod's file; PRAGMA integrity_check is not "ok"; the copy holds no hosts. The success
+# timestamp is written ONLY after the push returns 0 (CLAUDE.md "presence is not success").
+set -eu
+CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup}
+STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup}
+TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
+MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-26}
+NOW=${FELHOM_HUBBK_NOW:-$(date +%s)}
+. "$CONF/env"   # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
+
+log() { echo "felhom-hub-db-backup: $*"; }
+die() { echo "felhom-hub-db-backup: FAILED: $*" >&2; exit 1; }
+
+umask 077
+STAGE="$STATE/stage"
+mkdir -p "$STAGE"; chmod 700 "$STATE" "$STAGE"
+rm -f "$STAGE"/*
+trap 'if [ -f "$STAGE/hub.db" ]; then shred -u "$STAGE/hub.db" 2>/dev/null || rm -f "$STAGE/hub.db"; fi' EXIT
+
+SNAP=$(kubectl -n felhom-system exec deploy/hub -- sh -c 'ls -1 /data/snapshots/hub-*.db 2>/dev/null | tail -n 1') || die "listing snapshots in the hub pod"
+[ -n "$SNAP" ] || die "no snapshot in the hub pod's /data/snapshots"
+NAME=${SNAP##*/}
+STAMP=${NAME#hub-}; STAMP=${STAMP%.db}            # 20261005T020000Z
+case "$STAMP" in [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z) ;; *) die "unexpected snapshot name $NAME" ;; esac
+ISO=$(echo "$STAMP" | sed -E 's/^(....)(..)(..)T(..)(..)(..)Z$/\1-\2-\3T\4:\5:\6Z/')
+SNAP_EPOCH=$(date -u -d "$ISO" +%s) || die "cannot parse snapshot time $ISO"
+AGE=$((NOW - SNAP_EPOCH))
+[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest snapshot $NAME is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h) — the hub stopped snapshotting"
+log "snapshot $NAME, $((AGE / 60)) min old"
+
+WANT=$(kubectl -n felhom-system exec deploy/hub -- sh -c "wc -c < '$SNAP'" | tr -d ' \r\n') || die "sizing $NAME"
+kubectl -n felhom-system exec deploy/hub -- cat "$SNAP" > "$STAGE/hub.db" || die "copying $NAME out of the pod"
+GOT=$(wc -c < "$STAGE/hub.db" | tr -d ' ')
+[ "$GOT" = "$WANT" ] || die "copy is $GOT bytes, the pod's file is $WANT"
+
+IC=$(sqlite3 -readonly "$STAGE/hub.db" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
+[ "$IC" = "ok" ] || die "integrity_check: $IC"
+HOSTS=$(sqlite3 -readonly "$STAGE/hub.db" 'SELECT COUNT(*) FROM hosts;' 2>/dev/null) || die "cannot count hosts"
+[ "${HOSTS:-0}" -gt 0 ] || die "the copy holds no hosts"
+log "checked: $GOT bytes, integrity ok, $HOSTS host(s)"
+
+START=$(date +%s)
+PBS_PASSWORD_FILE="$CONF/token-push" PBS_FINGERPRINT="$PBS_FINGERPRINT" \
+  proxmox-backup-client backup hubdb.pxar:"$STAGE" --ns operator --backup-type host --backup-id dooplex-hub \
+  --keyfile "$CONF/enc.key" --crypt-mode encrypt --repository "$PBS_REPOSITORY_PUSH" \
+  || die "proxmox-backup-client backup"
+log "pushed $NAME to ep0 (ns operator) in $(( $(date +%s) - START )) s"
+
+TMP="$TEXTFILE_DIR/felhom_hub_db_backup.prom.$$"
+{
+  echo "# HELP felhom_hub_db_backup_last_success_timestamp_seconds Last successful push of the hub DB snapshot to ep0 (R-173)."
+  echo "# TYPE felhom_hub_db_backup_last_success_timestamp_seconds gauge"
+  echo "felhom_hub_db_backup_last_success_timestamp_seconds $(date +%s)"
+  echo "felhom_hub_db_backup_last_success_bytes $GOT"
+} > "$TMP"
+chmod 644 "$TMP"
+mv "$TMP" "$TEXTFILE_DIR/felhom_hub_db_backup.prom"
+log "success signal written"

scripts/hub-db-backup/felhom-hub-db-backup.service

@@ -0,0 +1,19 @@
+# Versioned in felhom.eu/scripts/hub-db-backup/ (R-231); installed by install.sh. Runbook: RUNBOOK-hub-db-offsite-backup.md.
+[Unit]
+Description=Felhom: push the hub DB snapshot to ep0 (R-173)
+Wants=network-online.target felhom-ep0-pbs-tunnel.service
+After=network-online.target felhom-ep0-pbs-tunnel.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/felhom-hub-db-backup
+Environment=HOME=/var/lib/felhom-hub-backup KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+UMask=0077
+TimeoutStartSec=45min
+Nice=10
+IOSchedulingClass=idle
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/var/lib/felhom-hub-backup /var/lib/node_exporter/textfile_collector
+NoNewPrivileges=yes

scripts/hub-db-backup/felhom-hub-db-backup.timer

@@ -0,0 +1,11 @@
+# Versioned in felhom.eu/scripts/hub-db-backup/ (R-231); installed by install.sh.
+[Unit]
+Description=Felhom: push the hub DB snapshot to ep0 (R-173) — schedule
+
+[Timer]
+OnCalendar=*-*-* 02:30:00
+Persistent=true
+RandomizedDelaySec=2min
+
+[Install]
+WantedBy=timers.target

scripts/hub-db-backup/felhom-hub-db-restore-test

@@ -0,0 +1,64 @@
+#!/bin/sh
+# felhom-hub-db-restore-test — restore the newest hub DB copy from ep0 with the READ-ONLY token and check it (R-173).
+# Runs on DooPlex as root from felhom-hub-db-restore-test.timer (Sun 04:30). Runbook Step 5. Pinned by
+# test_hub_db_backup.py.
+#
+# The success timestamp is written ONLY when: the newest copy on ep0 is at most MAX_AGE_H old; it restores and
+# decrypts; PRAGMA integrity_check is "ok"; it holds at least one host; and NO console password is stored readable
+# (every non-empty host_recovery.secret starts with "enc:v1:", the hub's seal, 05 §16.2).
+set -eu
+CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup}
+STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup}
+TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
+MAX_AGE_H=${FELHOM_HUBBK_RESTORE_MAX_AGE_H:-50}
+NOW=${FELHOM_HUBBK_NOW:-$(date +%s)}
+. "$CONF/env"   # PBS_REPOSITORY_RESTORE, PBS_FINGERPRINT
+
+log() { echo "felhom-hub-db-restore-test: $*"; }
+die() { echo "felhom-hub-db-restore-test: FAILED: $*" >&2; exit 1; }
+
+umask 077
+mkdir -p "$STATE"; chmod 700 "$STATE"
+T=$(mktemp -d "$STATE/restore.XXXXXX")
+trap 'find "$T" -type f -exec shred -u {} + 2>/dev/null; rm -rf "$T"' EXIT
+export PBS_PASSWORD_FILE="$CONF/token-restore" PBS_FINGERPRINT
+
+LIST=$(proxmox-backup-client snapshot list host/dooplex-hub --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
+  || die "listing snapshots on ep0"
+NEWEST=$(printf '%s' "$LIST" | python3 -c '
+import json, sys
+s = [x for x in json.load(sys.stdin) if x.get("backup-type") == "host" and x.get("backup-id") == "dooplex-hub"]
+if s:
+    n = max(s, key=lambda x: x["backup-time"])
+    print(n["backup-time"])
+') || die "reading the snapshot list"
+[ -n "$NEWEST" ] || die "no hub DB copy on ep0"
+AGE=$((NOW - NEWEST))
+[ "$AGE" -le $((MAX_AGE_H * 3600)) ] || die "newest copy on ep0 is $((AGE / 3600)) h old (limit ${MAX_AGE_H} h)"
+SNAPSHOT="host/dooplex-hub/$(date -u -d "@$NEWEST" +%Y-%m-%dT%H:%M:%SZ)"
+log "restoring $SNAPSHOT"
+
+proxmox-backup-client restore "$SNAPSHOT" hubdb.pxar "$T/out" --ns operator \
+  --keyfile "$CONF/enc.key" --repository "$PBS_REPOSITORY_RESTORE" || die "restore of $SNAPSHOT"
+DB="$T/out/hub.db"
+[ -s "$DB" ] || die "the restored archive holds no hub.db"
+
+IC=$(sqlite3 -readonly "$DB" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
+[ "$IC" = "ok" ] || die "integrity_check: $IC"
+HOSTS=$(sqlite3 -readonly "$DB" 'SELECT COUNT(*) FROM hosts;' 2>/dev/null) || die "cannot count hosts"
+[ "${HOSTS:-0}" -gt 0 ] || die "the restored copy holds no hosts"
+PLAIN=$(sqlite3 -readonly "$DB" "SELECT COUNT(*) FROM host_recovery WHERE COALESCE(secret,'') <> '' AND secret NOT LIKE 'enc:v1:%';" 2>/dev/null) \
+  || die "cannot read host_recovery"
+[ "$PLAIN" -eq 0 ] || die "$PLAIN console password(s) stored readable"
+SEALED=$(sqlite3 -readonly "$DB" "SELECT COUNT(*) FROM host_recovery WHERE secret LIKE 'enc:v1:%';")
+log "checked: integrity ok, $HOSTS host(s), $SEALED sealed console password(s), 0 readable"
+
+TMP="$TEXTFILE_DIR/felhom_hub_db_restore.prom.$$"
+{
+  echo "# HELP felhom_hub_db_restore_test_last_success_timestamp_seconds Last successful restore test of the hub DB copy on ep0 (R-173)."
+  echo "# TYPE felhom_hub_db_restore_test_last_success_timestamp_seconds gauge"
+  echo "felhom_hub_db_restore_test_last_success_timestamp_seconds $(date +%s)"
+} > "$TMP"
+chmod 644 "$TMP"
+mv "$TMP" "$TEXTFILE_DIR/felhom_hub_db_restore.prom"
+log "success signal written"

scripts/hub-db-backup/felhom-hub-db-restore-test.service

@@ -0,0 +1,19 @@
+# Versioned in felhom.eu/scripts/hub-db-backup/ (R-231); installed by install.sh. Runbook: RUNBOOK-hub-db-offsite-backup.md.
+[Unit]
+Description=Felhom: restore-test the hub DB copy on ep0 (R-173)
+Wants=network-online.target felhom-ep0-pbs-tunnel.service
+After=network-online.target felhom-ep0-pbs-tunnel.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/felhom-hub-db-restore-test
+Environment=HOME=/var/lib/felhom-hub-backup KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+UMask=0077
+TimeoutStartSec=45min
+Nice=10
+IOSchedulingClass=idle
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/var/lib/felhom-hub-backup /var/lib/node_exporter/textfile_collector
+NoNewPrivileges=yes

scripts/hub-db-backup/felhom-hub-db-restore-test.timer

@@ -0,0 +1,11 @@
+# Versioned in felhom.eu/scripts/hub-db-backup/ (R-231); installed by install.sh.
+[Unit]
+Description=Felhom: restore-test the hub DB copy on ep0 (R-173) — schedule
+
+[Timer]
+OnCalendar=Sun *-*-* 04:30:00
+Persistent=true
+RandomizedDelaySec=2min
+
+[Install]
+WantedBy=timers.target

scripts/hub-db-backup/install.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# install.sh — install the hub DB off-site backup units on DooPlex (R-173). Root. Idempotent.
+# Installs the two scripts and four units, writes /etc/felhom-hub-backup/env (no secrets) when absent, and does NOT
+# enable the timers — enable them by hand after the first manual run (runbook Step 7):
+#   systemctl enable --now felhom-hub-db-backup.timer felhom-hub-db-restore-test.timer
+# The tokens (token-push, token-restore) and enc.key are created separately, file to file, never by this script.
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+[ "$(id -u)" = 0 ] || { echo "install.sh: run as root" >&2; exit 1; }
+install -m 0755 "$HERE/felhom-hub-db-backup" /usr/local/sbin/felhom-hub-db-backup
+install -m 0755 "$HERE/felhom-hub-db-restore-test" /usr/local/sbin/felhom-hub-db-restore-test
+for u in felhom-hub-db-backup.service felhom-hub-db-backup.timer felhom-hub-db-restore-test.service felhom-hub-db-restore-test.timer; do
+  install -m 0644 "$HERE/$u" "/etc/systemd/system/$u"
+done
+install -d -m 0700 /etc/felhom-hub-backup /var/lib/felhom-hub-backup
+if [ ! -f /etc/felhom-hub-backup/env ]; then
+  umask 077
+  cat > /etc/felhom-hub-backup/env <<'ENV'
+# Not secret. The tokens are in token-push / token-restore (0600), the key in enc.key (0600).
+PBS_REPOSITORY_PUSH='dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite'
+PBS_REPOSITORY_RESTORE='dooplex-hub@pbs!restore@127.0.0.1:18007:felhom-offsite'
+# ep0's PBS certificate, the same pin DooPlex's PBS remote "ep0" uses (/etc/proxmox-backup/remote.cfg)
+PBS_FINGERPRINT='c6:07:28:3f:5b:7b:5a:41:90:28:d7:ca:4f:37:14:70:56:39:2e:2f:0b:71:e8:06:ca:60:4a:d5:56:5f:3c:fd'
+ENV
+fi
+systemctl daemon-reload
+echo "install.sh: installed; timers NOT enabled (see the header)"

scripts/hub-db-backup/test_hub_db_backup.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""Tests for felhom-hub-db-backup and felhom-hub-db-restore-test (R-173).
+
+No test reaches the hub, PBS or ep0: `kubectl` and `proxmox-backup-client` are fakes on PATH (the pod's
+/data/snapshots is a temp dir; the PBS "server" is a temp dir). `sqlite3`, `date`, `shred` are the real tools.
+Each test asserts the CONSEQUENCE: whether a push happened and whether the success signal (the file the alarm reads)
+was written. Run: python3 scripts/hub-db-backup/test_hub_db_backup.py
+"""
+import json
+import os
+import shutil
+import sqlite3
+import stat
+import subprocess
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PUSH = os.path.join(HERE, "felhom-hub-db-backup")
+RESTORE = os.path.join(HERE, "felhom-hub-db-restore-test")
+
+FAKE_KUBECTL = r'''#!/usr/bin/env python3
+import os, subprocess, sys
+a = sys.argv[1:]
+pod = os.environ["FAKE_POD_DATA"]
+i = a.index("--")
+cmd = a[i + 1:]
+def m(p): return p.replace("/data", pod, 1)
+if cmd[:2] == ["sh", "-c"]:
+    sys.exit(subprocess.call(["sh", "-c", cmd[2].replace("/data", pod)]))
+if cmd[0] == "cat":
+    if os.environ.get("FAKE_TRUNCATE"):
+        data = open(m(cmd[1]), "rb").read()
+        sys.stdout.buffer.write(data[: len(data) // 2]); sys.exit(0)
+    sys.exit(subprocess.call(["cat", m(cmd[1])]))
+sys.exit(97)
+'''
+
+FAKE_PBS = r'''#!/usr/bin/env python3
+import json, os, shutil, sys, time
+a = sys.argv[1:]
+srv = os.environ["FAKE_PBS_DIR"]
+open(os.path.join(srv, "calls.log"), "a").write(json.dumps({"argv": a, "pw": os.environ.get("PBS_PASSWORD_FILE", ""), "fp": os.environ.get("PBS_FINGERPRINT", "")}) + "\n")
+if a[0] == "backup":
+    if os.environ.get("FAKE_PBS_FAIL"): sys.exit(1)
+    src = a[1].split(":", 1)[1]
+    t = int(time.time())
+    d = os.path.join(srv, "snaps", str(t)); os.makedirs(d, exist_ok=True)
+    shutil.copy(os.path.join(src, "hub.db"), os.path.join(d, "hub.db"))
+    sys.exit(0)
+if a[0] == "snapshot" and a[1] == "list":
+    base = os.path.join(srv, "snaps")
+    out = [{"backup-type": "host", "backup-id": "dooplex-hub", "backup-time": int(x)} for x in (os.listdir(base) if os.path.isdir(base) else [])]
+    print(json.dumps(out)); sys.exit(0)
+if a[0] == "restore":
+    if os.environ.get("FAKE_PBS_FAIL"): sys.exit(1)
+    import calendar
+    t = calendar.timegm(time.strptime(a[1].split("/")[-1], "%Y-%m-%dT%H:%M:%SZ"))
+    os.makedirs(a[3], exist_ok=True)
+    shutil.copy(os.path.join(srv, "snaps", str(t), "hub.db"), os.path.join(a[3], "hub.db"))
+    sys.exit(0)
+sys.exit(98)
+'''
+
+
+def make_db(path, hosts=2, recovery=("enc:v1:abc", "enc:v1:def"), corrupt=False):
+    db = sqlite3.connect(path)
+    db.execute("CREATE TABLE hosts (host_id TEXT)")
+    db.execute("CREATE TABLE host_recovery (host_id TEXT, secret TEXT)")
+    db.executemany("INSERT INTO hosts VALUES (?)", [("h%d" % i,) for i in range(hosts)])
+    db.executemany("INSERT INTO host_recovery VALUES ('h', ?)", [(r,) for r in recovery])
+    db.execute("CREATE TABLE filler (x BLOB)")
+    db.executemany("INSERT INTO filler VALUES (randomblob(3000))", [()] * 40)
+    db.execute("CREATE INDEX filler_x ON filler(x)")
+    db.commit(); db.close()
+    if corrupt:  # overwrite a late page (index b-tree) so integrity_check reports errors but the file still opens
+        size = os.path.getsize(path)
+        with open(path, "r+b") as f:
+            f.seek(size - 4096 + 100); f.write(b"\xff" * 2000)
+
+
+def stamp(epoch):
+    return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(epoch))
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        self.t = tempfile.mkdtemp()
+        j = lambda *p: os.path.join(self.t, *p)
+        for d in ("bin", "pod/snapshots", "conf", "state", "textfile", "pbs"):
+            os.makedirs(j(d), exist_ok=True)
+        for name, body in (("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS)):
+            p = j("bin", name); open(p, "w").write(body); os.chmod(p, 0o755)
+        open(j("conf", "env"), "w").write(
+            "PBS_REPOSITORY_PUSH='dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite'\n"
+            "PBS_REPOSITORY_RESTORE='dooplex-hub@pbs!restore@127.0.0.1:18007:felhom-offsite'\n"
+            "PBS_FINGERPRINT='aa:bb'\n")
+        for f in ("token-push", "token-restore", "enc.key"):
+            open(j("conf", f), "w").write("x")
+        self.env = dict(os.environ, PATH=j("bin") + ":/usr/bin:/bin", FAKE_POD_DATA=j("pod"), FAKE_PBS_DIR=j("pbs"),
+                        FELHOM_HUBBK_CONF=j("conf"), FELHOM_HUBBK_STATE=j("state"), FELHOM_HUBBK_TEXTFILE_DIR=j("textfile"))
+        self.j = j
+
+    def tearDown(self):
+        shutil.rmtree(self.t, ignore_errors=True)
+
+    def snapshot(self, age_s=600, **kw):
+        p = self.j("pod", "snapshots", "hub-%s.db" % stamp(int(time.time()) - age_s))
+        make_db(p, **kw)
+        return p
+
+    def run_script(self, script, **extra):
+        env = dict(self.env, **extra)
+        return subprocess.run([script], env=env, capture_output=True, text=True, timeout=60)
+
+    def pushed(self):
+        d = self.j("pbs", "snaps")
+        return sorted(os.listdir(d)) if os.path.isdir(d) else []
+
+    def signal(self, name):
+        return os.path.exists(self.j("textfile", name))
+
+    def calls(self):
+        p = self.j("pbs", "calls.log")
+        return [json.loads(l) for l in open(p)] if os.path.exists(p) else []
+
+
+class Push(Base):
+    def test_happy_path_pushes_encrypted_to_operator_and_writes_signal(self):
+        self.snapshot()
+        r = self.run_script(PUSH)
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(len(self.pushed()), 1)
+        self.assertTrue(self.signal("felhom_hub_db_backup.prom"))
+        c = [x for x in self.calls() if x["argv"][0] == "backup"][0]
+        a = c["argv"]
+        for flag in ("--ns", "--backup-id", "--crypt-mode", "--keyfile", "--repository"):
+            self.assertIn(flag, a, "push without %s" % flag)
+        for flag, val in (("--ns", "operator"), ("--backup-id", "dooplex-hub"), ("--crypt-mode", "encrypt")):
+            self.assertEqual(a[a.index(flag) + 1], val)
+        self.assertTrue(a[a.index("--keyfile") + 1].endswith("/enc.key"))
+        self.assertIn("!push@", a[a.index("--repository") + 1])
+        self.assertTrue(c["pw"].endswith("/token-push"))
+        self.assertNotIn("x", " ".join(a).split())  # the token's value never on the command line
+        self.assertEqual(os.listdir(self.j("state", "stage")), [], "the staged plaintext copy is left behind")
+        txt = open(self.j("textfile", "felhom_hub_db_backup.prom")).read()
+        self.assertIn("felhom_hub_db_backup_last_success_timestamp_seconds ", txt)
+
+    def test_corrupt_copy_is_never_pushed(self):
+        self.snapshot(corrupt=True)
+        r = self.run_script(PUSH)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("integrity_check", r.stderr)
+        self.assertEqual(self.pushed(), [])
+        self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
+
+    def test_stale_snapshot_is_not_pushed_again(self):
+        self.snapshot(age_s=27 * 3600)
+        r = self.run_script(PUSH)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("stopped snapshotting", r.stderr)
+        self.assertEqual(self.pushed(), [])
+        self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
+
+    def test_truncated_copy_is_not_pushed(self):
+        self.snapshot()
+        r = self.run_script(PUSH, FAKE_TRUNCATE="1")
+        self.assertNotEqual(r.returncode, 0)
+        # the SIZE guard must be the one that refuses (half a file usually fails integrity_check too, which would
+        # mask a missing size check — red-proof P4's first run did exactly that)
+        self.assertIn("bytes, the pod's file is", r.stderr)
+        self.assertEqual(self.pushed(), [])
+        self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
+
+    def test_failed_push_writes_no_signal(self):
+        self.snapshot()
+        r = self.run_script(PUSH, FAKE_PBS_FAIL="1")
+        self.assertNotEqual(r.returncode, 0)
+        self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
+        self.assertEqual(os.listdir(self.j("state", "stage")), [])
+
+    def test_no_snapshot_fails(self):
+        r = self.run_script(PUSH)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
+
+    def test_empty_hosts_is_not_pushed(self):
+        self.snapshot(hosts=0)
+        r = self.run_script(PUSH)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertEqual(self.pushed(), [])
+
+    def test_newest_snapshot_is_the_one_pushed(self):
+        self.snapshot(age_s=25 * 3600, hosts=1)
+        self.snapshot(age_s=600, hosts=3)
+        self.assertEqual(self.run_script(PUSH).returncode, 0)
+        db = os.path.join(self.j("pbs", "snaps"), self.pushed()[0], "hub.db")
+        self.assertEqual(sqlite3.connect(db).execute("SELECT COUNT(*) FROM hosts").fetchone()[0], 3)
+
+
+class RestoreTest(Base):
+    def push_one(self, **kw):
+        self.snapshot(**kw)
+        r = self.run_script(PUSH)
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+    def test_happy_path_writes_signal_with_the_read_only_token(self):
+        self.push_one()
+        r = self.run_script(RESTORE)
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertTrue(self.signal("felhom_hub_db_restore.prom"))
+        for c in self.calls():
+            if c["argv"][0] in ("restore", "snapshot"):
+                self.assertTrue(c["pw"].endswith("/token-restore"), c)
+                self.assertIn("!restore@", c["argv"][c["argv"].index("--repository") + 1])
+        self.assertEqual([x for x in os.listdir(self.j("state")) if x.startswith("restore.")], [], "restored copy left behind")
+
+    def test_readable_console_password_fails(self):
+        self.push_one(recovery=("enc:v1:abc", "hunter2"))
+        r = self.run_script(RESTORE)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("stored readable", r.stderr)
+        self.assertNotIn("hunter2", r.stderr + r.stdout)
+        self.assertFalse(self.signal("felhom_hub_db_restore.prom"))
+
+    def test_no_copy_on_ep0_fails(self):
+        r = self.run_script(RESTORE)
+        self.assertNotEqual(r.returncode, 0)
+        self.assertFalse(self.signal("felhom_hub_db_restore.prom"))
+
+    def test_old_copy_fails(self):
+        self.push_one()
+        r = self.run_script(RESTORE, FELHOM_HUBBK_NOW=str(int(time.time()) + 51 * 3600))
+        self.assertNotEqual(r.returncode, 0)
+        self.assertFalse(self.signal("felhom_hub_db_restore.prom"))
+
+    def test_failed_restore_fails(self):
+        self.push_one()
+        r = self.run_script(RESTORE, FAKE_PBS_FAIL="1")
+        self.assertNotEqual(r.returncode, 0)
+        self.assertFalse(self.signal("felhom_hub_db_restore.prom"))
+
+
+class Units(unittest.TestCase):
+    def read(self, n):
+        return open(os.path.join(HERE, n)).read()
+
+    def test_schedules(self):
+        self.assertIn("OnCalendar=*-*-* 02:30:00", self.read("felhom-hub-db-backup.timer"))
+        self.assertIn("OnCalendar=Sun *-*-* 04:30:00", self.read("felhom-hub-db-restore-test.timer"))
+
+    def test_units_run_the_installed_scripts(self):
+        self.assertIn("ExecStart=/usr/local/sbin/felhom-hub-db-backup\n", self.read("felhom-hub-db-backup.service"))
+        self.assertIn("ExecStart=/usr/local/sbin/felhom-hub-db-restore-test\n", self.read("felhom-hub-db-restore-test.service"))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)