dooplex-offsite: Vaultwarden (the password manager) rides the nightly encrypted copy (R-923); R-923 filed (operator finding); R-922 ruling A recorded
gates / gates (push) Successful in 5m32s
gates / gates (push) Successful in 5m32s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config) and DooPlex's nightly secrets export to
|
||||
# felhom-dooplex-offsite — push Gitea (repositories, database dump, config), Vaultwarden (R-923) and DooPlex's nightly secrets export to
|
||||
# ep0's PBS, encrypted on DooPlex (R-232 (b)). Runs on DooPlex as root from felhom-dooplex-offsite.timer (00:20).
|
||||
# Plan: documentation/audits/dooplex-survival-2026-10-09/PLAN.md. Restore: documentation/runbooks/gitea-restore.md.
|
||||
# Pinned by test_dooplex_offsite.py.
|
||||
@@ -27,13 +27,15 @@ RETRY_SLEEP=${FELHOM_DXOFF_RETRY_SLEEP:-30}
|
||||
log() { echo "felhom-dooplex-offsite: $*"; }
|
||||
die() { echo "felhom-dooplex-offsite: FAILED: $*" >&2; exit 1; }
|
||||
K() { kubectl -n gitea-system exec deploy/gitea -c gitea -- "$@"; }
|
||||
V() { kubectl -n vaultwarden-system exec deploy/vaultwarden -- "$@"; }
|
||||
|
||||
umask 077
|
||||
STAGE="$STATE/stage"
|
||||
mkdir -p "$STATE"; chmod 700 "$STATE"
|
||||
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets"
|
||||
rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets" "$STAGE/root/vaultwarden"
|
||||
cleanup() {
|
||||
for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql"; do
|
||||
for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql" \
|
||||
"$STAGE/root/vaultwarden/db.sqlite3" "$STAGE/root/vaultwarden/rsa_key.pem"; do
|
||||
[ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
|
||||
done
|
||||
rm -rf "$STAGE"
|
||||
@@ -83,6 +85,34 @@ SAGE=$((NOW - $(stat -c %Y "$NEWEST")))
|
||||
cp "$SECRETS"/*-"$STAMP".*gpg "$STAGE/root/secrets/"
|
||||
log "secrets: $(ls "$STAGE/root/secrets" | wc -l | tr -d ' ') file(s) of $STAMP"
|
||||
|
||||
# 3b. the password manager (Vaultwarden, R-923) — its own `vaultwarden backup` (SQLite VACUUM INTO, consistent while it
|
||||
# runs) writes ONE file into /data; it is copied out and that one file removed. Plus rsa_key.pem and, when they exist,
|
||||
# attachments/, sends/, config.json (Vaultwarden's backup guidance). The vault items are encrypted under each user's
|
||||
# master password; this job never opens them. Refuses on: the backup command failing, an unexpected file name, a
|
||||
# failed integrity_check, no user.
|
||||
VOUT=$(V /vaultwarden backup 2>&1) || die "vaultwarden backup: $(printf '%s' "$VOUT" | tail -n 1)"
|
||||
VFILE=$(printf '%s\n' "$VOUT" | sed -n "s#^Backup to '\(.*\)' was successful.*#\1#p" | tail -n 1)
|
||||
case "$VFILE" in data/db_[0-9]*_[0-9]*.sqlite3) ;; *) die "vaultwarden backup: unexpected output (no backup file named)" ;; esac
|
||||
VPATH="/$VFILE"
|
||||
VRC=0; V cat "$VPATH" > "$STAGE/root/vaultwarden/db.sqlite3" || VRC=$?
|
||||
V rm -f "$VPATH" || log "WARNING: could not remove $VPATH from Vaultwarden's /data"
|
||||
[ "$VRC" -eq 0 ] || die "copying $VPATH out of the Vaultwarden pod"
|
||||
VLIST=$(V sh -c 'cd /data && for p in rsa_key.pem rsa_key.pub.pem config.json attachments sends; do if [ -e "$p" ]; then echo "$p"; fi; done') \
|
||||
|| die "listing Vaultwarden's files"
|
||||
[ -n "$VLIST" ] && { V tar -cf - -C /data $VLIST > "$STAGE/vw.tar" || die "copying Vaultwarden's files"; }
|
||||
if [ -s "$STAGE/vw.tar" ]; then
|
||||
LINKS=$(tar -tvf "$STAGE/vw.tar" | grep -c '^[lh]') || LINKS=0
|
||||
[ "$LINKS" -eq 0 ] || die "Vaultwarden's archive holds $LINKS link(s) — refused"
|
||||
tar -xof "$STAGE/vw.tar" -C "$STAGE/root/vaultwarden" || die "unpacking Vaultwarden's files"
|
||||
rm -f "$STAGE/vw.tar"
|
||||
fi
|
||||
VIC=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
|
||||
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
|
||||
VUSERS=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
|
||||
[ "${VUSERS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no user"
|
||||
echo "$VUSERS" > "$STAGE/root/vaultwarden/USERS"
|
||||
log "vaultwarden: integrity ok, $VUSERS user(s), files: db.sqlite3 $(echo $VLIST)"
|
||||
|
||||
# 4. the manifest the restore test checks, then the push
|
||||
echo "$GOT" > "$STAGE/root/REPOS"
|
||||
(cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \
|
||||
@@ -103,6 +133,7 @@ TMP="$TEXTFILE_DIR/felhom_dooplex_offsite.prom.$$"
|
||||
echo "felhom_dooplex_offsite_last_success_timestamp_seconds $(date +%s)"
|
||||
echo "felhom_dooplex_offsite_last_success_bytes $BYTES"
|
||||
echo "felhom_dooplex_offsite_last_success_repositories $GOT"
|
||||
echo "felhom_dooplex_offsite_last_success_vaultwarden_users $VUSERS"
|
||||
} > "$TMP"
|
||||
chmod 644 "$TMP"
|
||||
mv "$TMP" "$TEXTFILE_DIR/felhom_dooplex_offsite.prom"
|
||||
|
||||
@@ -21,7 +21,7 @@ die() { echo "felhom-dooplex-offsite-restore-test: FAILED: $*" >&2; exit 1; }
|
||||
umask 077
|
||||
mkdir -p "$STATE"; chmod 700 "$STATE"
|
||||
T=$(mktemp -d "$STATE/restore.XXXXXX")
|
||||
trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
|
||||
trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql "$T"/out/vaultwarden/db.sqlite3 "$T"/out/vaultwarden/rsa_key.pem; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
|
||||
export PBS_PASSWORD_FILE="$TOKENS/token-restore" PBS_FINGERPRINT
|
||||
|
||||
LIST=$(proxmox-backup-client snapshot list host/dooplex-gitea --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
|
||||
@@ -66,7 +66,17 @@ rm -rf "$T/fsck.git"
|
||||
[ -s "$O/gitea/gitea/conf/app.ini" ] || die "app.ini missing"
|
||||
pg_restore --list "$O/db/gitea.dump" >/dev/null || die "pg_restore cannot read gitea.dump"
|
||||
ls "$O"/secrets/*.gpg >/dev/null 2>&1 || die "no secrets file in the copy"
|
||||
log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s)"
|
||||
# Vaultwarden (R-923): rows, never contents. Copies made before 2026-10-09 midday have no vaultwarden/ — refused, since
|
||||
# the newest copy is the one tested and every copy since then carries it.
|
||||
VDB="$O/vaultwarden/db.sqlite3"
|
||||
[ -s "$VDB" ] || die "no Vaultwarden database in the copy"
|
||||
VIC=$(sqlite3 -readonly "$VDB" 'PRAGMA integrity_check;' 2>&1 | head -n 5) || true
|
||||
[ "$VIC" = "ok" ] || die "Vaultwarden integrity_check: $VIC"
|
||||
VUSERS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM users;' 2>/dev/null) || die "cannot count Vaultwarden users"
|
||||
VITEMS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM ciphers;' 2>/dev/null) || die "cannot count Vaultwarden items"
|
||||
[ "${VUSERS:-0}" -gt 0 ] && [ "$VUSERS" = "$(cat "$O/vaultwarden/USERS" 2>/dev/null)" ] || die "Vaultwarden users: $VUSERS, USERS says $(cat "$O/vaultwarden/USERS" 2>/dev/null)"
|
||||
[ "${VITEMS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no item"
|
||||
log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s), Vaultwarden $VUSERS user(s) / $VITEMS item(s)"
|
||||
|
||||
TMP="$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom.$$"
|
||||
{
|
||||
|
||||
@@ -25,6 +25,27 @@ import os, subprocess, sys
|
||||
a = sys.argv[1:]
|
||||
pod = os.environ["FAKE_POD_DATA"]
|
||||
cmd = a[a.index("--") + 1:]
|
||||
if "vaultwarden-system" in a:
|
||||
vw = os.environ["FAKE_VW_DATA"]
|
||||
if cmd == ["/vaultwarden", "backup"]:
|
||||
if os.environ.get("FAKE_VW_BACKUP_FAIL"):
|
||||
print("[ERROR] backup failed: database is locked"); sys.exit(1)
|
||||
name = "db_20261009_094401.sqlite3"
|
||||
import shutil as sh; sh.copy(os.path.join(vw, "db.sqlite3"), os.path.join(vw, name))
|
||||
print("[NOTICE] You are using a plain text `ADMIN_TOKEN` which is insecure.")
|
||||
print("Backup to 'data/%s' was successful" % name); sys.exit(0)
|
||||
if cmd[0] == "cat":
|
||||
sys.exit(subprocess.call(["cat", vw + cmd[1][len("/data"):]]))
|
||||
if cmd[:2] == ["rm", "-f"]:
|
||||
p = vw + cmd[2][len("/data"):]
|
||||
open(os.path.join(os.environ["FAKE_STATE"], "vw-removed"), "a").write(cmd[2] + "\n")
|
||||
if os.path.exists(p): os.remove(p)
|
||||
sys.exit(0)
|
||||
if cmd[:2] == ["sh", "-c"]:
|
||||
sys.exit(subprocess.call(["sh", "-c", cmd[2].replace("/data", vw)]))
|
||||
if cmd[0] == "tar":
|
||||
sys.exit(subprocess.call([vw if x == "/data" else x for x in cmd]))
|
||||
sys.exit(96)
|
||||
if cmd[0] == "tar":
|
||||
cnt = os.path.join(os.environ["FAKE_STATE"], "tar-calls")
|
||||
n = int(open(cnt).read()) if os.path.exists(cnt) else 0
|
||||
@@ -73,6 +94,31 @@ head -c 5 "$2" | grep -q '^PGDMP' || { echo "pg_restore: error: input file does
|
||||
'''
|
||||
|
||||
|
||||
# The CI runner has no sqlite3 CLI: a stand-in on Python's own sqlite3 module (the same SQLite library) answers the calls
|
||||
# the scripts make — `sqlite3 -readonly <db> '<sql>'`, one row per line, exit 1 on an error.
|
||||
FAKE_SQLITE3 = r'''#!/usr/bin/env python3
|
||||
import sqlite3, sys
|
||||
a = [x for x in sys.argv[1:] if x != "-readonly"]
|
||||
try:
|
||||
db = sqlite3.connect("file:" + a[0] + "?mode=ro", uri=True)
|
||||
for row in db.execute(a[1]):
|
||||
print("|".join("" if v is None else str(v) for v in row))
|
||||
except Exception as e:
|
||||
print("Error: " + str(e), file=sys.stderr); sys.exit(1)
|
||||
'''
|
||||
REAL_SQLITE3 = None if os.environ.get("FORCE_FAKE_SQLITE3") else shutil.which("sqlite3")
|
||||
|
||||
|
||||
def make_vw_db(path, users=1, items=3):
|
||||
import sqlite3
|
||||
db = sqlite3.connect(path)
|
||||
db.execute("CREATE TABLE users (uuid TEXT PRIMARY KEY, email TEXT)")
|
||||
db.execute("CREATE TABLE ciphers (uuid TEXT PRIMARY KEY, data TEXT)")
|
||||
for i in range(users): db.execute("INSERT INTO users VALUES (?, ?)", ("u%d" % i, "u%d@example.invalid" % i))
|
||||
for i in range(items): db.execute("INSERT INTO ciphers VALUES (?, ?)", ("c%d" % i, "2.encrypted"))
|
||||
db.commit(); db.close()
|
||||
|
||||
|
||||
def git(*a, cwd=None):
|
||||
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t")
|
||||
subprocess.run(["git", *a], cwd=cwd, check=True, capture_output=True, env=env)
|
||||
@@ -109,7 +155,14 @@ class Base(unittest.TestCase):
|
||||
open(os.path.join(self.conf, "enc.key"), "w").write("{}")
|
||||
open(os.path.join(self.conf, "env"), "w").write(
|
||||
"PBS_REPOSITORY_PUSH='u!push@h:1:s'\nPBS_REPOSITORY_RESTORE='u!restore@h:1:s'\nPBS_FINGERPRINT='aa'\n")
|
||||
for name, body in (("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS), ("pg_restore", FAKE_PG_RESTORE)):
|
||||
self.vw = j("vw")
|
||||
os.makedirs(self.vw)
|
||||
make_vw_db(os.path.join(self.vw, "db.sqlite3"))
|
||||
open(os.path.join(self.vw, "rsa_key.pem"), "w").write("-----TEST KEY-----\n")
|
||||
fakes = [("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS), ("pg_restore", FAKE_PG_RESTORE)]
|
||||
if not REAL_SQLITE3:
|
||||
fakes.append(("sqlite3", FAKE_SQLITE3))
|
||||
for name, body in fakes:
|
||||
p = os.path.join(self.bin, name)
|
||||
open(p, "w").write(body); os.chmod(p, 0o755)
|
||||
|
||||
@@ -130,7 +183,7 @@ class Base(unittest.TestCase):
|
||||
|
||||
def env(self, **kw):
|
||||
e = dict(os.environ, PATH=self.bin + ":" + os.environ["PATH"], FAKE_POD_DATA=self.pod, FAKE_PBS_DIR=self.pbs,
|
||||
FAKE_STATE=self.t, FELHOM_DXOFF_CONF=self.conf, FELHOM_DXOFF_TOKENS=self.tokens,
|
||||
FAKE_STATE=self.t, FAKE_VW_DATA=self.vw, FELHOM_DXOFF_CONF=self.conf, FELHOM_DXOFF_TOKENS=self.tokens,
|
||||
FELHOM_DXOFF_STATE=self.state, FELHOM_DXOFF_TEXTFILE_DIR=self.text, FELHOM_DXOFF_DUMPS=self.dumps,
|
||||
FELHOM_DXOFF_SECRETS=self.secrets)
|
||||
e.update({k: str(v) for k, v in kw.items()})
|
||||
@@ -228,6 +281,39 @@ class Push(Base):
|
||||
self.assertIn("link(s)", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_vaultwarden_is_copied_and_its_backup_file_removed(self):
|
||||
r = self.push()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
snap = os.path.join(self.pbs, "snaps", self.pushed()[0])
|
||||
self.assertTrue(os.path.isfile(os.path.join(snap, "vaultwarden/db.sqlite3")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(snap, "vaultwarden/rsa_key.pem")))
|
||||
self.assertEqual(open(os.path.join(snap, "vaultwarden/USERS")).read().strip(), "1")
|
||||
self.assertFalse(os.path.exists(os.path.join(self.vw, "db_20261009_094401.sqlite3")), "the backup file must be removed")
|
||||
self.assertEqual(open(os.path.join(self.t, "vw-removed")).read().strip(), "/data/db_20261009_094401.sqlite3")
|
||||
self.assertIn("vaultwarden_users 1", open(os.path.join(self.text, "felhom_dooplex_offsite.prom")).read())
|
||||
|
||||
def test_vaultwarden_backup_failing_refuses(self):
|
||||
r = self.push(FAKE_VW_BACKUP_FAIL=1)
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("vaultwarden backup", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_vaultwarden_with_no_user_refuses(self):
|
||||
os.remove(os.path.join(self.vw, "db.sqlite3"))
|
||||
make_vw_db(os.path.join(self.vw, "db.sqlite3"), users=0)
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("no user", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
self.assertFalse(os.path.exists(os.path.join(self.vw, "db_20261009_094401.sqlite3")), "removed even on refusal")
|
||||
|
||||
def test_a_corrupt_vaultwarden_copy_refuses(self):
|
||||
open(os.path.join(self.vw, "db.sqlite3"), "wb").write(b"SQLite format 3\x00" + b"\xff" * 4096)
|
||||
r = self.push()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("Vaultwarden integrity_check", r.stderr)
|
||||
self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
|
||||
|
||||
def test_failed_push_writes_no_signal(self):
|
||||
r = self.push(FAKE_PBS_FAIL=1)
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
@@ -285,6 +371,24 @@ class RestoreTest(Base):
|
||||
r = self.restore()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_restore_reports_vaultwarden_rows_never_contents(self):
|
||||
self.pushed_copy()
|
||||
r = self.restore()
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("Vaultwarden 1 user(s) / 3 item(s)", r.stdout)
|
||||
self.assertNotIn("encrypted", r.stdout + r.stderr)
|
||||
|
||||
def test_restore_without_vaultwarden_fails(self):
|
||||
snap = self.pushed_copy()
|
||||
os.remove(os.path.join(snap, "vaultwarden/db.sqlite3"))
|
||||
man = os.path.join(snap, "MANIFEST.sha256")
|
||||
kept = [l for l in open(man).readlines() if "vaultwarden/db.sqlite3" not in l]
|
||||
open(man, "w").writelines(kept)
|
||||
r = self.restore()
|
||||
self.assertNotEqual(r.returncode, 0)
|
||||
self.assertIn("no Vaultwarden database", r.stderr)
|
||||
self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
|
||||
|
||||
def test_an_unreadable_dump_fails(self):
|
||||
shutil.rmtree(self.dumps); os.makedirs(self.dumps)
|
||||
self.add_dump("20261008-220001", self.now - 1200, magic=b"XXXXX")
|
||||
|
||||
Reference in New Issue
Block a user