dooplex-offsite: the operator signing keys ride the nightly encrypted copy; restore test proves each key matches its public half (R-924, operator ruling option A)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

scripts/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 2026-10-09 (late afternoon) — dooplex-offsite carries the operator signing keys (R-924, operator ruling option A)
+
+- `felhom-dooplex-offsite`: `felhom-op-operational`, `felhom-rec-recovery`, `felhom_op_ed25519` and their `.pub` (from
+  `/mnt/5_hdd/felhom.eu/`) ride the nightly encrypted copy under `signing/`; a missing key or a link refuses the push.
+- `felhom-dooplex-offsite-restore-test`: each private key must derive its own public half (`ssh-keygen -y`), never printed.
+- 4 new tests (30 total), green with GNU and BusyBox tools; 3 red-proofs in `audits/dooplex-survival-2026-10-09/signing/`.
+
 ## 2026-10-09 (afternoon) — dooplex-offsite carries the password manager too (R-923)
 
 - `felhom-dooplex-offsite`: a new step runs Vaultwarden's own `vaultwarden backup` (SQLite `VACUUM INTO`, consistent

scripts/dooplex-offsite/felhom-dooplex-offsite

@@ -18,6 +18,8 @@ STATE=${FELHOM_DXOFF_STATE:-/var/lib/felhom-dooplex-offsite}
 TEXTFILE_DIR=${FELHOM_DXOFF_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
 DUMPS=${FELHOM_DXOFF_DUMPS:-/mnt/5_hdd/backup/postgresql/dumps}
 SECRETS=${FELHOM_DXOFF_SECRETS:-/mnt/5_hdd/backup/secrets/exports}
+SIGNING=${FELHOM_DXOFF_SIGNING:-/mnt/5_hdd/felhom.eu}
+SIGNING_KEYS="felhom-op-operational felhom-rec-recovery felhom_op_ed25519"
 DUMP_MAX_AGE_H=${FELHOM_DXOFF_DUMP_MAX_AGE_H:-7}
 SECRETS_MAX_AGE_H=${FELHOM_DXOFF_SECRETS_MAX_AGE_H:-30}
 NOW=${FELHOM_DXOFF_NOW:-$(date +%s)}
@@ -32,10 +34,11 @@ V() { kubectl -n vaultwarden-system exec deploy/vaultwarden -- "$@"; }
 umask 077
 STAGE="$STATE/stage"
 mkdir -p "$STATE"; chmod 700 "$STATE"
-rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets" "$STAGE/root/vaultwarden"
+rm -rf "$STAGE"; mkdir -p "$STAGE/root/gitea" "$STAGE/root/db" "$STAGE/root/secrets" "$STAGE/root/vaultwarden" "$STAGE/root/signing"
 cleanup() {
   for f in "$STAGE/root/gitea/gitea/conf/app.ini" "$STAGE/root/db/gitea.dump" "$STAGE/root/db/globals.sql" \
-           "$STAGE/root/vaultwarden/db.sqlite3" "$STAGE/root/vaultwarden/rsa_key.pem"; do
+           "$STAGE/root/vaultwarden/db.sqlite3" "$STAGE/root/vaultwarden/rsa_key.pem" \
+           "$STAGE/root/signing/felhom-op-operational" "$STAGE/root/signing/felhom-rec-recovery" "$STAGE/root/signing/felhom_op_ed25519"; do
     [ -f "$f" ] && [ ! -L "$f" ] && { shred -u "$f" 2>/dev/null || rm -f "$f"; }
   done
   rm -rf "$STAGE"
@@ -113,6 +116,17 @@ VUSERS=$(sqlite3 -readonly "$STAGE/root/vaultwarden/db.sqlite3" 'SELECT COUNT(*)
 echo "$VUSERS" > "$STAGE/root/vaultwarden/USERS"
 log "vaultwarden: integrity ok, $VUSERS user(s), files: db.sqlite3 $(echo $VLIST)"
 
+# 3c. the operator signing keys (R-924, operator ruling 2026-10-09: option A — on paper AND in this copy). Without them
+# no box accepts a signed agent update, config bundle or OS step after a loss of DooPlex. Each private key and its .pub;
+# refuses when one is missing or is a link (a missing key would otherwise go unnoticed until the day it is needed).
+for k in $SIGNING_KEYS; do
+  for f in "$k" "$k.pub"; do
+    [ -f "$SIGNING/$f" ] && [ ! -L "$SIGNING/$f" ] || die "signing key file $f missing (or a link) in $SIGNING"
+    cp "$SIGNING/$f" "$STAGE/root/signing/$f"
+  done
+done
+log "signing: $(echo $SIGNING_KEYS | wc -w | tr -d ' ') key(s) with their public halves"
+
 # 4. the manifest the restore test checks, then the push
 echo "$GOT" > "$STAGE/root/REPOS"
 (cd "$STAGE/root" && find . -type f ! -name MANIFEST.sha256 -print0 | sort -z | xargs -0 sha256sum > MANIFEST.sha256) \

scripts/dooplex-offsite/felhom-dooplex-offsite-restore-test

@@ -21,7 +21,7 @@ die() { echo "felhom-dooplex-offsite-restore-test: FAILED: $*" >&2; exit 1; }
 umask 077
 mkdir -p "$STATE"; chmod 700 "$STATE"
 T=$(mktemp -d "$STATE/restore.XXXXXX")
-trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql "$T"/out/vaultwarden/db.sqlite3 "$T"/out/vaultwarden/rsa_key.pem; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
+trap 'for f in "$T"/out/gitea/gitea/conf/app.ini "$T"/out/db/gitea.dump "$T"/out/db/globals.sql "$T"/out/vaultwarden/db.sqlite3 "$T"/out/vaultwarden/rsa_key.pem "$T"/out/signing/felhom-op-operational "$T"/out/signing/felhom-rec-recovery "$T"/out/signing/felhom_op_ed25519; do [ -f "$f" ] && shred -u "$f" 2>/dev/null; done; rm -rf "$T"' EXIT
 export PBS_PASSWORD_FILE="$TOKENS/token-restore" PBS_FINGERPRINT
 
 LIST=$(proxmox-backup-client snapshot list host/dooplex-gitea --ns operator --output-format json --repository "$PBS_REPOSITORY_RESTORE") \
@@ -76,7 +76,15 @@ VUSERS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM users;' 2>/dev/null) ||
 VITEMS=$(sqlite3 -readonly "$VDB" 'SELECT COUNT(*) FROM ciphers;' 2>/dev/null) || die "cannot count Vaultwarden items"
 [ "${VUSERS:-0}" -gt 0 ] && [ "$VUSERS" = "$(cat "$O/vaultwarden/USERS" 2>/dev/null)" ] || die "Vaultwarden users: $VUSERS, USERS says $(cat "$O/vaultwarden/USERS" 2>/dev/null)"
 [ "${VITEMS:-0}" -gt 0 ] || die "the Vaultwarden copy holds no item"
-log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s), Vaultwarden $VUSERS user(s) / $VITEMS item(s)"
+# The signing keys (R-924): each private key must still derive its own public half — a usable key, never printed.
+NSIGN=0
+for k in felhom-op-operational felhom-rec-recovery felhom_op_ed25519; do
+  [ -s "$O/signing/$k" ] && [ -s "$O/signing/$k.pub" ] || die "signing key $k missing from the copy"
+  DER=$(ssh-keygen -y -P '' -f "$O/signing/$k" 2>/dev/null | cut -d' ' -f1,2) || die "signing key $k does not open"
+  [ -n "$DER" ] && [ "$DER" = "$(cut -d' ' -f1,2 "$O/signing/$k.pub")" ] || die "signing key $k does not match its public half"
+  NSIGN=$((NSIGN + 1))
+done
+log "checked: $FILES files match the manifest, $GOT repositories pass git fsck, gitea.dump readable, $(ls "$O"/secrets | wc -l | tr -d ' ') secrets file(s), Vaultwarden $VUSERS user(s) / $VITEMS item(s), $NSIGN signing key(s) match"
 
 TMP="$TEXTFILE_DIR/felhom_dooplex_offsite_restore.prom.$$"
 {

scripts/dooplex-offsite/test_dooplex_offsite.py

@@ -106,6 +106,24 @@ try:
 except Exception as e:
     print("Error: " + str(e), file=sys.stderr); sys.exit(1)
 '''
+# ssh-keygen stand-in (the CI runner may lack OpenSSH): `ssh-keygen -y -P '' -f <key>` prints the public line a
+# test key file carries after "PUB:" — the same contract as the real tool (derive the public half from the private).
+FAKE_SSH_KEYGEN = r'''#!/bin/sh
+[ "$1" = "-y" ] || exit 97
+f=""; while [ $# -gt 0 ]; do [ "$1" = "-f" ] && f=$2; shift; done
+sed -n 's/^PUB://p' "$f" | grep . || { echo "Load key: invalid format" >&2; exit 255; }
+'''
+SIGNING_KEYS = ("felhom-op-operational", "felhom-rec-recovery", "felhom_op_ed25519")
+
+
+def make_signing(d):
+    os.makedirs(d, exist_ok=True)
+    for i, k in enumerate(SIGNING_KEYS):
+        pub = "ssh-ed25519 AAAATEST%d %s" % (i, k)
+        open(os.path.join(d, k), "w").write("-----BEGIN TEST KEY-----\nPUB:%s\n" % pub)
+        open(os.path.join(d, k + ".pub"), "w").write(pub + "\n")
+
+
 REAL_SQLITE3 = None if os.environ.get("FORCE_FAKE_SQLITE3") else shutil.which("sqlite3")
 
 
@@ -159,7 +177,10 @@ class Base(unittest.TestCase):
         os.makedirs(self.vw)
         make_vw_db(os.path.join(self.vw, "db.sqlite3"))
         open(os.path.join(self.vw, "rsa_key.pem"), "w").write("-----TEST KEY-----\n")
-        fakes = [("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS), ("pg_restore", FAKE_PG_RESTORE)]
+        self.signing = j("signing")
+        make_signing(self.signing)
+        fakes = [("kubectl", FAKE_KUBECTL), ("proxmox-backup-client", FAKE_PBS), ("pg_restore", FAKE_PG_RESTORE),
+                 ("ssh-keygen", FAKE_SSH_KEYGEN)]
         if not REAL_SQLITE3:
             fakes.append(("sqlite3", FAKE_SQLITE3))
         for name, body in fakes:
@@ -183,7 +204,7 @@ class Base(unittest.TestCase):
 
     def env(self, **kw):
         e = dict(os.environ, PATH=self.bin + ":" + os.environ["PATH"], FAKE_POD_DATA=self.pod, FAKE_PBS_DIR=self.pbs,
-                 FAKE_STATE=self.t, FAKE_VW_DATA=self.vw, FELHOM_DXOFF_CONF=self.conf, FELHOM_DXOFF_TOKENS=self.tokens,
+                 FAKE_STATE=self.t, FAKE_VW_DATA=self.vw, FELHOM_DXOFF_SIGNING=self.signing, FELHOM_DXOFF_CONF=self.conf, FELHOM_DXOFF_TOKENS=self.tokens,
                  FELHOM_DXOFF_STATE=self.state, FELHOM_DXOFF_TEXTFILE_DIR=self.text, FELHOM_DXOFF_DUMPS=self.dumps,
                  FELHOM_DXOFF_SECRETS=self.secrets)
         e.update({k: str(v) for k, v in kw.items()})
@@ -314,6 +335,21 @@ class Push(Base):
         self.assertIn("Vaultwarden integrity_check", r.stderr)
         self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
 
+    def test_signing_keys_ride_the_copy(self):
+        r = self.push()
+        self.assertEqual(r.returncode, 0, r.stderr)
+        snap = os.path.join(self.pbs, "snaps", self.pushed()[0])
+        for k in SIGNING_KEYS:
+            self.assertTrue(os.path.isfile(os.path.join(snap, "signing", k)), k)
+            self.assertTrue(os.path.isfile(os.path.join(snap, "signing", k + ".pub")), k)
+
+    def test_a_missing_signing_key_refuses(self):
+        os.remove(os.path.join(self.signing, "felhom-rec-recovery"))
+        r = self.push()
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("signing key file felhom-rec-recovery missing", r.stderr)
+        self.assertEqual(self.pushed(), []); self.assertFalse(self.signal())
+
     def test_failed_push_writes_no_signal(self):
         r = self.push(FAKE_PBS_FAIL=1)
         self.assertNotEqual(r.returncode, 0)
@@ -378,6 +414,25 @@ class RestoreTest(Base):
         self.assertIn("Vaultwarden 1 user(s) / 3 item(s)", r.stdout)
         self.assertNotIn("encrypted", r.stdout + r.stderr)
 
+    def test_restore_checks_every_signing_key_opens_and_matches(self):
+        self.pushed_copy()
+        r = self.restore()
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("3 signing key(s) match", r.stdout)
+        self.assertNotIn("BEGIN TEST KEY", r.stdout + r.stderr)
+
+    def test_restore_refuses_a_signing_key_that_does_not_match(self):
+        snap = self.pushed_copy()
+        p = os.path.join(snap, "signing", "felhom-op-operational.pub")
+        open(p, "w").write("ssh-ed25519 AAAAOTHER someone-else\n")
+        man = os.path.join(snap, "MANIFEST.sha256")
+        kept = [l for l in open(man).readlines() if not l.rstrip().endswith("felhom-op-operational.pub")]
+        open(man, "w").writelines(kept)
+        r = self.restore()
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("does not match its public half", r.stderr)
+        self.assertFalse(self.signal("felhom_dooplex_offsite_restore.prom"))
+
     def test_restore_without_vaultwarden_fails(self):
         snap = self.pushed_copy()
         os.remove(os.path.join(snap, "vaultwarden/db.sqlite3"))