diff --git a/documentation/audits/dooplex-survival-2026-10-09/partB/failmail-dry.txt b/documentation/audits/dooplex-survival-2026-10-09/partB/failmail-dry.txt new file mode 100644 index 00000000..9421bda0 --- /dev/null +++ b/documentation/audits/dooplex-survival-2026-10-09/partB/failmail-dry.txt @@ -0,0 +1,12 @@ +## 2026-10-09T08:14:02Z dry failure: transient unit /bin/false with the same OnFailure= line +-- No entries -- +## (transient units do not expand %n, so the instance is spelled out; the real units use %n — systemctl show above) +2026-10-09T10:14:08+02:00 dooplex systemd[1]: Started felhom-failmail-dryrun-r232.service - [systemd-run] /bin/false. +2026-10-09T10:14:08+02:00 dooplex systemd[1]: felhom-failmail-dryrun-r232.service: Main process exited, code=exited, status=1/FAILURE +2026-10-09T10:14:08+02:00 dooplex systemd[1]: felhom-failmail-dryrun-r232.service: Failed with result 'exit-code'. +2026-10-09T10:14:08+02:00 dooplex systemd[1]: felhom-failmail-dryrun-r232.service: Triggering OnFailure= dependencies. +2026-10-09T10:14:08+02:00 dooplex systemd[1]: Starting felhom-backup-failmail@felhom-failmail-dryrun-r232.service.service - Felhom: mail admin@ that felhom-failmail-dryrun-r232.service failed (R-232)... +2026-10-09T10:14:08+02:00 dooplex felhom-backup-failmail[3030045]: /opt/backup/scripts/backup-config.sh: line 143: NOTIFY_WEBHOOK_URL: unbound variable +2026-10-09T10:14:08+02:00 dooplex systemd[1]: felhom-backup-failmail@felhom-failmail-dryrun-r232.service.service: Main process exited, code=exited, status=1/FAILURE +2026-10-09T10:14:08+02:00 dooplex systemd[1]: felhom-backup-failmail@felhom-failmail-dryrun-r232.service.service: Failed with result 'exit-code'. +2026-10-09T10:14:08+02:00 dooplex systemd[1]: Failed to start felhom-backup-failmail@felhom-failmail-dryrun-r232.service.service - Felhom: mail admin@ that felhom-failmail-dryrun-r232.service failed (R-232). diff --git a/documentation/audits/dooplex-survival-2026-10-09/partB/install.txt b/documentation/audits/dooplex-survival-2026-10-09/partB/install.txt new file mode 100644 index 00000000..476edab9 --- /dev/null +++ b/documentation/audits/dooplex-survival-2026-10-09/partB/install.txt @@ -0,0 +1,16 @@ +## 2026-10-09T08:10:21Z install (felhom.eu 1707c928) +install.sh: installed; timers NOT enabled (see the header) +## key +key_rc=0 +total 24 +drwx------ 2 root root 4096 Oct 9 10:10 . +drwxr-xr-x 133 root root 12288 Oct 9 10:10 .. +-rw------- 1 root root 255 Oct 9 10:10 enc.key +-rw------- 1 root root 268 Oct 9 10:10 env +600 root +PBS_REPOSITORY_PUSH='dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite' +PBS_REPOSITORY_RESTORE='dooplex-hub@pbs!restore@127.0.0.1:18007:felhom-offsite' +PBS_FINGERPRINT= +OnFailure=felhom-backup-failmail@felhom-hub-db-backup.service.service + +OnFailure=felhom-backup-failmail@felhom-dooplex-offsite.service.service diff --git a/documentation/audits/dooplex-survival-2026-10-09/partB/push-1.txt b/documentation/audits/dooplex-survival-2026-10-09/partB/push-1.txt new file mode 100644 index 00000000..33d83f5d --- /dev/null +++ b/documentation/audits/dooplex-survival-2026-10-09/partB/push-1.txt @@ -0,0 +1,26 @@ +2026-10-09T10:10:34+02:00 dooplex systemd[1]: Starting felhom-dooplex-offsite.service - Felhom: push Gitea + DooPlex secrets to ep0, encrypted (R-232)... +2026-10-09T10:10:35+02:00 dooplex felhom-dooplex-offsite[2995184]: felhom-dooplex-offsite: database: 20261009-040001, 6218173 bytes, 250 min old +2026-10-09T10:11:53+02:00 dooplex felhom-dooplex-offsite[2995184]: felhom-dooplex-offsite: files: 10 repositories, 614 MB +2026-10-09T10:11:53+02:00 dooplex felhom-dooplex-offsite[2995184]: felhom-dooplex-offsite: secrets: 3 file(s) of 20261009_031011 +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Starting backup: [operator]:host/dooplex-gitea/2026-10-09T08:11:56Z +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Client name: dooplex +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Starting backup protocol: Fri Oct 9 10:11:56 2026 +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Using encryption key from '/etc/felhom-dooplex-offsite/enc.key'.. +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Encryption key fingerprint: 93:03:bf:d7:1f:4c:9e:fe +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: No previous manifest available. +2026-10-09T10:11:56+02:00 dooplex felhom-dooplex-offsite[3006280]: Upload directory '/var/lib/felhom-dooplex-offsite/stage/root' to 'dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite' as dooplex.pxar.didx +2026-10-09T10:12:51+02:00 dooplex felhom-dooplex-offsite[3006280]: dooplex.pxar: had to backup 540.664 MiB of 544.173 MiB (compressed 531.953 MiB) in 54.26 s (average 9.964 MiB/s) +2026-10-09T10:12:51+02:00 dooplex felhom-dooplex-offsite[3006280]: dooplex.pxar: backup was done incrementally, reused 3.508 MiB (0.6%) +2026-10-09T10:12:51+02:00 dooplex felhom-dooplex-offsite[3006280]: Uploaded backup catalog (1.25 MiB) +2026-10-09T10:12:53+02:00 dooplex felhom-dooplex-offsite[3006280]: Duration: 56.87s +2026-10-09T10:12:53+02:00 dooplex felhom-dooplex-offsite[3006280]: End Time: Fri Oct 9 10:12:53 2026 +2026-10-09T10:12:53+02:00 dooplex felhom-dooplex-offsite[2995184]: felhom-dooplex-offsite: pushed to ep0 (ns operator, host/dooplex-gitea) in 57 s +2026-10-09T10:12:53+02:00 dooplex felhom-dooplex-offsite[2995184]: felhom-dooplex-offsite: success signal written +2026-10-09T10:12:54+02:00 dooplex systemd[1]: felhom-dooplex-offsite.service: Deactivated successfully. +2026-10-09T10:12:54+02:00 dooplex systemd[1]: Finished felhom-dooplex-offsite.service - Felhom: push Gitea + DooPlex secrets to ep0, encrypted (R-232). +2026-10-09T10:12:54+02:00 dooplex systemd[1]: felhom-dooplex-offsite.service: Consumed 24.466s CPU time, 1.2G memory peak. +# HELP felhom_dooplex_offsite_last_success_timestamp_seconds Last successful push of Gitea + DooPlex secrets to ep0 (R-232). +# TYPE felhom_dooplex_offsite_last_success_timestamp_seconds gauge +felhom_dooplex_offsite_last_success_timestamp_seconds 1791533573 +felhom_dooplex_offsite_last_success_bytes 566109337 +felhom_dooplex_offsite_last_success_repositories 10 diff --git a/documentation/audits/dooplex-survival-2026-10-09/partB/red-proof.txt b/documentation/audits/dooplex-survival-2026-10-09/partB/red-proof.txt index 4f1eddb9..c6beb83e 100644 --- a/documentation/audits/dooplex-survival-2026-10-09/partB/red-proof.txt +++ b/documentation/audits/dooplex-survival-2026-10-09/partB/red-proof.txt @@ -29,3 +29,11 @@ FAILED (failures=1) ### GREEN, full suite Ran 20 tests in 35.065s OK +### RED: failmail under set -u, config reads an unset variable +FAIL: test_failmail_calls_notify_failure_with_the_unit_name (__main__.FailMail.test_failmail_calls_notify_failure_with_the_unit_name) +AssertionError: 1 != 0 : /tmp/tmp48157fhy/cfg.sh: line 1: NOTIFY_WEBHOOK_URL_UNSET_IN_TEST: unbound variable +Ran 2 tests in 0.007s +FAILED (failures=1) +### GREEN +Ran 20 tests in 35.056s +OK diff --git a/documentation/audits/dooplex-survival-2026-10-09/partB/restore-test-1.txt b/documentation/audits/dooplex-survival-2026-10-09/partB/restore-test-1.txt new file mode 100644 index 00000000..22268bcf --- /dev/null +++ b/documentation/audits/dooplex-survival-2026-10-09/partB/restore-test-1.txt @@ -0,0 +1,34 @@ +2026-10-09T10:13:03+02:00 dooplex systemd[1]: Starting felhom-dooplex-offsite-restore-test.service - Felhom: restore-test the Gitea + secrets copy on ep0 (R-232)... +2026-10-09T10:13:03+02:00 dooplex felhom-dooplex-offsite-restore-test[3016447]: felhom-dooplex-offsite-restore-test: restoring host/dooplex-gitea/2026-10-09T08:11:56Z +2026-10-09T10:13:04+02:00 dooplex felhom-dooplex-offsite-restore-test[3016610]: Using encryption key from '/etc/felhom-dooplex-offsite/enc.key'.. +2026-10-09T10:13:04+02:00 dooplex felhom-dooplex-offsite-restore-test[3016610]: Fingerprint: 93:03:bf:d7:1f:4c:9e:fe +2026-10-09T10:13:09+02:00 dooplex felhom-dooplex-offsite-restore-test[3016610]: progress 23% (126.51 MiB of 544.173 MiB in 5.3s, 24.049 MiB/s) +2026-10-09T10:13:19+02:00 dooplex felhom-dooplex-offsite-restore-test[3016610]: progress 74% (404.7 MiB of 544.173 MiB in 15.3s, 27.637 MiB/s) +2026-10-09T10:13:24+02:00 dooplex felhom-dooplex-offsite-restore-test[3016610]: restore complete (544.173 MiB processed in 20s, average 27.217 MiB/s) +2026-10-09T10:13:43+02:00 dooplex felhom-dooplex-offsite-restore-test[3016447]: felhom-dooplex-offsite-restore-test: checked: 27805 files match the manifest, 10 repositories pass git fsck, gitea.dump readable, 3 secrets file(s) +2026-10-09T10:13:43+02:00 dooplex felhom-dooplex-offsite-restore-test[3016447]: felhom-dooplex-offsite-restore-test: success signal written +2026-10-09T10:13:44+02:00 dooplex systemd[1]: felhom-dooplex-offsite-restore-test.service: Deactivated successfully. +2026-10-09T10:13:44+02:00 dooplex systemd[1]: Finished felhom-dooplex-offsite-restore-test.service - Felhom: restore-test the Gitea + secrets copy on ep0 (R-232). +2026-10-09T10:13:44+02:00 dooplex systemd[1]: felhom-dooplex-offsite-restore-test.service: Consumed 29.822s CPU time, 834.6M memory peak. +# HELP felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds Last successful restore test of the Gitea + secrets copy on ep0 (R-232). +# TYPE felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds gauge +felhom_dooplex_offsite_restore_test_last_success_timestamp_seconds 1791533623 +## ep0 filesystem (second channel, read-only ssh) +/mnt/pbs-datastore/ns/operator/host/: +total 16 +drwxr-xr-x 4 backup backup 4096 Oct 9 08:11 . +drwxr-xr-x 3 backup backup 4096 Oct 5 13:50 .. +drwxr-xr-x 3 backup backup 4096 Oct 9 08:11 dooplex-gitea +drwxr-xr-x 7 backup backup 4096 Oct 9 00:31 dooplex-hub + +/mnt/pbs-datastore/ns/operator/host/dooplex-gitea/: +total 16 +drwxr-xr-x 3 backup backup 4096 Oct 9 08:11 . +drwxr-xr-x 4 backup backup 4096 Oct 9 08:11 .. +drwxr-xr-x 2 backup backup 4096 Oct 9 08:12 2026-10-09T08:11:56Z +-rw-r--r-- 1 backup backup 21 Oct 9 08:11 owner +Filesystem Size Used Avail Use% Mounted on +/dev/sdb 98G 16G 83G 16% /mnt/pbs-datastore +## restore token cannot write / push token cannot forget +Error: permission check failed - missing Datastore.Modify|Datastore.Prune on /datastore/felhom-offsite/operator +forget_rc=0 diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index 9a0a8f1a..1012f853 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -9,7 +9,8 @@ - `felhom-dooplex-offsite-restore-test` (Sun 05:30, read-only token): manifest check, repo count, `git fsck` on every repository through a scratch repo with a known config (the pod's `config` files are never read by root git), `pg_restore --list`, secrets present. -- `felhom-backup-failmail@.service`: `OnFailure=` mail through R-232 (a)'s `notify_failure`; also added to both +- `felhom-backup-failmail@.service`: `OnFailure=` mail through R-232 (a)'s `notify_failure` (no `set -u`: the live dry run + found it dying on the shared config's unset `NOTIFY_WEBHOOK_URL`; red-proved); also added to both hub-DB units (`scripts/hub-db-backup/*.service`). - 20 tests (`test_dooplex_offsite.py`; fakes for kubectl/PBS/pg_restore, real git/tar), green with GNU and BusyBox tools; red-proofs for 6 checks in `audits/dooplex-survival-2026-10-09/partB/red-proof.txt`. diff --git a/scripts/dooplex-offsite/felhom-backup-failmail b/scripts/dooplex-offsite/felhom-backup-failmail index f41753f5..8377eded 100755 --- a/scripts/dooplex-offsite/felhom-backup-failmail +++ b/scripts/dooplex-offsite/felhom-backup-failmail @@ -2,7 +2,7 @@ # felhom-backup-failmail — mail admin@ that a Felhom backup unit on DooPlex failed (R-232 (a) route, reused). # Called by felhom-backup-failmail@.service, which the backup units name in OnFailure=. # Reuses notify_failure from /opt/backup/scripts/backup-config.sh (Resend; never prints the key; never fails). -set -u +# No `set -u`: backup-config.sh reads unset variables (NOTIFY_WEBHOOK_URL), which killed the first dry run. UNIT=${1:?unit name} CONFIG=${FELHOM_FAILMAIL_CONFIG:-/opt/backup/scripts/backup-config.sh} # shellcheck source=/dev/null diff --git a/scripts/dooplex-offsite/test_dooplex_offsite.py b/scripts/dooplex-offsite/test_dooplex_offsite.py index 1ef828d6..76ddcb91 100644 --- a/scripts/dooplex-offsite/test_dooplex_offsite.py +++ b/scripts/dooplex-offsite/test_dooplex_offsite.py @@ -313,7 +313,10 @@ class FailMail(unittest.TestCase): t = tempfile.mkdtemp() try: cfg = os.path.join(t, "cfg.sh"); out = os.path.join(t, "out") - open(cfg, "w").write('notify_failure() { echo "$1" > %s; return 0; }\n' % out) + # Like the real backup-config.sh, it reads a variable that is unset (NOTIFY_WEBHOOK_URL is commented out + # there): the 2026-10-09 dry run found the mail script under `set -u` dying on exactly that line. + open(cfg, "w").write('notify_failure() { if [ -n "${NOTIFY_WEBHOOK_URL_UNSET_IN_TEST}" ]; then :; fi; ' + 'echo "$1" > %s; return 0; }\n' % out) r = subprocess.run([FAILMAIL, "felhom-dooplex-offsite.service"], env=dict(os.environ, FELHOM_FAILMAIL_CONFIG=cfg), capture_output=True, text=True) self.assertEqual(r.returncode, 0, r.stderr)