R-173 option A in force (hub DB nightly to ep0, restore-tested, alarmed); R-519 proven live on 9202 and closed; R-173/R-232 narrowed; R-882..R-885 opened (332 -> 335); runbook §3 tested; hubdb-check
gates / gates (push) Successful in 59s
gates / gates (push) Successful in 59s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
## promtool, in pod/prometheus-55b675779d-t8c74, 2026-10-05T13:42:17Z
|
||||
### green
|
||||
SUCCESS
|
||||
|
||||
rc=0
|
||||
### red: threshold 26h -> 260h
|
||||
FAILED:
|
||||
alertname: HubDBBackupStale, time: 1d2h40m,
|
||||
exp:[
|
||||
0:
|
||||
Labels:{alertname="HubDBBackupStale", component="backup", instance="dooplex", severity="critical"}
|
||||
Annotations:{description="No successful push of the hub DB snapshot to ep0 for >26h (daily at 02:30). Check `journalctl -u felhom-hub-db-backup.service`, the tunnel `systemctl status felhom-ep0-pbs-tunnel`, and that the hub logs `db snapshot written` at 02:00.", summary="The Felhom hub database has not reached ep0 for 26 h (R-173)"}
|
||||
],
|
||||
got:[]
|
||||
|
||||
|
||||
command terminated with exit code 1
|
||||
rc=0
|
||||
### red: absent() removed
|
||||
SUCCESS
|
||||
|
||||
### red (re-run): absent() removed from HubDBBackupStale — first attempt above did NOT apply (indent mismatch)
|
||||
FAILED:
|
||||
alertname: HubDBBackupStale, time: 40m,
|
||||
Labels:{alertname="HubDBBackupStale", component="backup", severity="critical"}
|
||||
got:[]
|
||||
@@ -0,0 +1,52 @@
|
||||
rule_files: [bf.yml]
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
# 1. the push stops: last success at t=0, never again → fires once 26 h + 30 m have passed
|
||||
- interval: 5m
|
||||
input_series:
|
||||
- series: 'felhom_hub_db_backup_last_success_timestamp_seconds{instance="dooplex"}'
|
||||
values: '0+0x400'
|
||||
- series: 'felhom_hub_db_restore_test_last_success_timestamp_seconds{instance="dooplex"}'
|
||||
values: '0+0x400'
|
||||
alert_rule_test:
|
||||
- eval_time: 26h
|
||||
alertname: HubDBBackupStale
|
||||
exp_alerts: []
|
||||
- eval_time: 26h40m
|
||||
alertname: HubDBBackupStale
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, component: backup, instance: dooplex}
|
||||
exp_annotations:
|
||||
summary: "The Felhom hub database has not reached ep0 for 26 h (R-173)"
|
||||
description: "No successful push of the hub DB snapshot to ep0 for >26h (daily at 02:30). Check `journalctl -u felhom-hub-db-backup.service`, the tunnel `systemctl status felhom-ep0-pbs-tunnel`, and that the hub logs `db snapshot written` at 02:00."
|
||||
# 2. healthy: the push succeeds every 24 h → never fires
|
||||
- interval: 1h
|
||||
input_series:
|
||||
- series: 'felhom_hub_db_backup_last_success_timestamp_seconds{instance="dooplex"}'
|
||||
values: '0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 172800 172800 172800'
|
||||
- series: 'felhom_hub_db_restore_test_last_success_timestamp_seconds{instance="dooplex"}'
|
||||
values: '0+0x50'
|
||||
alert_rule_test:
|
||||
- eval_time: 50h
|
||||
alertname: HubDBBackupStale
|
||||
exp_alerts: []
|
||||
# 3. the metric never existed (script never ran) → fires on absent()
|
||||
- interval: 5m
|
||||
input_series:
|
||||
- series: 'up{job="node"}'
|
||||
values: '1+0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 40m
|
||||
alertname: HubDBBackupStale
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, component: backup}
|
||||
exp_annotations:
|
||||
summary: "The Felhom hub database has not reached ep0 for 26 h (R-173)"
|
||||
description: "No successful push of the hub DB snapshot to ep0 for >26h (daily at 02:30). Check `journalctl -u felhom-hub-db-backup.service`, the tunnel `systemctl status felhom-ep0-pbs-tunnel`, and that the hub logs `db snapshot written` at 02:00."
|
||||
- eval_time: 2h
|
||||
alertname: HubDBRestoreTestStale
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, component: backup}
|
||||
exp_annotations:
|
||||
summary: "The hub database copy on ep0 has not passed a restore test for 8 days (R-173)"
|
||||
description: "The weekly restore test (Sun 04:30) has not succeeded for >8 days. Check `journalctl -u felhom-hub-db-restore-test.service`."
|
||||
@@ -0,0 +1,5 @@
|
||||
## alarm drill start 2026-10-05T13:51:46Z: success file moved aside (absent case)
|
||||
backup_freshness.prom
|
||||
fan_metrics.prom
|
||||
felhom_hub_db_restore.prom
|
||||
node_housekeeping.prom
|
||||
@@ -0,0 +1,27 @@
|
||||
## manual push via unit, 2026-10-05T13:50:30Z
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
2026-10-05T15:50:08+02:00 dooplex systemd[1]: Starting felhom-hub-db-backup.service - Felhom: push the hub DB snapshot to ep0 (R-173)...
|
||||
2026-10-05T15:50:09+02:00 dooplex felhom-hub-db-backup[3036511]: felhom-hub-db-backup: snapshot hub-20261005T123037Z.db, 79 min old
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3036511]: felhom-hub-db-backup: checked: 369807360 bytes, integrity ok, 4 host(s)
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Starting backup: [operator]:host/dooplex-hub/2026-10-05T13:50:18Z
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Client name: dooplex
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Starting backup protocol: Mon Oct 5 15:50:18 2026
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Using encryption key from '/etc/felhom-hub-backup/enc.key'..
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Encryption key fingerprint: b2:19:bf:36:3b:97:3d:6c
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: No previous manifest available.
|
||||
2026-10-05T15:50:18+02:00 dooplex felhom-hub-db-backup[3037790]: Upload directory '/var/lib/felhom-hub-backup/stage' to 'dooplex-hub@pbs!push@127.0.0.1:18007:felhom-offsite' as hubdb.pxar.didx
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3037790]: hubdb.pxar: had to backup 352.676 MiB of 352.676 MiB (compressed 17.242 MiB) in 6.58 s (average 53.604 MiB/s)
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3037790]: Uploaded backup catalog (56 B)
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3037790]: Duration: 7.19s
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3037790]: End Time: Mon Oct 5 15:50:25 2026
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3036511]: felhom-hub-db-backup: pushed hub-20261005T123037Z.db to ep0 (ns operator) in 7 s
|
||||
2026-10-05T15:50:25+02:00 dooplex felhom-hub-db-backup[3036511]: felhom-hub-db-backup: success signal written
|
||||
2026-10-05T15:50:30+02:00 dooplex systemd[1]: felhom-hub-db-backup.service: Deactivated successfully.
|
||||
2026-10-05T15:50:30+02:00 dooplex systemd[1]: Finished felhom-hub-db-backup.service - Felhom: push the hub DB snapshot to ep0 (R-173).
|
||||
2026-10-05T15:50:30+02:00 dooplex systemd[1]: felhom-hub-db-backup.service: Consumed 9.566s CPU time, 584.8M memory peak.
|
||||
## textfile
|
||||
# HELP felhom_hub_db_backup_last_success_timestamp_seconds Last successful push of the hub DB snapshot to ep0 (R-173).
|
||||
# TYPE felhom_hub_db_backup_last_success_timestamp_seconds gauge
|
||||
felhom_hub_db_backup_last_success_timestamp_seconds 1791208225
|
||||
felhom_hub_db_backup_last_success_bytes 369807360
|
||||
@@ -0,0 +1 @@
|
||||
gitea.dooplex.hu/admin/felhom-controller:0.296.0 Up 6 seconds (healthy)
|
||||
@@ -0,0 +1,3 @@
|
||||
## deploy bookstack 2026-10-05T13:53:26Z (fields: DOMAIN APP_KEY DB_PASSWORD ADMIN_PASSWORD; SUBDOMAIN = catalog default)
|
||||
|
||||
HTTP 202
|
||||
@@ -0,0 +1,36 @@
|
||||
## run 1 (baseline) start 2026-10-05T14:04:36Z
|
||||
|
||||
HTTP 200
|
||||
{"ok":true,"data":{"enabled":true,"running":true}}
|
||||
|
||||
HTTP 200
|
||||
|
||||
## run 1 end 2026-10-05T14:05:17Z: {"ok":true,"data":{"db_dump":{"count":2,"duration":"32.784678377s","last_run":"2026-10-05T14:05:12.109790785Z","success":true},"enabled":true,"running":false}}
|
||||
2026/10/05 14:04:44 data_versions.go:131: [DEBUG] [backup] bookstack: stamped volume-dumps/bookstack_bookstack_config.tar (70144 B) with pins [lscr.io/linuxserver/bookstack:26.09.1@sha256:99cd1f5707c1911afad213adec5c9739763b76f843d1477142231834ecdcb6f7 mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1]
|
||||
2026/10/05 14:04:44 backup.go:848: [DEBUG] [backup] Dumping volume bookstack_bookstack_db_data for bookstack
|
||||
2026/10/05 14:04:45 backup.go:873: [INFO] [backup] Volume dump: bookstack/bookstack_bookstack_db_data → 153.4 MB
|
||||
2026/10/05 14:04:45 data_versions.go:131: [DEBUG] [backup] bookstack: stamped volume-dumps/bookstack_bookstack_db_data.tar (160868864 B) with pins [lscr.io/linuxserver/bookstack:26.09.1@sha256:99cd1f5707c1911afad213adec5c9739763b76f843d1477142231834ecdcb6f7 mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1]
|
||||
2026/10/05 14:04:45 backup.go:990: [INFO] [backup] Restarting bookstack after volume dump
|
||||
2026/10/05 14:04:51 backup.go:974: [INFO] [backup] Stopping paperless-ngx for safe volume dump
|
||||
2026/10/05 14:04:58 backup.go:848: [DEBUG] [backup] Dumping volume paperless-ngx_paperless_redis_data for paperless-ngx
|
||||
2026/10/05 14:04:58 backup.go:873: [INFO] [backup] Volume dump: paperless-ngx/paperless-ngx_paperless_redis_data → 7.4 MB
|
||||
2026/10/05 14:04:58 data_versions.go:131: [DEBUG] [backup] paperless-ngx: stamped volume-dumps/paperless-ngx_paperless_redis_data.tar (7792128 B) with pins [ghcr.io/paperless-ngx/paperless-ngx:2.20.15 postgres:18-alpine redis:7-alpine]
|
||||
2026/10/05 14:04:58 backup.go:848: [DEBUG] [backup] Dumping volume paperless-ngx_paperless_data for paperless-ngx
|
||||
2026/10/05 14:04:59 backup.go:873: [INFO] [backup] Volume dump: paperless-ngx/paperless-ngx_paperless_data → 5.8 MB
|
||||
2026/10/05 14:04:59 data_versions.go:131: [DEBUG] [backup] paperless-ngx: stamped volume-dumps/paperless-ngx_paperless_data.tar (6052352 B) with pins [ghcr.io/paperless-ngx/paperless-ngx:2.20.15 postgres:18-alpine redis:7-alpine]
|
||||
2026/10/05 14:04:59 backup.go:848: [DEBUG] [backup] Dumping volume paperless-ngx_paperless_postgres_data for paperless-ngx
|
||||
2026/10/05 14:04:59 backup.go:873: [INFO] [backup] Volume dump: paperless-ngx/paperless-ngx_paperless_postgres_data → 68.8 MB
|
||||
2026/10/05 14:04:59 data_versions.go:131: [DEBUG] [backup] paperless-ngx: stamped volume-dumps/paperless-ngx_paperless_postgres_data.tar (72107008 B) with pins [ghcr.io/paperless-ngx/paperless-ngx:2.20.15 postgres:18-alpine redis:7-alpine]
|
||||
2026/10/05 14:04:59 backup.go:990: [INFO] [backup] Restarting paperless-ngx after volume dump
|
||||
2026/10/05 14:05:11 backup.go:974: [INFO] [backup] Stopping privatebin for safe volume dump
|
||||
2026/10/05 14:05:11 backup.go:848: [DEBUG] [backup] Dumping volume privatebin_privatebin_data for privatebin
|
||||
2026/10/05 14:05:12 backup.go:873: [INFO] [backup] Volume dump: privatebin/privatebin_privatebin_data → 1.5 KB
|
||||
2026/10/05 14:05:12 data_versions.go:131: [DEBUG] [backup] privatebin: stamped volume-dumps/privatebin_privatebin_data.tar (1536 B) with pins [privatebin/pdo:2.0.6@sha256:4c141b2326f8b353598ce9ce7507a9cfecf2dad5c60a39fea903d430e296d8f5]
|
||||
2026/10/05 14:05:12 backup.go:986: [INFO] [backup] privatebin NOT restarted after the volume dump: the household stopped it meanwhile
|
||||
2026/10/05 14:05:12 backup.go:684: [INFO] [backup] App-data backup completed: 2 databases (483.2 KB total), 3 volume dump(s) (32.785s)
|
||||
2026/10/05 14:05:12 recovery_unit.go:289: [INFO] [backup] Recovery unit captured for bookstack → /mnt/sys_drive/felhom-data/backups/primary/bookstack (images=2, secrets-referenced=3, data_keys=0, portable-carried=2/2, withheld=1)
|
||||
2026/10/05 14:05:12 recovery_unit.go:289: [INFO] [backup] Recovery unit captured for paperless-ngx → /mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx/backups/primary/paperless-ngx (images=3, secrets-referenced=3, data_keys=0, portable-carried=2/2, withheld=1)
|
||||
2026/10/05 14:05:12 recovery_unit.go:289: [INFO] [backup] Recovery unit captured for privatebin → /mnt/sys_drive/felhom-data/backups/primary/privatebin (images=1, secrets-referenced=0, data_keys=0, portable-carried=0/0, withheld=0)
|
||||
|
||||
/var/lib/felhom/docker/volumes/felhom-controller-data/_data/data/appdata-run.json
|
||||
/var/lib/docker/volumes/felhom-controller-data/_data/data/appdata-run.json
|
||||
@@ -0,0 +1,12 @@
|
||||
## after run 1, 2026-10-05T14:05:34Z
|
||||
### run record
|
||||
{"running":false,"started_at":"0001-01-01T00:00:00Z","interrupted":"0001-01-01T00:00:00Z"}
|
||||
### restore points (bookstack)
|
||||
{"ok":true,"data":[{"time":"2026-10-05T14:04:39Z","short_id":"helyi","tier":1,"drive_label":"Belső SSD (rendszer)"}]}
|
||||
|
||||
HTTP 200
|
||||
|
||||
### volume dump file times
|
||||
total 157172
|
||||
-rw-r--r-- 1 root root 70144 2026-10-05T14:04:44 bookstack_bookstack_config.tar
|
||||
-rw-r--r-- 1 root root 160868864 2026-10-05T14:04:44 bookstack_bookstack_db_data.tar
|
||||
@@ -0,0 +1,34 @@
|
||||
## run 2 start 2026-10-05T14:05:50Z
|
||||
HTTP 200
|
||||
### watcher
|
||||
match 2026-10-05T14:05:58.856456447Z
|
||||
restarted 2026-10-05T14:05:59.669364637Z rc=0
|
||||
Up 10 seconds (healthy)
|
||||
### record at the moment of the cut
|
||||
{"running":true,"started_at":"2026-10-05T14:05:53.282495446Z","interrupted":"0001-01-01T00:00:00Z"}
|
||||
### record after restart
|
||||
{"running":false,"started_at":"2026-10-05T14:05:53.282495446Z","interrupted":"2026-10-05T14:05:53.282495446Z"}
|
||||
### controller log around the cut (previous + new process)
|
||||
2026/10/05 14:05:53 backup.go:557: [INFO] [backup] Starting database dump run
|
||||
2026/10/05 14:05:53 dbdump.go:208: [INFO] [backup] Discovered 2 databases
|
||||
2026/10/05 14:05:53 backup.go:596: [INFO] [backup] Discovered 2 database(s): paperless-postgres(postgres), bookstack-db(mariadb)
|
||||
2026/10/05 14:05:53 dbdump.go:410: [INFO] [backup] DB dump: paperless-postgres → paperless-ngx-postgres.sql (428.1 KB, 321ms, 72 tables)
|
||||
2026/10/05 14:05:54 dbdump.go:410: [INFO] [backup] DB dump: bookstack-db → bookstack-mariadb.sql (55.9 KB, 310ms, 41 tables)
|
||||
2026/10/05 14:05:54 backup.go:974: [INFO] [backup] Stopping bookstack for safe volume dump
|
||||
2026/10/05 14:05:58 backup.go:873: [INFO] [backup] Volume dump: bookstack/bookstack_bookstack_config → 49.0 KB
|
||||
2026/10/05 14:05:59 main.go:340: [INFO] felhom-controller 0.296.0 starting (customer: demo-hp, domain: enkisfelhom.hu)
|
||||
2026/10/05 14:05:59 appstop_marker.go:278: [WARN] [appstop] crash recovery: an app-data backup (volume dump) (op "volume-dump:bookstack") was interrupted and left 1 app(s) stopped — restarting them: [bookstack]
|
||||
2026/10/05 14:05:59 manager.go:1235: [INFO] [stacks] Starting stack: bookstack
|
||||
2026/10/05 14:06:06 appstop_marker.go:304: [INFO] [appstop] crash recovery: restarted bookstack after the interrupted an app-data backup (volume dump)
|
||||
2026/10/05 14:06:06 sync.go:117: [INFO] [sync] Starting catalog sync (repo: https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git, interval: 15m0s)
|
||||
2026/10/05 14:06:06 sync.go:208: [INFO] [sync] Starting catalog sync
|
||||
2026/10/05 14:06:06 restore_record_wiring.go:35: [INFO] [backup] restore record wired: /opt/docker/felhom-controller/data/restore-status.json (interrupted at startup: false)
|
||||
2026/10/05 14:06:06 run_record_wiring.go:27: [WARN] [backup] the app-data backup run started 2026-10-05T14:05:53Z was cut off by the stop — the backup pages say so until the next complete run (R-519)
|
||||
2026/10/05 14:06:06 scheduler.go:223: [INFO] [scheduler] Starting scheduler with 19 jobs
|
||||
2026/10/05 14:06:06 backup.go:1179: [INFO] [backup] Found 3 DB dump files across drives
|
||||
2026/10/05 14:06:06 dbdump.go:208: [INFO] [backup] Discovered 2 databases
|
||||
2026/10/05 14:06:06 [INFO] [backup] Discovered app data: 3 apps
|
||||
2026/10/05 14:06:06 recovery_unit.go:289: [INFO] [backup] Recovery unit captured for bookstack → /mnt/sys_drive/felhom-data/backups/primary/bookstack (images=2, secrets-referenced=3, data_keys=0, portable-carried=2/2, withheld=1)
|
||||
2026/10/05 14:06:06 recovery_unit.go:289: [INFO] [backup] Recovery unit captured for paperless-ngx → /mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx/backups/primary/paperless-ngx (images=3, secrets-referenced=3, data_keys=0, portable-carried=2/2, withheld=1)
|
||||
2026/10/05 14:06:06 backup.go:1250: [INFO] [backup] Backup status cache refreshed
|
||||
2026/10/05 14:06:09 manager.go:1601: [INFO] [stacks] bookstack lscr.io/linuxserver/bookstack:26.09.1@sha256:99cd1f5707c1911afad213adec5c9739763b76f843d1477142231834ecdcb6f7 running Up 3 seconds (health: starting)
|
||||
@@ -0,0 +1,23 @@
|
||||
## after the cut, 2026-10-05T14:06:23Z
|
||||
### GET /backups
|
||||
data-interrupted-run present: 1
|
||||
<div class="alert alert-warning" data-interrupted-run="true">A legutóbbi mentés (2026-10-05 16:05) megszakadt, mert a doboz vagy a vezérlő újraindult. Amit nem fejezett be, annak a korábbi mentése maradt meg — minden visszaállítási pont annyira friss, amennyire a legrégebbi része. A következő teljes mentés után ez az üzenet eltűnik.
|
||||
HTTP 200
|
||||
### GET /backups/apps
|
||||
data-interrupted-run present: 1
|
||||
<div class="alert alert-warning" data-interrupted-run="true">A legutóbbi mentés (2026-10-05 16:05) megszakadt, mert a doboz vagy a vezérlő újraindult. Amit nem fejezett be, annak a korábbi mentése maradt meg — minden visszaállítási pont annyira friss, amennyire a legrégebbi része. A következő teljes mentés után ez az üzenet eltűnik.
|
||||
HTTP 200
|
||||
### negative control: a marker that must NOT be on the page
|
||||
0
|
||||
### restore points (bookstack)
|
||||
{"ok":true,"data":[{"time":"2026-10-05T14:04:44Z","short_id":"helyi","tier":1,"drive_label":"Belső SSD (rendszer)"}]}
|
||||
### dump file times
|
||||
total 314252
|
||||
-rw-r--r-- 1 root root 50176 2026-10-05T14:05:58 bookstack_bookstack_config.tar
|
||||
-rw-r--r-- 1 root root 160868864 2026-10-05T14:04:44 bookstack_bookstack_db_data.tar
|
||||
-rw-r--r-- 1 root root 160867328 2026-10-05T14:05:58 bookstack_bookstack_db_data.tar.tmp
|
||||
drwxr-xr-x 2 root root 4096 2026-10-05T14:05:54 db-dumps
|
||||
drwxr-xr-x 2 root root 4096 2026-10-05T14:05:58 volume-dumps
|
||||
### bookstack after
|
||||
bookstack Up 27 seconds (healthy)
|
||||
bookstack-db Up 32 seconds (healthy)
|
||||
@@ -0,0 +1,12 @@
|
||||
## run 3 (complete) start 2026-10-05T14:06:44Z
|
||||
HTTP 200
|
||||
## run 3 end 2026-10-05T14:07:28Z: {"ok":true,"data":{"db_dump":{"count":2,"duration":"33.315240638s","last_run":"2026-10-05T14:07:20.615183031Z","success":true},"enabled":true,"running":false}}
|
||||
2026/10/05 14:05:12 backup.go:684: [INFO] [backup] App-data backup completed: 2 databases (483.2 KB total), 3 volume dump(s) (32.785s)
|
||||
2026/10/05 14:07:20 backup.go:684: [INFO] [backup] App-data backup completed: 2 databases (483.9 KB total), 3 volume dump(s) (33.315s)
|
||||
{"running":false,"started_at":"0001-01-01T00:00:00Z","interrupted":"0001-01-01T00:00:00Z"}
|
||||
total 157152
|
||||
-rw-r--r-- 1 root root 50688 2026-10-05T14:06:52 bookstack_bookstack_config.tar
|
||||
-rw-r--r-- 1 root root 160867328 2026-10-05T14:06:53 bookstack_bookstack_db_data.tar
|
||||
/backups data-interrupted-run: 0
|
||||
/backups/apps data-interrupted-run: 0
|
||||
restore points: {"ok":true,"data":[{"time":"2026-10-05T14:06:47Z","short_id":"helyi","tier":1,"drive_label":"Belső SSD (rendszer)"}]}
|
||||
@@ -0,0 +1,8 @@
|
||||
## teardown 2026-10-05T14:07:48Z
|
||||
stop: HTTP 200
|
||||
remove: HTTP 200
|
||||
containers: 0
|
||||
volumes: 0
|
||||
stackdir: none
|
||||
backups: none
|
||||
/root/.dbody
|
||||
@@ -0,0 +1,9 @@
|
||||
## runbook §3 drill 2026-10-05T14:09:41Z (scratch /var/lib/felhom-hub-backup/sec3.65qk, root 0700)
|
||||
step 1 restore host/dooplex-hub/2026-10-05T13:50:18Z
|
||||
restore complete (352.676 MiB processed in 3.9s, average 90.961 MiB/s)
|
||||
-rw------- 369807360 hub.db
|
||||
step 2 the seal key from Secret/offsite-secret-key → /var/lib/felhom-hub-backup/sec3.65qk/k (0600, not printed)
|
||||
key file bytes: 64
|
||||
step 3 hubdb-check (same key): hosts=4 console_passwords_opened=4 failed=0 absent=0 rc=0
|
||||
control (a random key): hosts=4 console_passwords_opened=0 failed=4 absent=0 hubdb-check: FAILED: not every console password opened with this key rc=0
|
||||
scratch shredded: gone
|
||||
@@ -0,0 +1,7 @@
|
||||
### R7 hubdb-check counts a failed open as opened
|
||||
=== RUN TestCheck_WrongKeyOpensNothing
|
||||
main_test.go:63: got {hosts:2 opened:1 failed:0 absent:1}, want opened=0 failed=1
|
||||
--- FAIL: TestCheck_WrongKeyOpensNothing (0.03s)
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/cmd/hubdb-check 0.040s
|
||||
FAIL
|
||||
@@ -0,0 +1,17 @@
|
||||
## snapshot list on ep0 with the READ-ONLY token, 2026-10-05T13:50:40Z
|
||||
+=======================================+=============+=====================================+
|
||||
| snapshot | size | files |
|
||||
+=======================================+=============+=====================================+
|
||||
| host/dooplex-hub/2026-10-05T13:50:18Z | 352.677 MiB | catalog.pcat1 hubdb.pxar index.json |
|
||||
+=======================================+=============+=====================================+
|
||||
unit rc=0
|
||||
Result=success
|
||||
2026-10-05T15:50:41+02:00 dooplex felhom-hub-db-restore-test[3040582]: felhom-hub-db-restore-test: restoring host/dooplex-hub/2026-10-05T13:50:18Z
|
||||
2026-10-05T15:50:46+02:00 dooplex felhom-hub-db-restore-test[3040582]: felhom-hub-db-restore-test: checked: integrity ok, 4 host(s), 4 sealed console password(s), 0 readable
|
||||
2026-10-05T15:50:46+02:00 dooplex felhom-hub-db-restore-test[3040582]: felhom-hub-db-restore-test: success signal written
|
||||
# HELP felhom_hub_db_restore_test_last_success_timestamp_seconds Last successful restore test of the hub DB copy on ep0 (R-173).
|
||||
# TYPE felhom_hub_db_restore_test_last_success_timestamp_seconds gauge
|
||||
felhom_hub_db_restore_test_last_success_timestamp_seconds 1791208246
|
||||
.cache
|
||||
.kube
|
||||
stage
|
||||
@@ -0,0 +1,12 @@
|
||||
## 2026-10-05T13:51:07Z token limits (each line = the tool output)
|
||||
push forget : Error: permission check failed - missing Datastore.Modify|Datastore.Prune on /datastore/felhom-offsite/operator
|
||||
restore forget : Error: permission check failed - missing Datastore.Modify|Datastore.Prune on /datastore/felhom-offsite/operator
|
||||
restore backup : Error: missing permissions 'Datastore.Backup' on '/datastore/felhom-offsite/operator'
|
||||
push list ns root (households): Error: permission check failed - missing Datastore.Audit|Datastore.Backup on /datastore/felhom-offsite
|
||||
restore list ns root (households): Error: permission check failed - missing Datastore.Audit|Datastore.Backup on /datastore/felhom-offsite
|
||||
push restore own copy: Error: missing key - manifest was created with key b2:19:bf:36:3b:97:3d:6c exit-file=absent
|
||||
## paper-key proof: restore with a key file rebuilt from the data field only
|
||||
restore rc=0
|
||||
integrity: ok hosts: 4
|
||||
## and with NO key: Error: missing key - manifest was created with key b2:19:bf:36:3b:97:3d:6c
|
||||
push restore own copy WITH key: restore complete (352.676 MiB processed in 4.6s, average 76.793 MiB/s) file=PRESENT
|
||||
Reference in New Issue
Block a user