From bcaa9576b263c36edb53349c17a180af16e48d78 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 20 Oct 2026 10:47:37 +0200 Subject: [PATCH] Night 2 evidence: P2 R-893 option A live proof on 9202 (forced failure put back exactly; normal restore OK; teardown); P5 R-462 coverage and currency list --- .../night2-2026-10-20/P2/live/00-baseline.txt | 55 ++++ .../night2-2026-10-20/P2/live/01-upgrade.txt | 16 + .../P2/live/02-sftp-target.txt | 10 + .../night2-2026-10-20/P2/live/03-deploy.txt | 3 + .../night2-2026-10-20/P2/live/04-marker-A.txt | 23 ++ .../P2/live/05-snapshot-1.txt | 18 + .../P2/live/06-snapshot-2.txt | 2 + .../P2/live/07-live-before-restore.txt | 21 ++ .../P2/live/08-prepare-and-inject.txt | 27 ++ .../P2/live/09-undo-folder-watch.txt | 5 + .../P2/live/10-forced-failure-http.txt | 3 + .../P2/live/11-controller-log-window-full.txt | 307 ++++++++++++++++++ .../P2/live/11-controller-log-window.txt | 29 ++ .../P2/live/12-after-failed-restore.txt | 24 ++ .../P2/live/13-normal-prepare.txt | 5 + .../P2/live/14-undo-folder-watch-normal.txt | 5 + .../P2/live/15-normal-restore-http.txt | 3 + .../P2/live/16-controller-log-normal-full.txt | 191 +++++++++++ .../P2/live/16-controller-log-normal.txt | 23 ++ .../P2/live/17-after-normal-restore.txt | 20 ++ .../night2-2026-10-20/P2/live/18-teardown.txt | 79 +++++ .../night2-2026-10-20/P2/live/README.md | 161 +++++++++ .../night2-2026-10-20/P2/live/tool-obs.sh.txt | 22 ++ .../audits/night2-2026-10-20/P5/r462-list.txt | 36 ++ 24 files changed, 1088 insertions(+) create mode 100644 documentation/audits/night2-2026-10-20/P2/live/00-baseline.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/01-upgrade.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/02-sftp-target.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/03-deploy.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/04-marker-A.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/05-snapshot-1.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/06-snapshot-2.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/07-live-before-restore.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/08-prepare-and-inject.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/09-undo-folder-watch.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/10-forced-failure-http.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window-full.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/12-after-failed-restore.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/13-normal-prepare.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/14-undo-folder-watch-normal.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/15-normal-restore-http.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal-full.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/17-after-normal-restore.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/18-teardown.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/live/README.md create mode 100644 documentation/audits/night2-2026-10-20/P2/live/tool-obs.sh.txt create mode 100644 documentation/audits/night2-2026-10-20/P5/r462-list.txt diff --git a/documentation/audits/night2-2026-10-20/P2/live/00-baseline.txt b/documentation/audits/night2-2026-10-20/P2/live/00-baseline.txt new file mode 100644 index 00000000..905b156d --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/00-baseline.txt @@ -0,0 +1,55 @@ +=== baseline Tue Oct 20 07:25:33 UTC 2026 +image file: gitea.dooplex.hu/admin/felhom-controller:0.331.0 +paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15 Up 7 hours (healthy) +paperless-postgres postgres:18-alpine Up 7 hours (healthy) +paperless-redis redis:7-alpine Up 7 hours (healthy) +immich-server ghcr.io/immich-app/immich-server:v3.2.4 Up 7 hours (healthy) +immich-postgres ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 Up 7 hours (healthy) +immich-machine-learning ghcr.io/immich-app/immich-machine-learning:v3.2.4 Up 7 hours (healthy) +immich-redis redis:7-alpine Up 7 hours (healthy) +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.331.0 Up 12 hours (healthy) +filebrowser gtstef/filebrowser:1.5.6-stable Up 12 hours (healthy) +traefik traefik:v3.7.13 Up 4 days +--- images present at baseline (names only) +alpine:latest +curlimages/curl:8.11.1 +ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +ghcr.io/puppeteer/puppeteer:latest +gitea.dooplex.hu/admin/felhom-controller:0.314.0-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.326.0-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.326.0-rc.2 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.2 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.3 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.4 +gitea.dooplex.hu/admin/felhom-controller:0.327.1-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.331.0 +gitea.dooplex.hu/admin/felhom-controller:0.331.0-rc.1 +gitea.dooplex.hu/admin/felhom-samba:1.2.0 +gtstef/filebrowser:1.3.3-stable +gtstef/filebrowser:1.5.6-stable +postgres:18-alpine +redis:7-alpine +redis:7.4-alpine +traefik:v3.6.7 +traefik:v3.7.13 +--- rallly template dir (pre-existing) +.felhom.yml +docker-compose.yml +hold-logs +--- rallly volumes: 0 +/dev/loop1 69G 11G 54G 17% /var/lib/felhom +--- offbox folder (listing only) +total 20 +drwx------ 2 root root 4096 Sep 30 08:40 . +drwxr-xr-x 4 root root 4096 Oct 20 07:25 .. +-rw-r--r-- 1 root root 91 Sep 30 08:40 known_hosts +-rw------- 1 root root 64 Sep 30 08:40 repo_password +-rw------- 1 root root 411 Sep 30 08:40 ssh_key +cb6b18f87662cf43d7e100f37366ce639ee5593fbf505751f5c1d831f8053fd1 settings.json +-rw-r--r-- 1 root root 10240 Oct 20 07:25 /root/r893b-offbox.before.tar +-rw-r--r-- 1 root root 27003 Oct 20 01:30 /root/r893b-settings.json.before +offbox target set: False +per-app offbox: None +restore holds: ['chaoscrash', 'chaosoom', 'chaosoomb', 'gokapi'] +top-level keys: ['app_backup', 'app_update', 'app_update_events_seeded', 'backup_window_start', 'claim_consumed_generation', 'claimed', 'db_validations', 'filebrowser_admin_decided_at', 'filebrowser_admin_password_enc', 'filebrowser_admin_state', 'language', 'offbox_enlarge_notice_seeded', 'password_hash', 'restore_holds', 'storage_paths', 'unhealthy_stop_event_seeded', 'unhealthy_stops'] +--- controller [ERROR] lines since container start: 0 diff --git a/documentation/audits/night2-2026-10-20/P2/live/01-upgrade.txt b/documentation/audits/night2-2026-10-20/P2/live/01-upgrade.txt new file mode 100644 index 00000000..3916b7d5 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/01-upgrade.txt @@ -0,0 +1,16 @@ +=== upgrade to 0.336.0-rc.1 Tue Oct 20 07:25:47 UTC 2026 +gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1 + +template parsing error: template: :1:23: executing "" at <.Config.Labels>: map has no entry for key "Labels" +restart rc=0 +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1 Up 8 seconds (healthy) +2026/10/20 07:25:49 undo.go:676: [INFO] [stacks] applied-meta backfill (R-646): recorded 0 []; skipped 0 [] — not current with the catalog, their pinned version's .felhom.yml is no longer on the box +2026/10/20 07:25:49 scheduler.go:120: [INFO] [scheduler] Registered periodic job: conversion-copy-release (every 1h0m0s) +2026/10/20 07:25:49 scheduler.go:85: [DEBUG] [scheduler] periodic job registered: name="conversion-copy-release" interval=1h0m0s totalJobs=13 +[ERROR] lines after start: 0 +image id sha256:0756d4d2b1c83a666c0d8b40b526983ac13fa792111a0ddbb8f430ba2f9588c3 +2026/10/20 07:25:49 main.go:353: [INFO] felhom-controller 0.336.0-rc.1 starting (customer: demo-hp, domain: enkisfelhom.hu) +2026/10/20 07:25:49 server.go:315: [DEBUG] [web] NewServer: initializing web server v0.336.0-rc.1 +2026/10/20 07:25:50 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 7ea0989d5c8d felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1 +{"ok":true,"message":"felhom-controller is healthy"} + diff --git a/documentation/audits/night2-2026-10-20/P2/live/02-sftp-target.txt b/documentation/audits/night2-2026-10-20/P2/live/02-sftp-target.txt new file mode 100644 index 00000000..d1524ec4 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/02-sftp-target.txt @@ -0,0 +1,10 @@ +=== scratch SFTP target Tue Oct 20 07:27:37 UTC 2026 +bridge=172.17.0.2 felhom-filebrowser=172.21.0.3 traefik-public=172.18.0.2 +docker.io/atmoz/sftp:alpine +5167fd39cf22cc30c1f0507e59377916d921640e12e08ddb4f6318db2c42fc34 +r893b-sftp atmoz/sftp:alpine Up 6 seconds 0.0.0.0:2222->22/tcp, [::]:2222->22/tcp +[172.17.0.1]:2222 ssh-ed25519 AAAAC3NzaC +=== off-site target configured through the household's endpoint 07:27:57 +07:27:57 POST /backup/offbox/config -> 302 Location: /backups/remote?flash_error=flash.offbox.target_saved_escrow_agent_down +07:27:57 POST /backup/offbox/confirm-escrow -> 302 Location: /backups/remote?flash=flash.offbox.escrow_confirmed +07:27:58 POST /backup/offbox/toggle -> 302 Location: /backups/remote?flash=flash.offbox.app_setting_updated diff --git a/documentation/audits/night2-2026-10-20/P2/live/03-deploy.txt b/documentation/audits/night2-2026-10-20/P2/live/03-deploy.txt new file mode 100644 index 00000000..e0cbb15d --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/03-deploy.txt @@ -0,0 +1,3 @@ +09:28:03 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +09:28:53 [1] deployed, controller state=running, pinned={'rallly': 'lukevella/rallly:4.15.3', 'rallly-postgres': 'postgres:18-alpine'} +deploy: True diff --git a/documentation/audits/night2-2026-10-20/P2/live/04-marker-A.txt b/documentation/audits/night2-2026-10-20/P2/live/04-marker-A.txt new file mode 100644 index 00000000..990e048e --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/04-marker-A.txt @@ -0,0 +1,23 @@ +=== A-written +=== marker A written 07:29:13 UTC +rallly lukevella/rallly:4.15.3 Up 43 seconds (healthy) +rallly-postgres postgres:18-alpine Up 48 seconds (healthy) +markers (phase=rows): A-before-snapshot-1=1 +r893_after exists: 0 +r893_only_old exists: 0 +r893_new2 exists: 0 +public tables: 34 +prisma migrations: 153 +db volume created: 2026-10-20T07:28:21Z +schema-only md5: 8c473c01fccf52724e62a1d73867b2cc (control, second dump: 8c473c01fccf52724e62a1d73867b2cc) +pg_dump: warning: there are circular foreign-key constraints among these tables: +pg_dump: detail: users +pg_dump: detail: provider_calendars +pg_dump: detail: calendar_connections +pg_dump: hint: You might not be able to restore the dump without using --disable-triggers or temporarily dropping the constraints. +pg_dump: hint: Consider using a full dump instead of a --data-only dump to avoid this problem. +data-only md5: b914f0553434b56f021043dd5946c2fe +per-table row counts md5: 72061ad756a2f2d3842ad58d92879db2 +per-table row counts: _prisma_migrations=153 accounts=0 calendar_connections=0 comments=0 credentials=0 event_types=0 instance_licenses=0 instance_settings=1 license_validations=0 licenses=0 options=0 participants=0 payment_methods=0 poll_activities=0 poll_invites=0 polls=0 provider_calendars=0 r893_marker=1 registered_instances=0 rescheduled_event_dates=0 scheduled_event_invites=0 scheduled_events=0 sessions=0 sheet_slots=0 sheets=0 space_api_keys=0 space_member_invites=0 space_members=0 spaces=0 subscriptions=0 user_notification_preferences=0 users=0 verifications=0 votes=0 +definition: image: lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +restore-undo folder: ls: cannot access '/mnt/sys_drive/felhom-data/backups/restore-undo/rallly': No such file or directory diff --git a/documentation/audits/night2-2026-10-20/P2/live/05-snapshot-1.txt b/documentation/audits/night2-2026-10-20/P2/live/05-snapshot-1.txt new file mode 100644 index 00000000..a9ceaf83 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/05-snapshot-1.txt @@ -0,0 +1,18 @@ +07:29:29 POST /backup/offbox/run -> 302 Location: /backups/remote?flash=flash.offbox.run_started +07:30:39 {"status": "ok", "snapshots": 1, "last_run": "2026-10-20T07:30:38Z", "last_duration": "1m8s", "repo_size_human": "7.2 MB", "last_error": ""} +=== before snapshot 2 (markers B, C; r893_after, r893_only_old) 07:30:50 UTC +rallly lukevella/rallly:4.15.3 Up 20 seconds (healthy) +rallly-postgres postgres:18-alpine Up 25 seconds (healthy) +markers (phase=rows): A-before-snapshot-1=1, B-after-snapshot-1=1, C-in-snapshot-2=1 +r893_after exists: 1 rows: 1 +r893_only_old exists: 1 rows: 1 +r893_new2 exists: 0 +public tables: 36 +prisma migrations: 153 +db volume created: 2026-10-20T07:28:21Z +schema-only md5: f872bd54b5e09991876d465efb13d754 (control, second dump: f872bd54b5e09991876d465efb13d754) +data-only md5: db5903abeeb4c3a6da32016ad71f01bb +per-table row counts md5: c032377e7ce421083e40fb875cfadb34 +per-table row counts: _prisma_migrations=153 accounts=0 calendar_connections=0 comments=0 credentials=0 event_types=0 instance_licenses=0 instance_settings=1 license_validations=0 licenses=0 options=0 participants=0 payment_methods=0 poll_activities=0 poll_invites=0 polls=0 provider_calendars=0 r893_after=1 r893_marker=3 r893_only_old=1 registered_instances=0 rescheduled_event_dates=0 scheduled_event_invites=0 scheduled_events=0 sessions=0 sheet_slots=0 sheets=0 space_api_keys=0 space_member_invites=0 space_members=0 spaces=0 subscriptions=0 user_notification_preferences=0 users=0 verifications=0 votes=0 +definition: image: lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +restore-undo folder: ls: cannot access '/mnt/sys_drive/felhom-data/backups/restore-undo/rallly': No such file or directory diff --git a/documentation/audits/night2-2026-10-20/P2/live/06-snapshot-2.txt b/documentation/audits/night2-2026-10-20/P2/live/06-snapshot-2.txt new file mode 100644 index 00000000..8ce4d615 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/06-snapshot-2.txt @@ -0,0 +1,2 @@ +07:30:56 POST /backup/offbox/run -> 302 Location: /backups/remote?flash=flash.offbox.run_started +07:32:07 {"status": "ok", "snapshots": 2, "last_run": "2026-10-20T07:32:04Z", "last_duration": "1m6s", "repo_size_human": "10.1 MB", "last_error": ""} diff --git a/documentation/audits/night2-2026-10-20/P2/live/07-live-before-restore.txt b/documentation/audits/night2-2026-10-20/P2/live/07-live-before-restore.txt new file mode 100644 index 00000000..21968038 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/07-live-before-restore.txt @@ -0,0 +1,21 @@ +=== LIVE before the restore 07:32:38 UTC +rallly lukevella/rallly:4.15.3 Up 40 seconds (healthy) +rallly-postgres postgres:18-alpine Up 45 seconds (healthy) +markers (phase=rows): A-before-snapshot-1=1, B-after-snapshot-1=1, C-in-snapshot-2=1, D-after-snapshot-2=1 +r893_after exists: 1 rows: 1 +r893_only_old exists: 0 +r893_new2 exists: 1 rows: 1 +public tables: 36 +prisma migrations: 153 +db volume created: 2026-10-20T07:28:21Z +schema-only md5: 05828c7714605cb5f2a6b3b691b52a22 (control, second dump: 05828c7714605cb5f2a6b3b691b52a22) +data-only md5: e5c1a72ffb7107fef1d06e918b78194b +per-table row counts md5: d8bcbc4ba364fe3fdee69a2e773cf5be +per-table row counts: _prisma_migrations=153 accounts=0 calendar_connections=0 comments=0 credentials=0 event_types=0 instance_licenses=0 instance_settings=1 license_validations=0 licenses=0 options=0 participants=0 payment_methods=0 poll_activities=0 poll_invites=0 polls=0 provider_calendars=0 r893_after=1 r893_marker=4 r893_new2=1 registered_instances=0 rescheduled_event_dates=0 scheduled_event_invites=0 scheduled_events=0 sessions=0 sheet_slots=0 sheets=0 space_api_keys=0 space_member_invites=0 space_members=0 spaces=0 subscriptions=0 user_notification_preferences=0 users=0 verifications=0 votes=0 +definition: image: lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +restore-undo folder: ls: cannot access '/mnt/sys_drive/felhom-data/backups/restore-undo/rallly': No such file or directory +markers (phase=rows): A-before-snapshot-1=1, B-after-snapshot-1=1, C-in-snapshot-2=1, D-after-snapshot-2=1 +public tables: 36 +schema-only md5: 05828c7714605cb5f2a6b3b691b52a22 (control, second dump: 05828c7714605cb5f2a6b3b691b52a22) +data-only md5: e5c1a72ffb7107fef1d06e918b78194b +per-table row counts md5: d8bcbc4ba364fe3fdee69a2e773cf5be diff --git a/documentation/audits/night2-2026-10-20/P2/live/08-prepare-and-inject.txt b/documentation/audits/night2-2026-10-20/P2/live/08-prepare-and-inject.txt new file mode 100644 index 00000000..b7d8c4b4 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/08-prepare-and-inject.txt @@ -0,0 +1,27 @@ +07:32:57 POST /backup/offbox/restore -> 302 Location: /backups/restore/app?name=rallly&full_prep=rallly&full_size=66.1 MB +07:32:57 POST /backup/offbox/restore -> 302 Location: /backups/restore/app?name=rallly&flash=flash.offbox.restore_started +07:33:01 200 {"running": false, "op": "offbox-restore", "stack": "rallly", "started_at": "2026-10-20T07:32:57.875089287Z", "last": {"op": "offbox-restore", "stack": "rallly", "ok": true, "message": "A(z) rallly teljes mentése visszaállítva ellenőrző mappába: /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly — a saját fájljaiddal együtt. A meglévő adatok változatlanok.", "finished_at": "2026-10-20T07:33:00.816791263Z"}, "last_recent": true} +=== prepared scratch 07:33:13 +total 16 +drwxr-xr-x 3 root root 4096 Oct 20 07:33 . +drwxr-xr-x 3 root root 4096 Oct 20 07:32 .. +-rw------- 1 root root 86 Oct 20 07:33 .felhom-restore-complete.json +drwxr-xr-x 3 root root 4096 Oct 14 14:05 mnt +/mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps: +total 120 +drwxr-xr-x 2 root root 4096 Oct 20 07:30 . +drwxr-xr-x 5 root root 4096 Oct 20 07:31 .. +-rw-r--r-- 1 root root 111959 Oct 20 07:30 rallly-postgres.sql + +/mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/volume-dumps: +total 67580 +drwxr-xr-x 2 root root 4096 Oct 20 07:31 . +drwxr-xr-x 5 root root 4096 Oct 20 07:31 .. +-rw-r--r-- 1 root root 69190656 Oct 20 07:31 rallly_rallly_postgres_data.tar +snapshot-2 content check: r893_only_old mentions=5 r893_new2 mentions=0 marker C row=1 marker D row=0 +completion markers before: 1 +injected before line 2700: +2699--- +2700:SELECT r893_forced_replay_failure(); -- R-893 option A proof 2026-10-20: forced error AFTER the volumes are replaced +2701--- PostgreSQL database dump complete +lines: 2705 diff --git a/documentation/audits/night2-2026-10-20/P2/live/09-undo-folder-watch.txt b/documentation/audits/night2-2026-10-20/P2/live/09-undo-folder-watch.txt new file mode 100644 index 00000000..0b952521 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/09-undo-folder-watch.txt @@ -0,0 +1,5 @@ +07:33:40.967 restore-undo: rallly 4096;rallly/20261020T073340Z 4096;rallly/20261020T073340Z/rallly_rallly_postgres_data.tar.tmp 48553984; +07:33:41.275 restore-undo: rallly 4096;rallly/20261020T073340Z 4096;rallly/20261020T073340Z/rallly_rallly_postgres_data.tar.tmp 69198848; +07:33:41.583 restore-undo: rallly 4096;rallly/20261020T073340Z 4096;rallly/20261020T073340Z/rallly_rallly_postgres_data.tar 69198848; +07:33:54.453 restore-undo: +watch end 07:35:57 diff --git a/documentation/audits/night2-2026-10-20/P2/live/10-forced-failure-http.txt b/documentation/audits/night2-2026-10-20/P2/live/10-forced-failure-http.txt new file mode 100644 index 00000000..88f21209 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/10-forced-failure-http.txt @@ -0,0 +1,3 @@ +=== reconstitute (forced failure) T0=2026-10-20T07:33:27Z +07:33:28 POST /backup/offbox/reconstitute -> 302 Location: /backups/restore/app?name=rallly&flash=flash.offbox.full_restore_started +07:33:55 200 {"running": false, "op": "offbox-reconstitute", "stack": "rallly", "started_at": "2026-10-20T07:33:28.242768479Z", "last": {"op": "offbox-reconstitute", "stack": "rallly", "ok": false, "message": "A teljes visszaállítás sikertelen: a(z) rallly visszaállítása sikertelen. Nem változott semmi: az alkalmazás adatai visszakerültek a visszaállítás előtti állapotba, és az alkalmazás újra fut", "finished_at": "2026-10-20T07:33:54.173657757Z"}, "last_recent": true} diff --git a/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window-full.txt b/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window-full.txt new file mode 100644 index 00000000..ff38d4fe --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window-full.txt @@ -0,0 +1,307 @@ +2026/10/20 07:33:28 auth.go:160: [DEBUG] [web] login attempt: visitor 192.168.0.180 (peer 172.18.0.3:34320, X-Forwarded-For "192.168.0.180", CF-Connecting-IP "") +2026/10/20 07:33:28 auth.go:192: [DEBUG] [web] login successful from 192.168.0.180, creating session +2026/10/20 07:33:28 session_store.go:140: [DEBUG] [web] sessions: saved 145 session(s) to dashboard-sessions.json +2026/10/20 07:33:28 auth.go:316: [DEBUG] [web] session created, expires=2026-10-27T07:33:28Z, active_sessions=145 +2026/10/20 07:33:28 auth.go:212: [INFO] [web] Login from 192.168.0.180 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for GET / +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: GET / from 172.18.0.3:34320 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for GET /launcher +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: GET /launcher from 172.18.0.3:34320 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for POST /backup/offbox/reconstitute +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: POST /backup/offbox/reconstitute from 172.18.0.3:34320 +2026/10/20 07:33:28 auth.go:160: [DEBUG] [web] login attempt: visitor 192.168.0.180 (peer 172.18.0.3:34320, X-Forwarded-For "192.168.0.180", CF-Connecting-IP "") +2026/10/20 07:33:28 auth.go:192: [DEBUG] [web] login successful from 192.168.0.180, creating session +2026/10/20 07:33:28 session_store.go:140: [DEBUG] [web] sessions: saved 146 session(s) to dashboard-sessions.json +2026/10/20 07:33:28 auth.go:316: [DEBUG] [web] session created, expires=2026-10-27T07:33:28Z, active_sessions=146 +2026/10/20 07:33:28 auth.go:212: [INFO] [web] Login from 192.168.0.180 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for GET / +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: GET / from 172.18.0.3:34320 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for GET /launcher +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: GET /launcher from 172.18.0.3:34320 +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:28 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:28 [WARN] [backup] ValidateDump: /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql is structurally valid (36 tables) but its accounts table has NO rows — the dump may predate the customer's data +2026/10/20 07:33:28 offbox_reconstitute.go:1253: [WARN] [offbox] rallly: the snapshot dump rallly-postgres.sql has no account rows — it may predate the customer's data +2026/10/20 07:33:29 restore_dbcreds.go:114: [INFO] [backup] Restore rallly: database password(s) [DB_PASSWORD] come from the backup, [] stay as installed (older than the data) — values not logged (R-1106) +2026/10/20 07:33:29 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/10/20 07:33:29 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 74f5e7510c45 rallly rallly lukevella/rallly:4.15.3 +d0758337136e rallly-postgres rallly postgres:18-alpine +2d8767d09c82 paperless-webserver paperless-ngx ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +5ac453c47c19 paperless-postgres paperless-ngx postgres:18-alpine +ac5fbd6a551f paperless-redis paperless-ngx redis:7-alpine +a5743e7ad6c9 immich-server immich ghcr.io/immich-app/immich-server:v3.2.4 +66a2711b7a0d immich-postgres immich ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 +4e0f1ce... +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container rallly (image=lukevella/rallly:4.15.3, not a database) +2026/10/20 07:33:29 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: rallly-postgres (id=d0758337136e) +2026/10/20 07:33:29 dbdump.go:198: [DEBUG] DiscoverDatabases: rallly-postgres → stack=rallly, dbUser=rallly, dbName=rallly +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/10/20 07:33:29 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5ac453c47c19) +2026/10/20 07:33:29 dbdump.go:915: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/10/20 07:33:29 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-server (image=ghcr.io/immich-app/immich-server:v3.2.4, not a database) +2026/10/20 07:33:29 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: immich-postgres (id=66a2711b7a0d) +2026/10/20 07:33:29 dbdump.go:198: [DEBUG] DiscoverDatabases: immich-postgres → stack=immich, dbUser=immich, dbName=immich +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-machine-learning (image=ghcr.io/immich-app/immich-machine-learning:v3.2.4, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container r893b-sftp (image=atmoz/sftp:alpine, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.5.6-stable, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1, not a database) +2026/10/20 07:33:29 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.7.13, not a database) +2026/10/20 07:33:29 dbdump.go:205: [DEBUG] DiscoverDatabases: found 3 database(s), skipped 10 non-DB container(s) +2026/10/20 07:33:29 dbdump.go:208: [INFO] [backup] Discovered 3 databases +2026/10/20 07:33:29 dbdump.go:277: [DEBUG] DumpOne: starting dump for container=rallly-postgres, stack=rallly, dbType=postgres, dest=/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073329Z-rallly-postgres.sql +2026/10/20 07:33:29 dbdump.go:314: [DEBUG] DumpOne: pg_dump command: docker exec d0758337136e pg_dump -U rallly -d rallly --clean --if-exists --no-owner --no-privileges +2026/10/20 07:33:29 [WARN] [backup] ValidateDump: /mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073329Z-rallly-postgres.sql is structurally valid (36 tables) but its accounts table has NO rows — the dump may predate the customer's data +2026/10/20 07:33:29 dbdump.go:424: [DEBUG] DumpOne: completed rallly-postgres → pre-restore-20261020T073329Z-rallly-postgres.sql (size=109.4 KB, valid=true, tables=36, duration=298ms) +2026/10/20 07:33:29 dbdump.go:429: [INFO] [backup] DB dump: rallly-postgres → pre-restore-20261020T073329Z-rallly-postgres.sql (109.4 KB, 298ms, 36 tables) +2026/10/20 07:33:29 offbox_reconstitute.go:262: [INFO] [offbox] rallly: pre-restore safety dump written → pre-restore-20261020T073329Z-rallly-postgres.sql (109.4 KB) +2026/10/20 07:33:29 manager.go:1437: [DEBUG] [stacks] StopStack rallly: current state=running deployed=true containers=2 +2026/10/20 07:33:29 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:33:29 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:33:29 manager.go:1695: [DEBUG] Running: docker compose down (in /opt/docker/stacks/rallly) +2026/10/20 07:33:29 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:33:29 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:29 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:33:29 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:29 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping rallly — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:29 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (3 skipped not due, 0 skipped no container) +2026/10/20 07:33:30 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:33:31 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:31 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:34 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:34 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:34 scheduler.go:85: [DEBUG] [scheduler] job docker-socket-watch: execution starting +2026/10/20 07:33:37 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:37 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:39 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:33:39 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:33:40 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:33:40 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:40 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:40 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping rallly — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:40 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (3 skipped not due, 0 skipped no container) +2026/10/20 07:33:40 manager.go:1717: [DEBUG] Command completed: docker compose down (took 10.8s) +2026/10/20 07:33:40 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 10.8s) +2026/10/20 07:33:40 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:40 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:40 manager.go:838: [INFO] [stacks] Status refresh: 9 containers across 70 stacks +2026/10/20 07:33:41 offbox_restore_undo.go:154: [INFO] [offbox] rallly: the live volumes are copied before the restore (1 copied, 0 new from the snapshot) in 867ms — /mnt/sys_drive/felhom-data/backups/restore-undo/rallly/20261020T073340Z (R-893) +2026/10/20 07:33:41 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:33:42 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:33:42 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:33:42 manager.go:1404: [INFO] [stacks] Starting stack rallly services only: [rallly-postgres] +2026/10/20 07:33:42 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:33:42 manager.go:1695: [DEBUG] Running: docker compose up -d rallly-postgres (in /opt/docker/stacks/rallly) +2026/10/20 07:33:43 manager.go:1717: [DEBUG] Command completed: docker compose up -d rallly-postgres (took 0.4s) +2026/10/20 07:33:43 manager.go:1415: [INFO] [stacks] Stack rallly services [rallly-postgres] started (took 0.4s) +2026/10/20 07:33:43 manager.go:838: [INFO] [stacks] Status refresh: 10 containers across 70 stacks +2026/10/20 07:33:43 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/10/20 07:33:43 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 306e430406e4 rallly-postgres rallly postgres:18-alpine +2d8767d09c82 paperless-webserver paperless-ngx ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +5ac453c47c19 paperless-postgres paperless-ngx postgres:18-alpine +ac5fbd6a551f paperless-redis paperless-ngx redis:7-alpine +a5743e7ad6c9 immich-server immich ghcr.io/immich-app/immich-server:v3.2.4 +66a2711b7a0d immich-postgres immich ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 +4e0f1ce12bf2 immich-machine-learning immich ghcr.io/immich... +2026/10/20 07:33:43 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: rallly-postgres (id=306e430406e4) +2026/10/20 07:33:43 dbdump.go:198: [DEBUG] DiscoverDatabases: rallly-postgres → stack=rallly, dbUser=rallly, dbName=rallly +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/10/20 07:33:43 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5ac453c47c19) +2026/10/20 07:33:43 dbdump.go:915: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/10/20 07:33:43 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-server (image=ghcr.io/immich-app/immich-server:v3.2.4, not a database) +2026/10/20 07:33:43 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: immich-postgres (id=66a2711b7a0d) +2026/10/20 07:33:43 dbdump.go:198: [DEBUG] DiscoverDatabases: immich-postgres → stack=immich, dbUser=immich, dbName=immich +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-machine-learning (image=ghcr.io/immich-app/immich-machine-learning:v3.2.4, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container r893b-sftp (image=atmoz/sftp:alpine, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.5.6-stable, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1, not a database) +2026/10/20 07:33:43 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.7.13, not a database) +2026/10/20 07:33:43 dbdump.go:205: [DEBUG] DiscoverDatabases: found 3 database(s), skipped 9 non-DB container(s) +2026/10/20 07:33:43 dbdump.go:208: [INFO] [backup] Discovered 3 databases +2026/10/20 07:33:43 restore_db.go:87: [INFO] [backup] Restore rallly: replaying DB dump into rallly-postgres (postgres) +2026/10/20 07:33:43 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:33:43 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:43 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:44 dbdump.go:814: [ERROR] [backup] postgres import into rallly-postgres FAILED: exit status 3 — engine output follows: +ERROR: function r893_forced_replay_failure() does not exist +LINE 1: SELECT r893_forced_replay_failure(); + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +2026/10/20 07:33:44 offbox_reconstitute.go:1100: [ERROR] [offbox] rallly: database replay failed, rolling back to the pre-restore state: importing postgres dump for rallly: postgres import into rallly-postgres failed: exit status 3 +2026/10/20 07:33:44 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/10/20 07:33:44 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 306e430406e4 rallly-postgres rallly postgres:18-alpine +2d8767d09c82 paperless-webserver paperless-ngx ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +5ac453c47c19 paperless-postgres paperless-ngx postgres:18-alpine +ac5fbd6a551f paperless-redis paperless-ngx redis:7-alpine +a5743e7ad6c9 immich-server immich ghcr.io/immich-app/immich-server:v3.2.4 +66a2711b7a0d immich-postgres immich ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 +4e0f1ce12bf2 immich-machine-learning immich ghcr.io/immich... +2026/10/20 07:33:44 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: rallly-postgres (id=306e430406e4) +2026/10/20 07:33:44 dbdump.go:198: [DEBUG] DiscoverDatabases: rallly-postgres → stack=rallly, dbUser=rallly, dbName=rallly +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/10/20 07:33:44 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5ac453c47c19) +2026/10/20 07:33:44 dbdump.go:915: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/10/20 07:33:44 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-server (image=ghcr.io/immich-app/immich-server:v3.2.4, not a database) +2026/10/20 07:33:44 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: immich-postgres (id=66a2711b7a0d) +2026/10/20 07:33:44 dbdump.go:198: [DEBUG] DiscoverDatabases: immich-postgres → stack=immich, dbUser=immich, dbName=immich +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-machine-learning (image=ghcr.io/immich-app/immich-machine-learning:v3.2.4, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-redis (image=redis:7-alpine, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container r893b-sftp (image=atmoz/sftp:alpine, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.5.6-stable, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1, not a database) +2026/10/20 07:33:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.7.13, not a database) +2026/10/20 07:33:44 dbdump.go:205: [DEBUG] DiscoverDatabases: found 3 database(s), skipped 9 non-DB container(s) +2026/10/20 07:33:44 dbdump.go:208: [INFO] [backup] Discovered 3 databases +2026/10/20 07:33:44 offbox_reconstitute.go:603: [DEBUG] [offbox] rallly: rallly-postgres was re-created during the restore (d0758337136e → 306e430406e4) — rolling back into the live container +2026/10/20 07:33:44 offbox_reconstitute.go:606: [INFO] [offbox] rallly: rolling back to the pre-restore state from pre-restore-20261020T073329Z-rallly-postgres.sql +2026/10/20 07:33:44 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073329Z-rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:33:45 dbdump.go:820: [INFO] [backup] Imported DB dump pre-restore-20261020T073329Z-rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:33:45 offbox_reconstitute.go:611: [INFO] [offbox] rallly: rollback complete — 1 database(s) returned to the pre-restore state +2026/10/20 07:33:45 manager.go:1437: [DEBUG] [stacks] StopStack rallly: current state=starting deployed=true containers=1 +2026/10/20 07:33:45 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:33:45 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:33:45 manager.go:1695: [DEBUG] Running: docker compose down (in /opt/docker/stacks/rallly) +2026/10/20 07:33:46 manager.go:1717: [DEBUG] Command completed: docker compose down (took 0.9s) +2026/10/20 07:33:46 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 0.9s) +2026/10/20 07:33:46 manager.go:838: [INFO] [stacks] Status refresh: 9 containers across 70 stacks +2026/10/20 07:33:46 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:33:46 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:46 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:47 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:33:47 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:33:47 offbox_restore_undo.go:232: [WARN] [offbox] rallly: the restore failed (importing postgres dump for rallly: postgres import into rallly-postgres failed: exit status 3) — the live volumes and definition are put back; starting the app (R-893) +2026/10/20 07:33:47 notifier.go:1373: [WARN] notifier disabled (no hub configured): DROPPED event backup_run_failures (severity error) — further backup_run_failures events are logged at DEBUG only +2026/10/20 07:33:47 manager.go:1348: [DEBUG] [stacks] StartStack rallly: current state=stopped deployed=true +2026/10/20 07:33:47 manager.go:1351: [INFO] [stacks] Starting stack: rallly +2026/10/20 07:33:47 manager.go:1358: [DEBUG] [stacks] StartStack rallly: prepared 10 env vars for compose +2026/10/20 07:33:47 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:33:47 manager.go:1695: [DEBUG] Running: docker compose up -d (in /opt/docker/stacks/rallly) +2026/10/20 07:33:49 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:49 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714348KB) → used≈12498MB avail≈13400MB +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714348 availKB=15892428 → total=25898MB avail=13400MB used=12498MB (48.3%) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=12.1GB avail=52.7GB (18.6%) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=193.7GB avail=696.4GB (21.8%) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.95 2.29 1.70 5/1649 4507" → 1m=1.95 5m=2.29 15m=1.70 +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 51.2°C (hwmon2) +2026/10/20 07:33:49 info.go:13: [DEBUG] [system] GetInfo done in 57ms — mem=12498MB/25898MB (48.3%), rootDisk=12.1GB/68.4GB (18.6%), load=1.95/2.29/1.70, temp=51.2°C (hwmon2), cpu=19.4% +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job docker-socket-watch: execution starting +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:33:49 scheduler.go:450: [INFO] [scheduler] Running job: stack-scan +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job stack-scan: execution starting +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job after-unclean-boot: execution starting +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/10/20 07:33:49 scheduler.go:85: [DEBUG] [scheduler] job ring-spill: execution starting +2026/10/20 07:33:50 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:33:50 manager.go:920: [DEBUG] [stacks] restart-policy of down member "rallly" = "unless-stopped" +2026/10/20 07:33:50 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:50 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "dawarich" deployed=false composePath=/opt/docker/stacks/dawarich/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "grimmory" deployed=false composePath=/opt/docker/stacks/grimmory/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "grocy" deployed=false composePath=/opt/docker/stacks/grocy/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=true composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "karakeep" deployed=false composePath=/opt/docker/stacks/karakeep/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "lubelogger" deployed=false composePath=/opt/docker/stacks/lubelogger/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "metube" deployed=false composePath=/opt/docker/stacks/metube/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=false composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "nzbget" deployed=false composePath=/opt/docker/stacks/nzbget/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "prowlarr" deployed=false composePath=/opt/docker/stacks/prowlarr/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "qbittorrent" deployed=false composePath=/opt/docker/stacks/qbittorrent/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "radicale" deployed=false composePath=/opt/docker/stacks/radicale/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=true composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=false composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "sabnzbd" deployed=false composePath=/opt/docker/stacks/sabnzbd/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "transmission" deployed=false composePath=/opt/docker/stacks/transmission/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=false composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=false composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/10/20 07:33:50 manager.go:675: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/10/20 07:33:50 manager.go:1958: [DEBUG] [stacks] getCatalogTemplateSlugs: found 65 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/10/20 07:33:50 manager.go:756: [DEBUG] [stacks] ScanStacks: catalog has 65 template slugs for orphan detection +2026/10/20 07:33:50 manager.go:787: [INFO] [stacks] ScanStacks complete: 70 stacks found (4 deployed, 66 available) +2026/10/20 07:33:50 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:33:50 scheduler.go:467: [INFO] [scheduler] Job stack-scan completed (took 181ms) +2026/10/20 07:33:50 healthprobe.go:105: [DEBUG] [stacks] RunHealthProbes: rallly has a health check but no container to probe — candidates: [rallly(stopped) rallly-postgres(starting)] +2026/10/20 07:33:50 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (2 skipped not due, 1 skipped no container) +2026/10/20 07:33:52 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:52 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:53 manager.go:1717: [DEBUG] Command completed: docker compose up -d (took 6.2s) +2026/10/20 07:33:53 manager.go:1366: [INFO] [stacks] Stack rallly started successfully (took 6.2s) +2026/10/20 07:33:54 installed.go:409: [DEBUG] [stacks] installed-images rallly: unchanged (2 service(s)) — app.yaml not rewritten +2026/10/20 07:33:54 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:33:54 offbox_handlers.go:547: [ERROR] [web] off-box reconstitute rallly (async): a(z) rallly visszaállítása sikertelen. Nem változott semmi: az alkalmazás adatai visszakerültek a visszaállítás előtti állapotba, és az alkalmazás újra fut +2026/10/20 07:33:55 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:33:55 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:33:57 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:33:57 manager.go:1695: [DEBUG] Running: docker compose ps -a --format table {{.Name}} {{.Image}} {{.State}} {{.Status}} (in /opt/docker/stacks/rallly) +2026/10/20 07:33:57 manager.go:1717: [DEBUG] Command completed: docker compose ps -a --format table {{.Name}} {{.Image}} {{.State}} {{.Status}} (took 0.1s) +2026/10/20 07:33:57 manager.go:1811: [INFO] [stacks] Stack rallly post-start status: +2026/10/20 07:33:57 manager.go:1814: [INFO] [stacks] rallly lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 running Up 3 seconds (health: starting) +2026/10/20 07:33:57 manager.go:1814: [INFO] [stacks] rallly-postgres postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 running Up 9 seconds (healthy) +2026/10/20 07:33:59 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:33:59 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:33:59 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:59 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/10/20 07:33:59 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (2 skipped not due, 0 skipped no container) +2026/10/20 07:34:00 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:34:04 scheduler.go:85: [DEBUG] [scheduler] job docker-socket-watch: execution starting diff --git a/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window.txt b/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window.txt new file mode 100644 index 00000000..2d53cd35 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/11-controller-log-window.txt @@ -0,0 +1,29 @@ +2026/10/20 07:33:28 auth.go:144: [DEBUG] [web] auth: valid session for POST /backup/offbox/reconstitute +2026/10/20 07:33:28 server.go:637: [DEBUG] [web] ServeHTTP: POST /backup/offbox/reconstitute from 172.18.0.3:34320 +2026/10/20 07:33:28 offbox_reconstitute.go:1253: [WARN] [offbox] rallly: the snapshot dump rallly-postgres.sql has no account rows — it may predate the customer's data +2026/10/20 07:33:29 restore_dbcreds.go:114: [INFO] [backup] Restore rallly: database password(s) [DB_PASSWORD] come from the backup, [] stay as installed (older than the data) — values not logged (R-1106) +2026/10/20 07:33:29 offbox_reconstitute.go:262: [INFO] [offbox] rallly: pre-restore safety dump written → pre-restore-20261020T073329Z-rallly-postgres.sql (109.4 KB) +2026/10/20 07:33:29 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:33:40 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 10.8s) +2026/10/20 07:33:41 offbox_restore_undo.go:154: [INFO] [offbox] rallly: the live volumes are copied before the restore (1 copied, 0 new from the snapshot) in 867ms — /mnt/sys_drive/felhom-data/backups/restore-undo/rallly/20261020T073340Z (R-893) +2026/10/20 07:33:41 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:33:42 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:33:42 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:33:42 manager.go:1404: [INFO] [stacks] Starting stack rallly services only: [rallly-postgres] +2026/10/20 07:33:43 manager.go:1415: [INFO] [stacks] Stack rallly services [rallly-postgres] started (took 0.4s) +2026/10/20 07:33:43 restore_db.go:87: [INFO] [backup] Restore rallly: replaying DB dump into rallly-postgres (postgres) +2026/10/20 07:33:43 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:33:44 dbdump.go:814: [ERROR] [backup] postgres import into rallly-postgres FAILED: exit status 3 — engine output follows: +2026/10/20 07:33:44 offbox_reconstitute.go:1100: [ERROR] [offbox] rallly: database replay failed, rolling back to the pre-restore state: importing postgres dump for rallly: postgres import into rallly-postgres failed: exit status 3 +2026/10/20 07:33:44 offbox_reconstitute.go:603: [DEBUG] [offbox] rallly: rallly-postgres was re-created during the restore (d0758337136e → 306e430406e4) — rolling back into the live container +2026/10/20 07:33:44 offbox_reconstitute.go:606: [INFO] [offbox] rallly: rolling back to the pre-restore state from pre-restore-20261020T073329Z-rallly-postgres.sql +2026/10/20 07:33:44 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073329Z-rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:33:45 offbox_reconstitute.go:611: [INFO] [offbox] rallly: rollback complete — 1 database(s) returned to the pre-restore state +2026/10/20 07:33:45 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:33:46 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 0.9s) +2026/10/20 07:33:46 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:33:47 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:33:47 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:33:47 offbox_restore_undo.go:232: [WARN] [offbox] rallly: the restore failed (importing postgres dump for rallly: postgres import into rallly-postgres failed: exit status 3) — the live volumes and definition are put back; starting the app (R-893) +2026/10/20 07:33:47 manager.go:1351: [INFO] [stacks] Starting stack: rallly +2026/10/20 07:33:54 offbox_handlers.go:547: [ERROR] [web] off-box reconstitute rallly (async): a(z) rallly visszaállítása sikertelen. Nem változott semmi: az alkalmazás adatai visszakerültek a visszaállítás előtti állapotba, és az alkalmazás újra fut diff --git a/documentation/audits/night2-2026-10-20/P2/live/12-after-failed-restore.txt b/documentation/audits/night2-2026-10-20/P2/live/12-after-failed-restore.txt new file mode 100644 index 00000000..32c0cb06 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/12-after-failed-restore.txt @@ -0,0 +1,24 @@ +=== AFTER the failed restore + put-back 07:34:25 UTC +rallly lukevella/rallly:4.15.3 Up 31 seconds (healthy) +rallly-postgres postgres:18-alpine Up 37 seconds (healthy) +markers (phase=rows): A-before-snapshot-1=1, B-after-snapshot-1=1, C-in-snapshot-2=1, D-after-snapshot-2=1 +r893_after exists: 1 rows: 1 +r893_only_old exists: 0 +r893_new2 exists: 1 rows: 1 +public tables: 36 +prisma migrations: 153 +db volume created: 2026-10-20T07:33:46Z +schema-only md5: 05828c7714605cb5f2a6b3b691b52a22 (control, second dump: 05828c7714605cb5f2a6b3b691b52a22) +data-only md5: e5c1a72ffb7107fef1d06e918b78194b +per-table row counts md5: d8bcbc4ba364fe3fdee69a2e773cf5be +per-table row counts: _prisma_migrations=153 accounts=0 calendar_connections=0 comments=0 credentials=0 event_types=0 instance_licenses=0 instance_settings=1 license_validations=0 licenses=0 options=0 participants=0 payment_methods=0 poll_activities=0 poll_invites=0 polls=0 provider_calendars=0 r893_after=1 r893_marker=4 r893_new2=1 registered_instances=0 rescheduled_event_dates=0 scheduled_event_invites=0 scheduled_events=0 sessions=0 sheet_slots=0 sheets=0 space_api_keys=0 space_member_invites=0 space_members=0 spaces=0 subscriptions=0 user_notification_preferences=0 users=0 verifications=0 votes=0 +definition: image: lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +restore-undo folder: ls: cannot access '/mnt/sys_drive/felhom-data/backups/restore-undo/rallly': No such file or directory +restore-undo root: total 8 drwx------ 2 root root 4096 Oct 20 07:33 . drwxr-xr-x 5 root root 4096 Oct 20 07:33 .. +live unit db-dumps: pre-restore-20261020T073329Z-rallly-postgres.sql rallly-postgres.sql +restore holds: ['chaoscrash', 'chaosoom', 'chaosoomb', 'gokapi'] | rallly hold: None +=== product view 07:34:31 +200 {"ok": true, "data": {"name": "rallly", "meta": {"display_name": "Rallly", "description": "Időpont szavazás (Doodle alternatíva)", "category": "productivity", "subdomain": "poll", "slug": "rallly", "catalog_since": "2026-09-30", "resources": {"mem_request": "384M", "mem_limit": "1024M", "pi_compatible": true, "needs_hdd": false, "hungarian_ui": false}, "deploy_fields": [{"env_var": "DOMAIN", "label": "Domain", "type": "domain", "generate": "", "default": "", "required": true, "placeholder": "", "description": "A szerver domain neve", "locked_after_deploy": true}, {"env_var": "SUBDOMAIN", "labe +stack: {'state': 'running', 'deployed': True, 'status': None, 'held': None, 'hold': None, 'restore_hold': None, 'health': None} +09:34:40 gate: poll is gated — passed as the household (cookie set) +front door poll.enkisfelhom.hu/ -> 0 200 diff --git a/documentation/audits/night2-2026-10-20/P2/live/13-normal-prepare.txt b/documentation/audits/night2-2026-10-20/P2/live/13-normal-prepare.txt new file mode 100644 index 00000000..ee2d50f4 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/13-normal-prepare.txt @@ -0,0 +1,5 @@ +07:36:09 POST /backup/offbox/restore -> 302 Location: /backups/restore/app?name=rallly&full_prep=rallly&full_size=66.1 MB +07:36:09 POST /backup/offbox/restore -> 302 Location: /backups/restore/app?name=rallly&flash=flash.offbox.restore_started +07:36:13 200 {"running": false, "op": "offbox-restore", "stack": "rallly", "started_at": "2026-10-20T07:36:09.798980152Z", "last": {"op": "offbox-restore", "stack": "rallly", "ok": true, "message": "A(z) rallly teljes mentése visszaállítva ellenőrző mappába: /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly — a saját fájljaiddal együtt. A meglévő adatok változatlanok.", "finished_at": "2026-10-20T07:36:12.377386791Z"}, "last_recent": true} +=== re-prepared scratch 07:36:15: injected line count=0 (must be 0); completion markers=1; r893_only_old mentions=5; r893_new2 mentions=0; lines=2704 +rallly-postgres.sql diff --git a/documentation/audits/night2-2026-10-20/P2/live/14-undo-folder-watch-normal.txt b/documentation/audits/night2-2026-10-20/P2/live/14-undo-folder-watch-normal.txt new file mode 100644 index 00000000..50e8ca01 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/14-undo-folder-watch-normal.txt @@ -0,0 +1,5 @@ +07:36:41.401 restore-undo: rallly 4096;rallly/20261020T073641Z 4096; +07:36:41.709 restore-undo: rallly 4096;rallly/20261020T073641Z 4096;rallly/20261020T073641Z/rallly_rallly_postgres_data.tar.tmp 69198848; +07:36:42.322 restore-undo: rallly 4096;rallly/20261020T073641Z 4096;rallly/20261020T073641Z/rallly_rallly_postgres_data.tar 69198848; +07:36:57.965 restore-undo: +watch end 07:38:27 diff --git a/documentation/audits/night2-2026-10-20/P2/live/15-normal-restore-http.txt b/documentation/audits/night2-2026-10-20/P2/live/15-normal-restore-http.txt new file mode 100644 index 00000000..98951525 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/15-normal-restore-http.txt @@ -0,0 +1,3 @@ +=== reconstitute (normal) T1=2026-10-20T07:36:28Z +07:36:28 POST /backup/offbox/reconstitute -> 302 Location: /backups/restore/app?name=rallly&flash=flash.offbox.full_restore_started +07:36:59 200 {"running": false, "op": "offbox-reconstitute", "stack": "rallly", "started_at": "2026-10-20T07:36:28.909561139Z", "last": {"op": "offbox-reconstitute", "stack": "rallly", "ok": true, "message": "A(z) rallly: 0 fájl és 1 adatkötet és az adatbázis visszaállítva (mentés: 2026. 10. 20. 09:30) — az alkalmazás újraindult.", "finished_at": "2026-10-20T07:36:57.852528534Z"}, "last_recent": true} diff --git a/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal-full.txt b/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal-full.txt new file mode 100644 index 00000000..201e5e7e --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal-full.txt @@ -0,0 +1,191 @@ +2026/10/20 07:36:28 auth.go:160: [DEBUG] [web] login attempt: visitor 192.168.0.180 (peer 172.18.0.3:34320, X-Forwarded-For "192.168.0.180", CF-Connecting-IP "") +2026/10/20 07:36:28 auth.go:192: [DEBUG] [web] login successful from 192.168.0.180, creating session +2026/10/20 07:36:28 session_store.go:140: [DEBUG] [web] sessions: saved 152 session(s) to dashboard-sessions.json +2026/10/20 07:36:28 auth.go:316: [DEBUG] [web] session created, expires=2026-10-27T07:36:28Z, active_sessions=152 +2026/10/20 07:36:28 auth.go:212: [INFO] [web] Login from 192.168.0.180 +2026/10/20 07:36:28 auth.go:144: [DEBUG] [web] auth: valid session for GET / +2026/10/20 07:36:28 server.go:637: [DEBUG] [web] ServeHTTP: GET / from 172.18.0.3:34320 +2026/10/20 07:36:28 auth.go:144: [DEBUG] [web] auth: valid session for GET /launcher +2026/10/20 07:36:28 server.go:637: [DEBUG] [web] ServeHTTP: GET /launcher from 172.18.0.3:34320 +2026/10/20 07:36:28 auth.go:144: [DEBUG] [web] auth: valid session for POST /backup/offbox/reconstitute +2026/10/20 07:36:28 server.go:637: [DEBUG] [web] ServeHTTP: POST /backup/offbox/reconstitute from 172.18.0.3:34320 +2026/10/20 07:36:29 auth.go:160: [DEBUG] [web] login attempt: visitor 192.168.0.180 (peer 172.18.0.3:34320, X-Forwarded-For "192.168.0.180", CF-Connecting-IP "") +2026/10/20 07:36:29 auth.go:192: [DEBUG] [web] login successful from 192.168.0.180, creating session +2026/10/20 07:36:29 session_store.go:140: [DEBUG] [web] sessions: saved 153 session(s) to dashboard-sessions.json +2026/10/20 07:36:29 auth.go:316: [DEBUG] [web] session created, expires=2026-10-27T07:36:29Z, active_sessions=153 +2026/10/20 07:36:29 auth.go:212: [INFO] [web] Login from 192.168.0.180 +2026/10/20 07:36:29 auth.go:144: [DEBUG] [web] auth: valid session for GET / +2026/10/20 07:36:29 server.go:637: [DEBUG] [web] ServeHTTP: GET / from 172.18.0.3:34320 +2026/10/20 07:36:29 auth.go:144: [DEBUG] [web] auth: valid session for GET /launcher +2026/10/20 07:36:29 server.go:637: [DEBUG] [web] ServeHTTP: GET /launcher from 172.18.0.3:34320 +2026/10/20 07:36:29 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:29 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:29 [WARN] [backup] ValidateDump: /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql is structurally valid (36 tables) but its accounts table has NO rows — the dump may predate the customer's data +2026/10/20 07:36:29 offbox_reconstitute.go:1253: [WARN] [offbox] rallly: the snapshot dump rallly-postgres.sql has no account rows — it may predate the customer's data +2026/10/20 07:36:29 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:36:29 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:36:30 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:30 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:30 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping rallly — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:30 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:30 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (3 skipped not due, 0 skipped no container) +2026/10/20 07:36:30 restore_dbcreds.go:114: [INFO] [backup] Restore rallly: database password(s) [DB_PASSWORD] come from the backup, [] stay as installed (older than the data) — values not logged (R-1106) +2026/10/20 07:36:30 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/10/20 07:36:30 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 0be48f112fd3 rallly rallly lukevella/rallly:4.15.3 +28f60240bf89 rallly-postgres rallly postgres:18-alpine +2d8767d09c82 paperless-webserver paperless-ngx ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +5ac453c47c19 paperless-postgres paperless-ngx postgres:18-alpine +ac5fbd6a551f paperless-redis paperless-ngx redis:7-alpine +a5743e7ad6c9 immich-server immich ghcr.io/immich-app/immich-server:v3.2.4 +66a2711b7a0d immich-postgres immich ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 +4e0f1ce... +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container rallly (image=lukevella/rallly:4.15.3, not a database) +2026/10/20 07:36:30 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: rallly-postgres (id=28f60240bf89) +2026/10/20 07:36:30 dbdump.go:198: [DEBUG] DiscoverDatabases: rallly-postgres → stack=rallly, dbUser=rallly, dbName=rallly +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/10/20 07:36:30 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5ac453c47c19) +2026/10/20 07:36:30 dbdump.go:915: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/10/20 07:36:30 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-server (image=ghcr.io/immich-app/immich-server:v3.2.4, not a database) +2026/10/20 07:36:30 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: immich-postgres (id=66a2711b7a0d) +2026/10/20 07:36:30 dbdump.go:198: [DEBUG] DiscoverDatabases: immich-postgres → stack=immich, dbUser=immich, dbName=immich +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-machine-learning (image=ghcr.io/immich-app/immich-machine-learning:v3.2.4, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-redis (image=redis:7-alpine, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container r893b-sftp (image=atmoz/sftp:alpine, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.5.6-stable, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1, not a database) +2026/10/20 07:36:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.7.13, not a database) +2026/10/20 07:36:30 dbdump.go:205: [DEBUG] DiscoverDatabases: found 3 database(s), skipped 10 non-DB container(s) +2026/10/20 07:36:30 dbdump.go:208: [INFO] [backup] Discovered 3 databases +2026/10/20 07:36:30 dbdump.go:277: [DEBUG] DumpOne: starting dump for container=rallly-postgres, stack=rallly, dbType=postgres, dest=/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073630Z-rallly-postgres.sql +2026/10/20 07:36:30 dbdump.go:314: [DEBUG] DumpOne: pg_dump command: docker exec 28f60240bf89 pg_dump -U rallly -d rallly --clean --if-exists --no-owner --no-privileges +2026/10/20 07:36:30 [WARN] [backup] ValidateDump: /mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/pre-restore-20261020T073630Z-rallly-postgres.sql is structurally valid (36 tables) but its accounts table has NO rows — the dump may predate the customer's data +2026/10/20 07:36:30 dbdump.go:424: [DEBUG] DumpOne: completed rallly-postgres → pre-restore-20261020T073630Z-rallly-postgres.sql (size=109.4 KB, valid=true, tables=36, duration=294ms) +2026/10/20 07:36:30 dbdump.go:429: [INFO] [backup] DB dump: rallly-postgres → pre-restore-20261020T073630Z-rallly-postgres.sql (109.4 KB, 294ms, 36 tables) +2026/10/20 07:36:30 offbox_reconstitute.go:262: [INFO] [offbox] rallly: pre-restore safety dump written → pre-restore-20261020T073630Z-rallly-postgres.sql (109.4 KB) +2026/10/20 07:36:30 manager.go:1437: [DEBUG] [stacks] StopStack rallly: current state=running deployed=true containers=2 +2026/10/20 07:36:30 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:36:30 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:36:30 manager.go:1695: [DEBUG] Running: docker compose down (in /opt/docker/stacks/rallly) +2026/10/20 07:36:32 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:32 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:34 scheduler.go:85: [DEBUG] [scheduler] job docker-socket-watch: execution starting +2026/10/20 07:36:35 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:35 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:38 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:38 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:39 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:36:39 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:36:40 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:40 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:40 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping rallly — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:40 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (2 skipped not due, 0 skipped no container) +2026/10/20 07:36:40 healthprobe.go:187: [DEBUG] Health probe immich: API GET :2283/api/server/ping → 200 (2ms) +2026/10/20 07:36:40 healthprobe.go:167: [INFO] Health probes: 1 ok (of 1 probed) +2026/10/20 07:36:41 manager.go:1717: [DEBUG] Command completed: docker compose down (took 10.7s) +2026/10/20 07:36:41 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 10.7s) +2026/10/20 07:36:41 manager.go:838: [INFO] [stacks] Status refresh: 9 containers across 70 stacks +2026/10/20 07:36:41 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:41 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:42 offbox_restore_undo.go:154: [INFO] [offbox] rallly: the live volumes are copied before the restore (1 copied, 0 new from the snapshot) in 875ms — /mnt/sys_drive/felhom-data/backups/restore-undo/rallly/20261020T073641Z (R-893) +2026/10/20 07:36:42 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:36:43 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:36:43 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:36:43 manager.go:1404: [INFO] [stacks] Starting stack rallly services only: [rallly-postgres] +2026/10/20 07:36:43 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:36:43 manager.go:1695: [DEBUG] Running: docker compose up -d rallly-postgres (in /opt/docker/stacks/rallly) +2026/10/20 07:36:43 manager.go:1717: [DEBUG] Command completed: docker compose up -d rallly-postgres (took 0.4s) +2026/10/20 07:36:43 manager.go:1415: [INFO] [stacks] Stack rallly services [rallly-postgres] started (took 0.4s) +2026/10/20 07:36:43 manager.go:838: [INFO] [stacks] Status refresh: 10 containers across 70 stacks +2026/10/20 07:36:43 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/10/20 07:36:43 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: a449dce25682 rallly-postgres rallly postgres:18-alpine +2d8767d09c82 paperless-webserver paperless-ngx ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +5ac453c47c19 paperless-postgres paperless-ngx postgres:18-alpine +ac5fbd6a551f paperless-redis paperless-ngx redis:7-alpine +a5743e7ad6c9 immich-server immich ghcr.io/immich-app/immich-server:v3.2.4 +66a2711b7a0d immich-postgres immich ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 +4e0f1ce12bf2 immich-machine-learning immich ghcr.io/immich... +2026/10/20 07:36:43 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: rallly-postgres (id=a449dce25682) +2026/10/20 07:36:44 dbdump.go:198: [DEBUG] DiscoverDatabases: rallly-postgres → stack=rallly, dbUser=rallly, dbName=rallly +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/10/20 07:36:44 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5ac453c47c19) +2026/10/20 07:36:44 dbdump.go:915: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/10/20 07:36:44 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-server (image=ghcr.io/immich-app/immich-server:v3.2.4, not a database) +2026/10/20 07:36:44 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: immich-postgres (id=66a2711b7a0d) +2026/10/20 07:36:44 dbdump.go:198: [DEBUG] DiscoverDatabases: immich-postgres → stack=immich, dbUser=immich, dbName=immich +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-machine-learning (image=ghcr.io/immich-app/immich-machine-learning:v3.2.4, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container immich-redis (image=redis:7-alpine, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container r893b-sftp (image=atmoz/sftp:alpine, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.5.6-stable, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.336.0-rc.1, not a database) +2026/10/20 07:36:44 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.7.13, not a database) +2026/10/20 07:36:44 dbdump.go:205: [DEBUG] DiscoverDatabases: found 3 database(s), skipped 9 non-DB container(s) +2026/10/20 07:36:44 dbdump.go:208: [INFO] [backup] Discovered 3 databases +2026/10/20 07:36:44 restore_db.go:87: [INFO] [backup] Restore rallly: replaying DB dump into rallly-postgres (postgres) +2026/10/20 07:36:44 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:36:44 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:44 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:45 dbdump.go:820: [INFO] [backup] Imported DB dump rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:36:45 restore_db.go:97: [INFO] [backup] Restore rallly: replayed 1 DB dump(s) +2026/10/20 07:36:45 manager.go:1348: [DEBUG] [stacks] StartStack rallly: current state=starting deployed=true +2026/10/20 07:36:45 manager.go:1351: [INFO] [stacks] Starting stack: rallly +2026/10/20 07:36:45 manager.go:1358: [DEBUG] [stacks] StartStack rallly: prepared 10 env vars for compose +2026/10/20 07:36:45 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:36:45 manager.go:1695: [DEBUG] Running: docker compose up -d (in /opt/docker/stacks/rallly) +2026/10/20 07:36:47 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:47 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:49 manager.go:1717: [DEBUG] Command completed: docker compose up -d (took 4.4s) +2026/10/20 07:36:49 manager.go:1366: [INFO] [stacks] Stack rallly started successfully (took 4.4s) +2026/10/20 07:36:49 installed.go:409: [DEBUG] [stacks] installed-images rallly: unchanged (2 service(s)) — app.yaml not rewritten +2026/10/20 07:36:49 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:49 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job ring-spill: execution starting +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job docker-socket-watch: execution starting +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:36:49 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 10s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:49 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/10/20 07:36:49 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (2 skipped not due, 0 skipped no container) +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/10/20 07:36:49 scheduler.go:85: [DEBUG] [scheduler] job after-unclean-boot: execution starting +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714348KB) → used≈12410MB avail≈13488MB +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714348 availKB=15996140 → total=25898MB avail=13488MB used=12410MB (47.9%) +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=12.1GB avail=52.7GB (18.7%) +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=193.7GB avail=696.4GB (21.8%) +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.48 1.89 1.65 6/1675 5879" → 1m=1.48 5m=1.89 15m=1.65 +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 51.1°C (hwmon2) +2026/10/20 07:36:50 info.go:13: [DEBUG] [system] GetInfo done in 77ms — mem=12410MB/25898MB (47.9%), rootDisk=12.1GB/68.4GB (18.7%), load=1.48/1.89/1.65, temp=51.1°C (hwmon2), cpu=16.9% +2026/10/20 07:36:50 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:50 main.go:2287: [INFO] [deadapp] check alive: 20 scans since boot, 4 deployed app(s) evaluated, 0 currently down +2026/10/20 07:36:50 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:50 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:52 manager.go:1686: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DB_PASSWORD, DOMAIN, SECRET_PASSWORD, SUBDOMAIN, IMPORT_PATH] (10 app + 0 system) +2026/10/20 07:36:52 manager.go:1695: [DEBUG] Running: docker compose ps -a --format table {{.Name}} {{.Image}} {{.State}} {{.Status}} (in /opt/docker/stacks/rallly) +2026/10/20 07:36:52 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:52 restore.go:298: [DEBUG] [backup] Post-restore health check: rallly not yet running, waiting... +2026/10/20 07:36:52 manager.go:1717: [DEBUG] Command completed: docker compose ps -a --format table {{.Name}} {{.Image}} {{.State}} {{.Status}} (took 0.1s) +2026/10/20 07:36:52 manager.go:1811: [INFO] [stacks] Stack rallly post-start status: +2026/10/20 07:36:52 manager.go:1814: [INFO] [stacks] rallly lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 running Up 3 seconds (health: starting) +2026/10/20 07:36:52 manager.go:1814: [INFO] [stacks] rallly-postgres postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 running Up 9 seconds (healthy) +2026/10/20 07:36:53 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:53 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:56 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:56 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:57 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:36:57 restore.go:293: [DEBUG] [backup] Post-restore health check: rallly is running +2026/10/20 07:36:57 offbox_reconstitute.go:1158: [INFO] [offbox] reconstituted rallly from snapshot c3d94d03: 0 file(s) placed, 1 volume(s) replayed, 1 DB dump(s) replayed, safety dump=pre-restore-20261020T073630Z-rallly-postgres.sql, skewed=false +2026/10/20 07:36:57 offbox_handlers.go:551: [INFO] [web] off-box reconstitute rallly completed (async): files=0 dbs=1 snapshot=c3d94d03 +2026/10/20 07:36:59 auth.go:144: [DEBUG] [web] auth: valid session for GET /api/backup/restore-status +2026/10/20 07:36:59 router.go:78: [DEBUG] [api] GET /api/backup/restore-status (path=/backup/restore-status) +2026/10/20 07:36:59 scheduler.go:85: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/20 07:36:59 scheduler.go:85: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/20 07:37:00 manager.go:838: [INFO] [stacks] Status refresh: 11 containers across 70 stacks +2026/10/20 07:37:00 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping immich — last check 20s ago, effective interval 5m0s, healthy=true +2026/10/20 07:37:00 healthprobe.go:64: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/10/20 07:37:00 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (2 skipped not due, 0 skipped no container) +2026/10/20 07:37:00 healthprobe.go:187: [DEBUG] Health probe rallly: HTTP GET :3000/ → 200 (54ms) +2026/10/20 07:37:00 healthprobe.go:167: [INFO] Health probes: 1 ok (of 1 probed) diff --git a/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal.txt b/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal.txt new file mode 100644 index 00000000..bbaa1d11 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/16-controller-log-normal.txt @@ -0,0 +1,23 @@ +2026/10/20 07:36:28 auth.go:144: [DEBUG] [web] auth: valid session for POST /backup/offbox/reconstitute +2026/10/20 07:36:28 server.go:637: [DEBUG] [web] ServeHTTP: POST /backup/offbox/reconstitute from 172.18.0.3:34320 +2026/10/20 07:36:29 offbox_reconstitute.go:1253: [WARN] [offbox] rallly: the snapshot dump rallly-postgres.sql has no account rows — it may predate the customer's data +2026/10/20 07:36:30 restore_dbcreds.go:114: [INFO] [backup] Restore rallly: database password(s) [DB_PASSWORD] come from the backup, [] stay as installed (older than the data) — values not logged (R-1106) +2026/10/20 07:36:30 offbox_reconstitute.go:262: [INFO] [offbox] rallly: pre-restore safety dump written → pre-restore-20261020T073630Z-rallly-postgres.sql (109.4 KB) +2026/10/20 07:36:30 manager.go:1440: [INFO] [stacks] Stopping stack: rallly +2026/10/20 07:36:41 manager.go:1449: [INFO] [stacks] Stack rallly stopped successfully (took 10.7s) +2026/10/20 07:36:42 offbox_restore_undo.go:154: [INFO] [offbox] rallly: the live volumes are copied before the restore (1 copied, 0 new from the snapshot) in 875ms — /mnt/sys_drive/felhom-data/backups/restore-undo/rallly/20261020T073641Z (R-893) +2026/10/20 07:36:42 restore.go:215: [INFO] [backup] Restoring Docker volume rallly_rallly_postgres_data for rallly +2026/10/20 07:36:43 restore.go:253: [DEBUG] [backup] Volume rallly_rallly_postgres_data restored successfully +2026/10/20 07:36:43 restore.go:258: [INFO] [backup] Restored 1 Docker volume(s) for rallly +2026/10/20 07:36:43 manager.go:1404: [INFO] [stacks] Starting stack rallly services only: [rallly-postgres] +2026/10/20 07:36:43 manager.go:1415: [INFO] [stacks] Stack rallly services [rallly-postgres] started (took 0.4s) +2026/10/20 07:36:44 restore_db.go:87: [INFO] [backup] Restore rallly: replaying DB dump into rallly-postgres (postgres) +2026/10/20 07:36:44 dbdump.go:800: [DEBUG] [backup] ImportDump: importing /mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly/mnt/sys_drive/felhom-data/backups/primary/rallly/db-dumps/rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:36:45 dbdump.go:820: [INFO] [backup] Imported DB dump rallly-postgres.sql into rallly-postgres (postgres) +2026/10/20 07:36:45 restore_db.go:97: [INFO] [backup] Restore rallly: replayed 1 DB dump(s) +2026/10/20 07:36:45 manager.go:1351: [INFO] [stacks] Starting stack: rallly +2026/10/20 07:36:49 manager.go:1366: [INFO] [stacks] Stack rallly started successfully (took 4.4s) +2026/10/20 07:36:52 restore.go:298: [DEBUG] [backup] Post-restore health check: rallly not yet running, waiting... +2026/10/20 07:36:57 restore.go:293: [DEBUG] [backup] Post-restore health check: rallly is running +2026/10/20 07:36:57 offbox_reconstitute.go:1158: [INFO] [offbox] reconstituted rallly from snapshot c3d94d03: 0 file(s) placed, 1 volume(s) replayed, 1 DB dump(s) replayed, safety dump=pre-restore-20261020T073630Z-rallly-postgres.sql, skewed=false +2026/10/20 07:36:57 offbox_handlers.go:551: [INFO] [web] off-box reconstitute rallly completed (async): files=0 dbs=1 snapshot=c3d94d03 diff --git a/documentation/audits/night2-2026-10-20/P2/live/17-after-normal-restore.txt b/documentation/audits/night2-2026-10-20/P2/live/17-after-normal-restore.txt new file mode 100644 index 00000000..21ee6da0 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/17-after-normal-restore.txt @@ -0,0 +1,20 @@ +=== AFTER the normal restore of snapshot 2 07:38:31 UTC +rallly lukevella/rallly:4.15.3 Up About a minute (healthy) +rallly-postgres postgres:18-alpine Up About a minute (healthy) +markers (phase=rows): A-before-snapshot-1=1, B-after-snapshot-1=1, C-in-snapshot-2=1 +r893_after exists: 1 rows: 1 +r893_only_old exists: 1 rows: 1 +r893_new2 exists: 0 +public tables: 36 +prisma migrations: 153 +db volume created: 2026-10-20T07:36:42Z +schema-only md5: f872bd54b5e09991876d465efb13d754 (control, second dump: f872bd54b5e09991876d465efb13d754) +data-only md5: 1e688d1bc96fba7e5140f391707a762a +per-table row counts md5: c032377e7ce421083e40fb875cfadb34 +per-table row counts: _prisma_migrations=153 accounts=0 calendar_connections=0 comments=0 credentials=0 event_types=0 instance_licenses=0 instance_settings=1 license_validations=0 licenses=0 options=0 participants=0 payment_methods=0 poll_activities=0 poll_invites=0 polls=0 provider_calendars=0 r893_after=1 r893_marker=3 r893_only_old=1 registered_instances=0 rescheduled_event_dates=0 scheduled_event_invites=0 scheduled_events=0 sessions=0 sheet_slots=0 sheets=0 space_api_keys=0 space_member_invites=0 space_members=0 spaces=0 subscriptions=0 user_notification_preferences=0 users=0 verifications=0 votes=0 +definition: image: lukevella/rallly:4.15.3@sha256:8cd979aefe8d06e1822bc67054eb2d31088cdb831aa3f7b5615b3768f35431d9 image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +restore-undo folder: ls: cannot access '/mnt/sys_drive/felhom-data/backups/restore-undo/rallly': No such file or directory +restore holds: ['chaoscrash', 'chaosoom', 'chaosoomb', 'gokapi'] +stack: {'state': 'running', 'deployed': True, 'status': None, 'held': None, 'hold': None, 'restore_hold': None, 'health': None} +09:38:37 gate: poll is gated — passed as the household (cookie set) +front door poll.enkisfelhom.hu/ -> 0 200 diff --git a/documentation/audits/night2-2026-10-20/P2/live/18-teardown.txt b/documentation/audits/night2-2026-10-20/P2/live/18-teardown.txt new file mode 100644 index 00000000..84f28796 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/18-teardown.txt @@ -0,0 +1,79 @@ +10:43:11 [X] stop -> 200 {'ok': True, 'message': 'Stack rallly stop completed'} +10:43:43 [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'rallly', 'volumes_removed': ['rallly_rallly_postgres_data'], 'hdd_paths_removed': [], 'hdd_paths_preserved': [], 'hdd_note': ' +10:43:52 [X] after remove: deployed=False leftovers='/opt/docker/stacks/rallly' +remove: 200 +=== machine teardown 08:44:07 +rallly stack dir: .felhom.yml docker-compose.yml hold-logs (template dir pre-existed with the same three entries) +rallly containers: 0; rallly volumes: 0 +ls: cannot access '/mnt/felhom-drives/scratch_hdd/backups/offsite-restore/rallly': No such file or directory +ls: cannot access '/mnt/sys_drive/felhom-data/backups/primary/rallly': No such file or directory +r893b-sftp +2 +restore-undo (empty, created by this drill) removed +cb6b18f87662cf43d7e100f37366ce639ee5593fbf505751f5c1d831f8053fd1 settings.json +total 20 +drwx------ 2 root root 4096 Sep 30 08:40 . +drwxr-xr-x 4 root root 4096 Oct 20 08:44 .. +-rw-r--r-- 1 root root 91 Sep 30 08:40 known_hosts +-rw------- 1 root root 64 Sep 30 08:40 repo_password +-rw------- 1 root root 411 Sep 30 08:40 ssh_key +restart rc=0 +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.331.0 Up 8 seconds (healthy) +2026/10/20 08:44:08 main.go:353: [INFO] felhom-controller 0.331.0 starting (customer: demo-hp, domain: enkisfelhom.hu) +[ERROR] lines after start: 0 +2 +0 +=== after teardown 08:45:17 +image file: gitea.dooplex.hu/admin/felhom-controller:0.331.0 +filebrowser gtstef/filebrowser:1.5.6-stable Up About a minute (healthy) +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.331.0 Up About a minute (healthy) +paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15 Up About an hour (healthy) +paperless-postgres postgres:18-alpine Up About an hour (healthy) +paperless-redis redis:7-alpine Up About an hour (healthy) +immich-server ghcr.io/immich-app/immich-server:v3.2.4 Up About an hour (healthy) +immich-postgres ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 Up About an hour (healthy) +immich-machine-learning ghcr.io/immich-app/immich-machine-learning:v3.2.4 Up About an hour (healthy) +immich-redis redis:7-alpine Up About an hour (healthy) +traefik traefik:v3.7.13 Up 4 days +--- images now +alpine:latest +curlimages/curl:8.11.1 +ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +ghcr.io/puppeteer/puppeteer:latest +gitea.dooplex.hu/admin/felhom-controller:0.314.0-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.326.0-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.326.0-rc.2 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.2 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.3 +gitea.dooplex.hu/admin/felhom-controller:0.327.0-rc.4 +gitea.dooplex.hu/admin/felhom-controller:0.327.1-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.331.0 +gitea.dooplex.hu/admin/felhom-controller:0.331.0-rc.1 +gitea.dooplex.hu/admin/felhom-samba:1.2.0 +gtstef/filebrowser:1.3.3-stable +gtstef/filebrowser:1.5.6-stable +postgres:18-alpine +redis:7-alpine +redis:7.4-alpine +traefik:v3.6.7 +traefik:v3.7.13 +offbox target set: False +per-app offbox: None +restore holds: ['chaoscrash', 'chaosoom', 'chaosoomb', 'gokapi'] +[ERROR] lines since start: 0 +/mnt/felhom-drives/scratch_hdd/backups/offsite-restore/: +total 8 +drwxr-xr-x 2 root root 4096 Oct 20 08:43 . +drwxr-xr-x 5 root root 4096 Oct 19 18:52 .. + +/mnt/sys_drive/felhom-data/backups/: +total 16 +drwxr-xr-x 4 root root 4096 Oct 20 08:44 . +drwxr-xr-x 4 root root 4096 Sep 13 20:26 .. +drwxr-xr-x 3 root root 4096 Oct 20 08:43 primary +drwxr-xr-x 3 root root 4096 Oct 12 20:51 secondary +IMAGE LIST IDENTICAL TO BASELINE +=== host layer (demo-hp) +0 +DooPlex scratch: throwaway key shredded +=== hub layer: nothing — 9202 has no hub diff --git a/documentation/audits/night2-2026-10-20/P2/live/README.md b/documentation/audits/night2-2026-10-20/P2/live/README.md new file mode 100644 index 00000000..5ca56302 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/README.md @@ -0,0 +1,161 @@ +# P2 live: R-893 option A, proved on scratch guest 9202 (2026-10-20, by day) + +Night brief 2026-10-20 part 2, Part 2 (operator answer „D20 A"). Controller **0.336.0-rc.1** (commit `93fb251`, +image id `sha256:0756d4d2…`) on Tier 0 only: demo-hp, scratch guest 9202. Code read live: +`felhom-controller/controller/internal/backup/offbox_restore_undo.go` and `offbox_reconstitute.go` at `93fb251`. +Architecture: `07-backup-architecture.md` §6.3 (R-893); `09` §3 decision 192 and D20 = A. +The method repeats `night-2026-10-19/P3/` (the measurement of the old behaviour) step for step, so the two sets of +numbers can be compared. + +## Method (endpoint level, the household's buttons; no browser) + +- 9202 was on **0.331.0**. For the run it went to 0.336.0-rc.1: pull, write `/etc/felhom-controller-image`, + restart `felhom-controller-bootstrap.service`. Healthy, 0 `[ERROR]` (`01-upgrade.txt`). It was put back after the run. +- **Scratch off-site target:** `atmoz/sftp:alpine` container `r893b-sftp` on 9202 itself (port 2222). The repo is on + 9202's own disk at `/var/lib/felhom/r893b-sftp` (nvme-scratch, not `pve/data`). The controller reached it at + `172.17.0.1:2222`, which does not depend on DHCP. Set up through `POST /backup/offbox/config` with a throwaway + ed25519 key and the server's own host key pinned. 9202 has no agent, so the escrow was confirmed with the legacy + manual `POST /backup/offbox/confirm-escrow`. Only `rallly` was switched on (`POST /backup/offbox/toggle`). +- **Throwaway app:** Rallly 4.15.3 + postgres:18-alpine. Its DB is in the named volume `rallly_rallly_postgres_data`. + Deployed through the product (`box_walk.deploy`). +- **Observables, all positive** (`tool-obs.sh.txt`): + - the `r893_marker` rows; + - tables that exist only at certain times; + - the public base-table count; + - an md5 of `pg_dump --schema-only`, with pg 18's random `\restrict` lines removed and **a second dump as its + control** (equal every time); + - an md5 of the per-table row counts; + - an md5 of `pg_dump --data-only`; + - the definition's image lines; + - the `backups/restore-undo/rallly` folder. +- **Forced error:** the same as last night. One line, `SELECT r893_forced_replay_failure();`, goes into the PREPARED + scratch copy of the snapshot's dump, just before its completion marker. The completeness gate still passes, and the + replay fails **after the volume was replaced**. +- **Restore:** `POST /backup/offbox/reconstitute` with `app=rallly confirm=1`. The outcome was read from + `/api/backup/restore-status`, the source the restore page polls. +- **Watch on the undo folder:** while each restore ran, a watcher on 9202 listed + `/mnt/sys_drive/felhom-data/backups/restore-undo` every 0.3 s. This shows the copy from a different channel than + the controller's log. + +## Timeline (UTC) + +| Time | Act | File | +|---|---|---| +| 07:25:33 | baseline: 0.331.0, no off-site target, `settings.json` and the `offbox/` folder copied | `00-baseline.txt` | +| 07:25:47 | 9202 on 0.336.0-rc.1, healthy | `01-upgrade.txt` | +| 07:27:37–07:27:58 | SFTP target up; off-site configured, escrow confirmed, `rallly` switched on | `02-sftp-target.txt` | +| 07:28:03–07:28:53 | Rallly deployed | `03-deploy.txt` | +| 07:29:13 | marker **A** (34 tables) | `04-marker-A.txt` | +| 07:29:29–07:30:38 | off-site run → snapshot 1 | `05-snapshot-1.txt` | +| 07:30:50 | marker **B** + `r893_after`; `r893_only_old` + marker **C** (36 tables) | `05-snapshot-1.txt` | +| 07:30:56–07:32:04 | off-site run → **snapshot 2** (`c3d94d03`) | `06-snapshot-2.txt` | +| 07:32:38 | `r893_only_old` DROPPED, marker **D**, `r893_new2`: the LIVE record | `07-live-before-restore.txt` | +| 07:32:57–07:33:13 | full restore of snapshot 2 prepared (66.1 MB; content checked: C present, D absent); the failing line injected | `08-prepare-and-inject.txt` | +| 07:33:28 | reconstitute (forced failure) | `10-forced-failure-http.txt`, `11-controller-log-window*.txt` | +| 07:33:29 | undo dump of the live DB; app stopped 07:33:29–07:33:40 | | +| 07:33:40.9–41.6 | **live volume copied** to `restore-undo/rallly/20261020T073340Z/rallly_rallly_postgres_data.tar` (69 198 848 B), tmp then rename | `09-undo-folder-watch.txt` | +| 07:33:42 | snapshot's volume in place (`Restored 1 Docker volume(s)`) | | +| 07:33:44 | replay FAILED (`function r893_forced_replay_failure() does not exist`, exit 3); rows rolled back 07:33:45 | | +| 07:33:46–47 | app stopped; **the copy poured back** (`Restored 1 Docker volume(s)`, second time); „the live volumes and definition are put back; starting the app (R-893)" | | +| 07:33:53 | `Stack rallly started successfully` | | +| 07:33:54 | restore op ends ok=false; the copy folder **gone** (watcher: absent at 07:33:54.45) | | +| 07:34:25 | after state read: identical to 07:32:38 | `12-after-failed-restore.txt` | +| 07:36:09–07:36:15 | full restore prepared again, with no injection (injected-line count 0) | `13-normal-prepare.txt` | +| 07:36:28–07:36:57 | normal reconstitute: copy made 07:36:41–42, volume + dump replayed, app restarted, copy removed 07:36:57.9 | `14-…`, `15-…`, `16-controller-log-normal*.txt` | +| 07:38:31 | after state read = snapshot 2 | `17-after-normal-restore.txt` | +| (stop) | the session was paused by an API rate limit. When it resumed, 9202 was re-read first: nothing was in progress, no restore-undo copy, step 4 already recorded | | +| 08:43–08:45 | teardown, three layers | `18-teardown.txt` | + +## Step 3: the forced failure, before and after + +| | LIVE before the restore (07:32:38) | After the failed restore + put-back (07:34:25) | 2026-10-19 on 0.335.0 (old behaviour) | +|---|---|---|---| +| marker rows | A, B, C, D | **A, B, C, D** | A, B, C, D | +| `r893_only_old` (in snapshot 2 only) | absent | **absent** | present, 1 row | +| `r893_new2` (written after snapshot 2) | present, 1 row | **present, 1 row** | present, 1 row | +| public base tables | 36 | **36** | 37 | +| schema-only md5 | `05828c77…` | **`05828c77…`** (control equal) | — | +| per-table row-count md5 | `d8bcbc4b…` | **`d8bcbc4b…`** | — | +| data-only md5 | `e5c1a72f…` | **`e5c1a72f…`** | — | +| definition | rallly 4.15.3, postgres 18 (digest pins) | same | same | +| app | running | **running, healthy**; controller `state=running`; front door 200 | stopped, held (`restore_mixed`) | +| restore hold for rallly | none | **none** (holds: chaoscrash, chaosoom, chaosoomb, gokapi, as at baseline) | `restore_mixed` | +| `backups/restore-undo/rallly` | absent | **absent** (it existed 07:33:40–07:33:54) | (did not exist) | + +The DB volume's `CreatedAt` moved from 07:28:21 to 07:33:46. That is expected: the put-back recreates the volume +from the copy. Every content observable is equal, the data-only dump included. + +## Step 4: the normal restore of snapshot 2 + +| | Recorded just before snapshot 2 (07:30:50) | After the normal restore (07:38:31) | +|---|---|---| +| marker rows | A, B, C | **A, B, C** | +| `r893_only_old` | present, 1 row | **present, 1 row** | +| `r893_new2` | absent | **absent** | +| public base tables | 36 | **36** | +| schema-only md5 | `f872bd54…` | **`f872bd54…`** | +| per-table row-count md5 | `c032377e…` | **`c032377e…`** | +| data-only md5 | `db5903ab…` | `1e688d1b…` (differs; see below) | +| app | running | **running, healthy**; front door 200; no hold | +| restore-undo folder | — | **absent** (existed 07:36:41–07:36:57) | + +- **Table count.** Snapshot 2 has **36** base tables, not 37. It held `r893_only_old` but not `r893_new2`. The + „37" in the brief is last night's MIXED state (36 live tables + the stray `r893_only_old`), not the snapshot's. +- **The data-only md5.** It differs from the read taken 6 s before the snapshot's own dump started, while every row + count and the schema are equal. Not explained here. It may be a row value that Rallly touched between that read + and the dump (the off-site run restarted the app), or row order after the reload. It was not part of the + acceptance list, and no row count moved. In step 3 the same observable was byte-equal, because there the volume + itself was put back. + +## The sentences the household sees + +- **Failed restore, put back** (`err.backup.restore_failed_put_back`, from `/api/backup/restore-status`): „A teljes + visszaállítás sikertelen: a(z) rallly visszaállítása sikertelen. Nem változott semmi: az alkalmazás adatai + visszakerültek a visszaállítás előtti állapotba, és az alkalmazás újra fut". Measured **true**: the data is + identical and the app runs. It does **not** contain „vedd fel velünk a kapcsolatot". +- **Normal restore:** „A(z) rallly: 0 fájl és 1 adatkötet és az adatbázis visszaállítva (mentés: 2026. 10. 20. + 09:30) — az alkalmazás újraindult." True. 09:30 is local time (CEST) for snapshot 2's dump at 07:30 UTC. +- Flash keys seen on the way: `flash.offbox.target_saved_escrow_agent_down` (no agent on 9202, expected), + `escrow_confirmed`, `run_started`, `restore_started`, `full_restore_started`. + +## Not measured (said plainly) + +- **The version-change branch** (the snapshot's older definition written, then the live one written back). It needs + an app at an older catalog step, the same limit as last night. Pinned by `r893_putback_test.go`. +- **The `_files` sentence** (`restore_failed_put_back_files`). Rallly has no drive files, so 0 files were placed. +- **The refusals before the stop:** the fit check refusing (`restore_undo_no_space`), an unmeasurable volume + (`restore_undo_unmeasured`) and a failed copy (`restore_undo_copy_failed`). Only the positive path of the fit + check ran: it passed and the app was stopped after it. +- **A put-back that itself fails** (hold + kept copy). +- **The operator notification** for a put-back. 9202 has no hub, so it was not observable. + +## Side observations + +- Each manual off-site run restarted Rallly (its dump leg), as last night. +- The controller logged one expected `[ERROR]` pair per failure (the import and the rollback line) and one + `[ERROR] [web] off-box reconstitute … (async)` line that carries the household sentence. The sentence is good + news, so ERROR is a loud level for a put-back that worked. This is noted for the parent, not changed. +- The empty folder `backups/restore-undo/` is left behind after the per-app folder is removed (the code removes the + per-app folder only). This drill removed it by name. + +## Teardown (three layers) + +- **Machine (9202):** + - Rallly was removed through the product (`POST /api/stacks/rallly/stop`, then `/remove` with `remove_hdd_data` + + `remove_backups`). Result: `volumes_removed: rallly_rallly_postgres_data`; 0 containers, 0 volumes; no + `backups/primary/rallly`, no `offsite-restore/rallly`. The template folder `/opt/docker/stacks/rallly` + (`.felhom.yml`, `docker-compose.yml`, `hold-logs`) was there before and stays. + - `r893b-sftp` was removed. So were the images `atmoz/sftp:alpine` and `lukevella/rallly:4.15.3` (both absent at + baseline), the repo folder, the key and the empty `backups/restore-undo/`. + - `settings.json` was put back from the 07:25 copy (sha256 `cb6b18f8…` = baseline) and `offbox/` from its tar, + with the controller stopped. + - The controller is back on **0.331.0**: healthy, 0 `[ERROR]`. Off-site target unset, no per-app off-site switch, + restore holds as at baseline. + - The 0.336.0-rc.1 image was removed by name. The image list is **identical to the baseline**. All + `/root/r893b-*` and `/tmp/r893b-*` files are gone. + - Not put back: `restore-status.json`, the controller's record of the last op. It now names this drill's last + restore, as it named last night's at baseline. +- **Host (demo-hp):** only the per-call `/tmp/r893b-g-*` script copies, each removed after its call; 0 left. Nothing + on `pve/data`. No whole-guest backup taken. +- **Hub:** nothing. 9202 has no hub. +- **DooPlex:** the throwaway key was shredded from the session scratchpad. diff --git a/documentation/audits/night2-2026-10-20/P2/live/tool-obs.sh.txt b/documentation/audits/night2-2026-10-20/P2/live/tool-obs.sh.txt new file mode 100644 index 00000000..d45fa504 --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/live/tool-obs.sh.txt @@ -0,0 +1,22 @@ +#!/bin/bash +# r893b-obs.sh LABEL — positive observables of the Rallly database (read-only) +P="docker exec rallly-postgres psql -U rallly -d rallly -Atq" +echo "=== $1 $(date -u +%T) UTC" +docker ps -a --filter label=com.docker.compose.project=rallly --format '{{.Names}} {{.Image}} {{.Status}}' +echo "markers (phase=rows): $($P -c "select string_agg(phase||'='||n, ', ' order by phase) from (select phase, count(*) n from r893_marker group by phase) x" 2>&1)" +for t in r893_after r893_only_old r893_new2; do + ex=$($P -c "select count(*) from information_schema.tables where table_schema='public' and table_name='$t'") + if [ "$ex" = 1 ]; then echo "$t exists: 1 rows: $($P -c "select count(*) from $t")"; else echo "$t exists: 0"; fi +done +echo "public tables: $($P -c "select count(*) from information_schema.tables where table_schema='public' and table_type='BASE TABLE'")" +echo "prisma migrations: $($P -c 'select count(*) from _prisma_migrations' 2>&1)" +echo "db volume created: $(docker volume inspect -f '{{.CreatedAt}}' rallly_rallly_postgres_data 2>&1)" +S1=$(docker exec rallly-postgres pg_dump -U rallly -d rallly --schema-only | grep -Ev '^\\(un)?restrict ' | md5sum | cut -c1-32) +S2=$(docker exec rallly-postgres pg_dump -U rallly -d rallly --schema-only | grep -Ev '^\\(un)?restrict ' | md5sum | cut -c1-32) +echo "schema-only md5: $S1 (control, second dump: $S2)" +echo "data-only md5: $(docker exec rallly-postgres pg_dump -U rallly -d rallly --data-only | grep -Ev '^\\(un)?restrict ' | md5sum | cut -c1-32)" +RC=$(for t in $($P -c "select table_name from information_schema.tables where table_schema='public' and table_type='BASE TABLE' order by table_name"); do echo "$t=$($P -c "select count(*) from \"$t\"")"; done) +echo "per-table row counts md5: $(echo "$RC" | md5sum | cut -c1-32)" +echo "per-table row counts: $(echo "$RC" | tr '\n' ' ')" +echo "definition: $(grep -E '^\s*image:' /opt/docker/stacks/rallly/docker-compose.yml | tr -s ' ' | tr '\n' ' ')" +echo "restore-undo folder: $(ls -A /mnt/sys_drive/felhom-data/backups/restore-undo/rallly 2>&1 | tr '\n' ' ')" diff --git a/documentation/audits/night2-2026-10-20/P5/r462-list.txt b/documentation/audits/night2-2026-10-20/P5/r462-list.txt new file mode 100644 index 00000000..47c9948e --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P5/r462-list.txt @@ -0,0 +1,36 @@ +# Part 5 — R-462: the update test for the installed apps (read 2026-10-20T08:45:32Z, catalog c50adfc) + +## installed apps on the test boxes (hub, latest controller report per customer) +bentopdf 2, opengist 2, privatebin 2, bookstack 2, paperless-ngx 2; 1 each: prowlarr radarr qbittorrent sonarr home-assistant romm docmost plex calibre-web adventurelog kimai + +## fixture coverage (upgrade_boxport.available() + upgrade_fixtures.FIXTURES) +box-ported fixtures: 58 ; bench-only fixtures: 0 +installed apps with a fixture: ['opengist', 'privatebin', 'bookstack', 'paperless-ngx', 'prowlarr', 'radarr', 'qbittorrent', 'sonarr', 'home-assistant', 'romm', 'docmost', 'plex', 'calibre-web', 'adventurelog', 'kimai'] +installed apps without one: ['bentopdf'] (bentopdf keeps no data: nothing to seed) + +## upstream currency (scripts/check-currency.py, registries read) +opengist opengist 1.15 same-shape: major 1.15, all 1.15 — SHAPE-SWITCH? 1.15.2 +privatebin privatebin 2.0.6 same-shape: major 2.1.0, all 2.1.0 — behind +bookstack bookstack 26.09.1 same-shape: major 26.09.1, all 26.09.1 — current +bookstack bookstack-db 12.3 same-shape: major 12.3, all 13.0 — behind +paperless-ngx paperless-webserver 2.20.15 same-shape: major 2.20.15, all 3.3.3 — behind +paperless-ngx paperless-postgres 18-alpine same-shape: major 18-alpine, all 18-alpine — SHAPE-SWITCH? 18.6 +paperless-ngx paperless-redis 7-alpine same-shape: major 7-alpine, all 8-alpine — behind +prowlarr prowlarr version-2.6.5.5623 same-shape: major version-2.6.5.5623, all version-2.6.5.5623 — current +radarr radarr 6.4.4 same-shape: major 6.4.4, all 6.4.4 — current +qbittorrent qbittorrent 5.2.4-1 same-shape: major 5.2.4-1, all 5.2.4-1 — current +sonarr sonarr 4.0.20 same-shape: major 4.0.20, all 4.0.20 — SHAPE-SWITCH? 5.14 +home-assistant home-assistant 2026.9.4 same-shape: major 2026.10.2, all 2026.10.2 — behind +romm romm 5.3.1 same-shape: major 5.4.0, all 5.4.0 — behind +romm romm-db 11.8 same-shape: major 11.8, all 13.0 — behind +romm romm-redis 7-alpine same-shape: major 7-alpine, all 8-alpine — behind +docmost docmost 0.96.0 same-shape: major 0.96.0, all 0.96.0 — current +docmost docmost-postgres 18-alpine same-shape: major 18-alpine, all 18-alpine — SHAPE-SWITCH? 18.6 +docmost docmost-redis 7-alpine same-shape: major 7-alpine, all 8-alpine — behind +plex plex 1.41.4.9463-630c9f557 same-shape: major 1.43.4.10903-e5521bd8c, all 1.43.4.10903-e5521bd8c — behind +calibre-web calibre-web v4.0.8 same-shape: major v4.0.8, all v4.0.8 — current +adventurelog adventurelog v0.13.0 same-shape: major v0.13.0, all v0.13.0 — current +adventurelog adventurelog-postgres 16-3.5-alpine same-shape: major 16-3.6-alpine, all 18-3.6-alpine — behind +adventurelog adventurelog-frontend v0.13.0 same-shape: major v0.13.0, all v0.13.0 — current +kimai kimai 2.67.0 same-shape: major 2.70.0, all 2.70.0 — behind +kimai kimai-db 11.8 same-shape: major 11.8, all 13.0 — behind