From a7971c4822cd1c2bba5229c62894bf4cbbf41656 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 24 Sep 2026 23:04:43 +0200 Subject: [PATCH] =?UTF-8?q?night=202026-09-25=20(in=20progress):=20part=20?= =?UTF-8?q?7=20shipped=20as=20controller=20v0.271.0=20=E2=80=94=20decision?= =?UTF-8?q?s=2031-33,=20evidence=20A/B/C/F,=20register=20(R-685..R-687;=20?= =?UTF-8?q?closed=20R-672=20R-673=20R-684=20R-680=20R-678)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../architecture/00-capability-map.md | 2 +- documentation/architecture/03-host-agent.md | 11 + .../architecture/07-backup-architecture.md | 13 + documentation/architecture/08-alarm-ladder.md | 6 + .../architecture/09-update-architecture.md | 37 +- .../night-2026-09-25/A/A1-demo-felhom.txt | 24 + .../audits/night-2026-09-25/A/A1-demo-hp.txt | 25 + .../night-2026-09-25/A/A2-demo-felhom.txt | 15 + .../audits/night-2026-09-25/A/A2-demo-hp.txt | 15 + .../A/A3-demo-felhom-restore-test.txt | 38 + .../night-2026-09-25/A/A3-demo-hp-refusal.txt | 26 + .../night-2026-09-25/A/A4-hp-backup-run.txt | 77 + .../night-2026-09-25/A/A4-hp-backup-space.txt | 40 + .../B/redproofs/D12-default-off.txt | 11 + .../D13-files-may-change-ignored.txt | 19 + .../B/redproofs/D13-needs-person-ignored.txt | 19 + .../B/redproofs/D14-two-steps-a-night.txt | 23 + .../B/redproofs/D20-gate-does-not-wait.txt | 11 + .../B/redproofs/D20-leg-ignores-W5h.txt | 19 + .../B/redproofs/D28-updating-app-sampled.txt | 19 + .../B/redproofs/R678-no-rescan-at-done.txt | 13 + .../B/redproofs/R680-no-failed-step-check.txt | 41 + .../night-2026-09-25/B/redproofs/SUMMARY.txt | 41 + .../B/redproofs/chain-leg-not-deferred.txt | 11 + .../redproofs/wiring-SetUpdateLeg-dropped.txt | 11 + .../C/C0-9202-window-before.txt | 1 + .../night-2026-09-25/C/C0-deploy-9202.txt | 8 + .../night-2026-09-25/C/C0-repoint-drill.txt | 10 + .../audits/night-2026-09-25/C/C1-setup.json | 119 ++ .../audits/night-2026-09-25/C/C1-setup.log | 22 + .../C/C1-state-before-night1.json | 124 ++ .../night-2026-09-25/C/data-readback.txt | 8 + .../night-2026-09-25/C/night1/app-lines.txt | 8 + .../C/night1/controller-full.log | 1235 +++++++++++++++++ .../C/night1/controller-night.log | 80 ++ .../C/night1/pages-after.json | 162 +++ .../C/night1/window-moved.txt | 12 + .../night-2026-09-25/C/night2/app-lines.txt | 8 + .../C/night2/controller-full.log | 702 ++++++++++ .../C/night2/controller-night.log | 51 + .../C/night2/pages-after.json | 171 +++ .../night-2026-09-25/C/night2/runner.log | 7 + .../C/night2/window-moved.txt | 12 + .../C/night3-kill/app-lines.txt | 8 + .../C/night3-kill/controller-after-kill.log | 22 + .../C/night3-kill/controller-before-kill.log | 484 +++++++ .../C/night3-kill/pages-after.json | 171 +++ .../night-2026-09-25/C/night3-kill/runner.log | 19 + .../C/night3-kill/window-moved.txt | 12 + .../C/night4-power/app-lines.txt | 8 + .../C/night4-power/controller-after-boot.log | 8 + .../C/night4-power/controller-before-cut.log | 372 +++++ .../C/night4-power/pages-after.json | 171 +++ .../C/night4-power/runner.log | 5 + .../C/night4-power/window-moved.txt | 12 + .../C/night5-off/app-lines.txt | 8 + .../C/night5-off/controller-full.log | 186 +++ .../C/night5-off/controller-night.log | 7 + .../C/night5-off/pages-after.json | 171 +++ .../night-2026-09-25/C/night5-off/runner.log | 7 + .../C/night5-off/window-moved.txt | 12 + .../night-2026-09-25/C/night5-switch-off.txt | 2 + .../C/night6-on/app-lines.txt | 8 + .../C/night6-on/controller-full.log | 495 +++++++ .../C/night6-on/controller-night.log | 28 + .../C/night6-on/pages-after.json | 164 +++ .../night-2026-09-25/C/night6-on/runner.log | 7 + .../C/night6-on/window-moved.txt | 12 + .../night-2026-09-25/C/night6-switch-on.txt | 2 + .../D/D0-demo-felhom-waiting.json | 12 + .../D/D0-demo-hp-waiting.json | 122 ++ .../night-2026-09-25/D/D0-waiting-steps.md | 11 + .../audits/night-2026-09-25/D/D1-floor.txt | 4 + .../audits/night-2026-09-25/D/D2-arrival.txt | 3 + .../night-2026-09-25/D/D3-switch-window.txt | 19 + .../audits/night-2026-09-25/E/E0-drift.json | 728 ++++++++++ .../audits/night-2026-09-25/E/E0-drift.log | 64 + .../night-2026-09-25/E/E1-bench-create.txt | 7 + .../E/apps/n8n-control/bench/MV-n8n.log | 30 + .../bench/evidence/MV-n8n/compose-final.log | 0 .../n8n-control/bench/evidence/MV-n8n/run.log | 3 + .../bench/evidence/MV-n8n/verdict.json | 27 + .../E/apps/n8n/bench/MV-n8n.log | 30 + .../bench/evidence/MV-n8n/compose-final.log | 0 .../E/apps/n8n/bench/evidence/MV-n8n/run.log | 3 + .../n8n/bench/evidence/MV-n8n/verdict.json | 27 + .../E/apps/n8n/box/badges.json | 21 + .../E/apps/n8n/box/observables-before.json | 15 + .../audits/night-2026-09-25/E/benchq-Q1.txt | 5 + .../audits/night-2026-09-25/E/drift.py | 219 +++ .../F/F1-agent-refusal-live.txt | 32 + .../F/F2-agent-preflight-live.txt | 10 + .../audits/night-2026-09-25/F/F3-sign-hp.txt | 26 + .../night-2026-09-25/F/F3-sign-n100.txt | 24 + .../F/redproof-r685-liststorage.txt | 7 + .../F/redproof-r685-nocheck.txt | 7 + .../audits/night-2026-09-25/PROGRESS.md | 20 + .../night-2026-09-25/REPORT-DRAFT-AB.md | 31 + .../night-2026-09-25/phase0-capacity.txt | 41 + .../night-2026-09-25/phase0-drill-reset.txt | 2 + .../night-2026-09-25/tools/a4_backup.py | 35 + .../audits/night-2026-09-25/tools/applines.py | 13 + .../audits/night-2026-09-25/tools/benchq.py | 48 + .../audits/night-2026-09-25/tools/c1_state.py | 15 + .../audits/night-2026-09-25/tools/c_night.py | 33 + .../night-2026-09-25/tools/c_night3_kill.py | 38 + .../night-2026-09-25/tools/c_night4_power.py | 45 + .../audits/night-2026-09-25/tools/chaos.py | 482 +++++++ .../night-2026-09-25/tools/chaos_schedule.py | 65 + .../night-2026-09-25/tools/d_waiting.py | 47 + .../audits/night-2026-09-25/tools/d_watch.py | 24 + .../night-2026-09-25/tools/drillmark.py | 24 + .../night-2026-09-25/tools/drillretest.py | 21 + .../audits/night-2026-09-25/tools/fixtures.py | 1085 +++++++++++++++ .../night-2026-09-25/tools/liveCsetup.py | 118 ++ .../audits/night-2026-09-25/tools/movewin.py | 19 + .../audits/night-2026-09-25/tools/repoint.py | 60 + .../audits/night-2026-09-25/tools/run_edge.py | 193 +++ .../audits/night-2026-09-25/tools/seedread.py | 24 + .../audits/night-2026-09-25/tools/switch.py | 14 + .../audits/night-2026-09-25/tools/teardown.py | 29 + .../tools/unattended_caller_ladder.py | 111 ++ .../audits/night-2026-09-25/tools/walk.py | 507 +++++++ documentation/backlog/CLOSED-ITEMS.md | 6 +- documentation/backlog/OPEN-ITEMS.md | 10 +- scripts/wire_contract_gate.py | 9 + 126 files changed, 10324 insertions(+), 9 deletions(-) create mode 100644 documentation/audits/night-2026-09-25/A/A1-demo-felhom.txt create mode 100644 documentation/audits/night-2026-09-25/A/A1-demo-hp.txt create mode 100644 documentation/audits/night-2026-09-25/A/A2-demo-felhom.txt create mode 100644 documentation/audits/night-2026-09-25/A/A2-demo-hp.txt create mode 100644 documentation/audits/night-2026-09-25/A/A3-demo-felhom-restore-test.txt create mode 100644 documentation/audits/night-2026-09-25/A/A3-demo-hp-refusal.txt create mode 100644 documentation/audits/night-2026-09-25/A/A4-hp-backup-run.txt create mode 100644 documentation/audits/night-2026-09-25/A/A4-hp-backup-space.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D12-default-off.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D13-files-may-change-ignored.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D13-needs-person-ignored.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D14-two-steps-a-night.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D20-gate-does-not-wait.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D20-leg-ignores-W5h.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/D28-updating-app-sampled.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/R678-no-rescan-at-done.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/R680-no-failed-step-check.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/SUMMARY.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/chain-leg-not-deferred.txt create mode 100644 documentation/audits/night-2026-09-25/B/redproofs/wiring-SetUpdateLeg-dropped.txt create mode 100644 documentation/audits/night-2026-09-25/C/C0-9202-window-before.txt create mode 100644 documentation/audits/night-2026-09-25/C/C0-deploy-9202.txt create mode 100644 documentation/audits/night-2026-09-25/C/C0-repoint-drill.txt create mode 100644 documentation/audits/night-2026-09-25/C/C1-setup.json create mode 100644 documentation/audits/night-2026-09-25/C/C1-setup.log create mode 100644 documentation/audits/night-2026-09-25/C/C1-state-before-night1.json create mode 100644 documentation/audits/night-2026-09-25/C/data-readback.txt create mode 100644 documentation/audits/night-2026-09-25/C/night1/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night1/controller-full.log create mode 100644 documentation/audits/night-2026-09-25/C/night1/controller-night.log create mode 100644 documentation/audits/night-2026-09-25/C/night1/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night1/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night2/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night2/controller-full.log create mode 100644 documentation/audits/night-2026-09-25/C/night2/controller-night.log create mode 100644 documentation/audits/night-2026-09-25/C/night2/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night2/runner.log create mode 100644 documentation/audits/night-2026-09-25/C/night2/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/controller-after-kill.log create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/controller-before-kill.log create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/runner.log create mode 100644 documentation/audits/night-2026-09-25/C/night3-kill/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/controller-after-boot.log create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/controller-before-cut.log create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/runner.log create mode 100644 documentation/audits/night-2026-09-25/C/night4-power/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/controller-full.log create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/controller-night.log create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/runner.log create mode 100644 documentation/audits/night-2026-09-25/C/night5-off/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night5-switch-off.txt create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/app-lines.txt create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/controller-full.log create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/controller-night.log create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/pages-after.json create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/runner.log create mode 100644 documentation/audits/night-2026-09-25/C/night6-on/window-moved.txt create mode 100644 documentation/audits/night-2026-09-25/C/night6-switch-on.txt create mode 100644 documentation/audits/night-2026-09-25/D/D0-demo-felhom-waiting.json create mode 100644 documentation/audits/night-2026-09-25/D/D0-demo-hp-waiting.json create mode 100644 documentation/audits/night-2026-09-25/D/D0-waiting-steps.md create mode 100644 documentation/audits/night-2026-09-25/D/D1-floor.txt create mode 100644 documentation/audits/night-2026-09-25/D/D2-arrival.txt create mode 100644 documentation/audits/night-2026-09-25/D/D3-switch-window.txt create mode 100644 documentation/audits/night-2026-09-25/E/E0-drift.json create mode 100644 documentation/audits/night-2026-09-25/E/E0-drift.log create mode 100644 documentation/audits/night-2026-09-25/E/E1-bench-create.txt create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/MV-n8n.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/compose-final.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/run.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/verdict.json create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/bench/MV-n8n.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/compose-final.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/run.log create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/verdict.json create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/box/badges.json create mode 100644 documentation/audits/night-2026-09-25/E/apps/n8n/box/observables-before.json create mode 100644 documentation/audits/night-2026-09-25/E/benchq-Q1.txt create mode 100644 documentation/audits/night-2026-09-25/E/drift.py create mode 100644 documentation/audits/night-2026-09-25/F/F1-agent-refusal-live.txt create mode 100644 documentation/audits/night-2026-09-25/F/F2-agent-preflight-live.txt create mode 100644 documentation/audits/night-2026-09-25/F/F3-sign-hp.txt create mode 100644 documentation/audits/night-2026-09-25/F/F3-sign-n100.txt create mode 100644 documentation/audits/night-2026-09-25/F/redproof-r685-liststorage.txt create mode 100644 documentation/audits/night-2026-09-25/F/redproof-r685-nocheck.txt create mode 100644 documentation/audits/night-2026-09-25/PROGRESS.md create mode 100644 documentation/audits/night-2026-09-25/REPORT-DRAFT-AB.md create mode 100644 documentation/audits/night-2026-09-25/phase0-capacity.txt create mode 100644 documentation/audits/night-2026-09-25/phase0-drill-reset.txt create mode 100644 documentation/audits/night-2026-09-25/tools/a4_backup.py create mode 100644 documentation/audits/night-2026-09-25/tools/applines.py create mode 100644 documentation/audits/night-2026-09-25/tools/benchq.py create mode 100644 documentation/audits/night-2026-09-25/tools/c1_state.py create mode 100644 documentation/audits/night-2026-09-25/tools/c_night.py create mode 100644 documentation/audits/night-2026-09-25/tools/c_night3_kill.py create mode 100644 documentation/audits/night-2026-09-25/tools/c_night4_power.py create mode 100644 documentation/audits/night-2026-09-25/tools/chaos.py create mode 100644 documentation/audits/night-2026-09-25/tools/chaos_schedule.py create mode 100644 documentation/audits/night-2026-09-25/tools/d_waiting.py create mode 100644 documentation/audits/night-2026-09-25/tools/d_watch.py create mode 100644 documentation/audits/night-2026-09-25/tools/drillmark.py create mode 100644 documentation/audits/night-2026-09-25/tools/drillretest.py create mode 100644 documentation/audits/night-2026-09-25/tools/fixtures.py create mode 100644 documentation/audits/night-2026-09-25/tools/liveCsetup.py create mode 100644 documentation/audits/night-2026-09-25/tools/movewin.py create mode 100644 documentation/audits/night-2026-09-25/tools/repoint.py create mode 100644 documentation/audits/night-2026-09-25/tools/run_edge.py create mode 100644 documentation/audits/night-2026-09-25/tools/seedread.py create mode 100644 documentation/audits/night-2026-09-25/tools/switch.py create mode 100644 documentation/audits/night-2026-09-25/tools/teardown.py create mode 100644 documentation/audits/night-2026-09-25/tools/unattended_caller_ladder.py create mode 100644 documentation/audits/night-2026-09-25/tools/walk.py diff --git a/documentation/architecture/00-capability-map.md b/documentation/architecture/00-capability-map.md index 8e45f4a0..169a79c4 100644 --- a/documentation/architecture/00-capability-map.md +++ b/documentation/architecture/00-capability-map.md @@ -111,7 +111,7 @@ likewise silent. Evidence: `audits/DRILL-r361-2026-08-22/evidence/06-part3-decis | **A crash-looping or out-of-memory app is STOPPED by the box, the household and operator are told, and Start gives one more try** (decision 28) | controller **v0.269.0**, hub **v0.123.0** | **PROVEN-LIVE (2026-09-24)** | `audits/night-2026-09-24/A3/21-gokapi-page-event.txt`, `A3/30-start-then-trip2.*`, `E/round-01/07/08.json` | gokapi stopped at 6 restarts in 10 min; Start lifted it, 9 fast restarts, stopped again with the support-informed sentence (hu + en); chaos hour: OOM storm stopped after a power cut (+185 s), a crash loop under a backup run (+116 s), a storm on a nearly full disk (+102 s). | | **A held app whose page says support is informed can only be removed KEEPING its data** (decision 27) | controller **v0.269.0** | **PROVEN-LIVE (2026-09-24)** | `audits/night-2026-09-24/A2/10-held-no-copy-keep-data.*` | the dialog reads `keep_data_only`; Remove with data or backups → 409 in both languages; the app stayed installed. | | **The box runs the exact TESTED image of a floating tag, and an installed app keeps its image until a guarded Update moves it** (`09` §6.4 part 6, box half) | controller **v0.269.1** (`stacks/digest.go`) | **PROVEN-LIVE (2026-09-24)** | `audits/night-2026-09-24/B/21-floating-tag-0.269.1.*`, `B/01-compose-accepts-digests.txt` | redis:7-alpine: a newer tested digest → the „Frissítés elérhető" badge, the sync left the running file alone (v0.269.0's sync did not — fixed), Update pulled exactly that digest. Compose accepts `tag@digest` for all 25 ladder apps (37 digests). | -| **Automatic updates at night — the update leg** (`09` §6.4 part 7) | — | **SPIKED, NOT BUILT (2026-09-24)** | `audits/night-2026-09-24/C/31-night.json`, `09` §6.4.2 | a caller pressing only the public Update climbed 3 apps (1–3 steps) and set aside a failing one in 7 min 18 s; three gaps filed (R-678, R-679, R-680). | +| **Automatic updates at night — the update leg** (`09` §6.4 part 7) | controller **v0.271.0** | **PROVEN-LIVE on scratch 9202 (six simulated nights, 2026-09-24/25) — the demo boxes' first real night: see `audits/DRILL-night-2026-09-25.md` Part D** | `audits/night-2026-09-25/C/`, `B/redproofs/` | after the off-site leg on every path; one step per app per night; `needs_person` never, `files_may_change` only with a whole copy; a failed step not re-pressed until the catalog re-tests it (R-680); a power cut mid-step resumed and finished; a controller kill mid-step put back; switch off = nothing pressed; data read back after every night. **Not claimed:** W+5h with steps left and a FAILING off-site leg live (unit only), the full-system gate waiting live (9202 has no agent), resume of the leg after a restart (R-686). | | **What `restart` and `update` do to a deployed app whose compose file the catalog already moved** | controller **v0.235.0** | **CHANGED 2026-09-06 — they NO LONGER upgrade it.** The row below records what shipped; this text records what it replaced, because every box under v0.235.0 still behaves the old way. **Up to v0.234.0: PROVEN-LIVE (2026-09-01) — they UPGRADE it.** Every lifecycle action ends in `docker compose up -d`, which makes the container match the file and PULLS the image itself when it is missing (measured: 18.3 s with a pull, 0.5 s without; negative control with an unchanged file did not even recreate the container). This is DELIBERATE on the restart path — `Manager.RestartStack` says so in a comment — but the syncer moves the file under a deployed app on a 15-minute cycle with no deployed check (R-438), and NOTHING tells the customer. | `audits/SPIKE-app-update-2026-09-01.md` §2, §3 | **No safety copy is taken by any of them** — `writeSafetyDump` is DATABASE-ONLY and is not on the update path at all. R-438, R-440, R-443. | | **Whether the box UPGRADES an app by itself, with nobody pressing anything** | controller | **PROVEN-LIVE (2026-09-01) — YES, but only when an app fails to come back.** A plain power cut does NOT upgrade: Docker's `restart: unless-stopped` restores the old containers and the reconciler logs `no boot-orphaned apps (nothing to start)`. When an app does NOT return, `Reconciler.Run` (`bootrecon.go:269`) calls `StartStack` -> `compose up -d` and the app comes back on the NEW version, unattended (measured). **13 non-API call sites across 9 files reach `up -d` this way** — not the five previously believed. | `audits/SPIKE-app-update-2026-09-01.md` §2, §8 | The drive-return gate (`intermediary.go:222`) and `AppStopGuard.Recover` (`appstop_marker.go:283`) call the same function; located by reading, **not exercised live** — stated as such. | | **Whether an app UPGRADE can be undone** | controller + catalog | **PROVEN-LIVE (2026-09-01) — NO, and "rollback" is the wrong word for it.** Once a migration has RUN, putting the old image tag back yields a container that refuses to start: Nextcloud — *"the version of the data (32.0.9.2) is higher than the docker image version (31.0.14.1) and downgrading is not supported"*. A 3-major jump is refused outright (*"only possible to upgrade one major version at a time"*) and IS recoverable, precisely because nothing migrated. Positive control: the data is not destroyed — returning to 32.0.9 restored both seeded markers byte-identical. | `audits/SPIKE-app-update-2026-09-01.md` §7 | The only route back is restoring DATA from a copy taken BEFORE the update — which no update path takes. And a restore's image-level rollback is itself overwritten by the syncer within 15 minutes (R-441). R-40 is confirmed live by the same measurement. | diff --git a/documentation/architecture/03-host-agent.md b/documentation/architecture/03-host-agent.md index f9289cd2..a649010f 100644 --- a/documentation/architecture/03-host-agent.md +++ b/documentation/architecture/03-host-agent.md @@ -332,6 +332,17 @@ per Part 1: **snapshot** (LVM-thin, transient, whole-guest rollback — not a ba > `/etc/felhom-agent/agent.json.pre-r672`. On demo-hp, a full restore-test of 9201 does not fit today > (21.1 GiB restored needs 30.3 GiB; 22.1 GiB free) — the preflight will refuse it, correctly, until the > pool has room. +> +> **Delivered 2026-09-24 night** to demo-felhom and demo-hp by CC-signed `agent_update` jobs; the restore test is +> back ON on both (the `-1` config kept as `agent.json.night-0925-off`). Peti's box has not received it. + +> **A whole-box backup that cannot fit is skipped with a reason, before anything starts (R-685, agent v0.134.0).** +> Before a vzdump to a LOCAL (non-PBS) target: free ≥ the guest's newest archive on that target × 1.25 + 1 GiB, +> because PVE prunes old archives only AFTER a successful backup. A shortfall is a named skip (`skipped: not enough +> space: has X GiB free; the last archive … needs about Z GiB`) that reaches the tier view and the +> operator; it FAILS OPEN on a PBS target, a first backup or unreadable usage. **Free space comes from +> `GET /nodes//storage` — `GET /storage` is the definitions and has NO usage** (a build that read it let a +> real vzdump start in its own live test, 2026-09-24 night — aborted, no archive left). - **Quiescing (controller-driven for app-consistency) — implemented (slice 8B):** an LXC has no fsfreeze (`proxmox-platform.md` §4.2), so app-consistency is the controller's job: it learns a diff --git a/documentation/architecture/07-backup-architecture.md b/documentation/architecture/07-backup-architecture.md index a7e0fd02..63434385 100644 --- a/documentation/architecture/07-backup-architecture.md +++ b/documentation/architecture/07-backup-architecture.md @@ -347,6 +347,19 @@ never moved by an update or an undo, and the unit restore accepts it. offsets — db-dump at W, Tier-2 at W+60m, offsite at W+105m — so they can never be misordered (`cmd/controller/main.go:604-607`). Both boxes run W = `02:30`. +**[FACT, controller v0.271.0, 2026-09-25] A fourth leg: automatic app updates, CHAINED after the off-site +leg** (`09` §3 decisions 11 and 20, §6.4 part 7). The `offbox-backup` job runs the update leg when its off-site +half ends, on every path (no target, ok, failed, panicked); a box without an off-site tier runs it at W+105m. The +leg starts no step at or after **W+5h**, and the whole-guest gate — open [W+2h, W+6h) — **defers while the leg +runs until W+5h**, then only for a step already in flight, never past W+5h30m (decision 31). So on an update +night the whole-guest backup starts later inside its own window; it is never skipped for an update. Live +numbers: `audits/DRILL-night-2026-09-25.md` Part D. + +**[FACT, 2026-09-25] demo-hp's local whole-guest target is its host ROOT disk; retention there is now +keep-last=1** (operator ruling 2026-09-24, option A; R-684). PVE prunes AFTER a successful backup, so the target +must hold keep-last + 1 archives during the run — measured: a 9201 archive is 8.18 GB, the root disk 39 GB. The +warning that should precede such a failure is R-685. + **[FACT] What the whole-guest tiers do NOT carry.** `mp8 /mnt/felhom-drives` and `mp9 /etc/felhom-bootstrap` are **host bind mounts** and are out of `vzdump` scope entirely (LIVE `pct config 9201`, both hosts). So the whole-guest tiers carry the guest and **none of the customer's diff --git a/documentation/architecture/08-alarm-ladder.md b/documentation/architecture/08-alarm-ladder.md index 69298f75..5d72e1a7 100644 --- a/documentation/architecture/08-alarm-ladder.md +++ b/documentation/architecture/08-alarm-ladder.md @@ -245,6 +245,12 @@ container restarts fast (9 in 32 s after a Start, measured). No healthy app in a harness samples, 40 live containers) restarted more than once on a first start; the one borderline shape is immich's first-start import (12 restarts, 2026-09-17 — broken that night; R-676 watches it). **Never judged:** an app that is deploying, updating, held, or stopped by a backup or a quiesce. +**Precisely (read from source, controller v0.271.0, 2026-09-25):** "updating" covers an automatic update's +whole step — pull, start, verify AND its undo — because `Updating` stays true until the step ends +(`TestD28_NoCrashLoopStopDuringAnAutomaticStep`; the leg's own pages are therefore never stopped mid-step). +"Deploying" does **not** cover a deploy's FIRST START: the flag clears when `compose up -d` returns, and the +app is sampled from then on — a first start that restarts ≥ 6 times in 10 min is stopped (R-676). An automatic +update is a product stop in the §5 sense: a step's restarts are the product's own, never an alarm. **Measured in the chaos hour (night 2026-09-24):** stopped at +185 s after a power cut mid-storm (the counting starts again after the boot), +116 s for a crash loop under a backup run, +102 s for a storm with the disk 1 GB above the floor. diff --git a/documentation/architecture/09-update-architecture.md b/documentation/architecture/09-update-architecture.md index b4dd4e0b..8824eb6a 100644 --- a/documentation/architecture/09-update-architecture.md +++ b/documentation/architecture/09-update-architecture.md @@ -370,6 +370,30 @@ R-636's louder repeated alarm. (a) broke the second. Shipped as controller **v0.269.1** — a second release in the session, instead of shipping (a) to the fleet with the floor. +### 2026-09-25 (night) — three decisions taken by CC unattended, building §6.4 part 7 + +31. **After W+5h the full-system backup waits only for an automatic step ALREADY in flight, and never past + W+5h30m** — *decided by CC unattended 2026-09-25 — operator may reverse.* *One sentence:* when the leg is + still running at W+5h, does the full-system backup start at once or wait for the step? **Options:** (a) start + at once — the backup's quiesce stops an app in the middle of its update's verify, which fails the step into + an undo during a backup; (b) wait for the step in flight, capped at W+5h30m. **Costs:** (a) a spurious undo + and a backup of an app mid-change; (b) the full-system backup keeps at least 30 minutes of its window instead + of 60 on the worst night; a stuck flag cannot hold it past the cap. **Why (b):** §6.4.2 point 3 already says + "a step already running finishes"; this makes the gate agree with it. `quiesce.updateLegDefers`, + `TestD20_UpdateLegDefers`. +32. **The controller's own self-update waits for the whole automatic leg, not only for a step in flight** — + *decided by CC unattended 2026-09-25 — operator may reverse.* *One sentence:* may the controller swap itself + between two automatic steps? **Options:** (a) yes, as for manual updates (the lock covers only a step in + flight); (b) no, the lock covers the whole leg. **Costs:** (a) the swap restarts the controller and the rest + of the night's leg is lost (it is not resumed); (b) a floor-served controller release waits up to the leg's + length (at most until W+5h) on an update night. **Why (b):** the leg is the night's only update window + (decision 11); a release can wait an hour, a household's app cannot wait a day for nothing. +33. **The automatic leg takes ONE step per app per night** — *follows the operator's own brief of 2026-09-24 + ("one app at a time, one tested step per app"), recorded here because §6.4.2 point 6 (a) said "rescan between + steps", which reads as climbing several.* **Cost:** an app three steps behind takes three nights. **Why:** + each step then gets a night of the household using it before the next one, and a failure is one step wide. + Operator may widen it. `TestLeg_OneStepPerAppPerNight`. + --- ## 3b. ANSWERED 2026-09-23 — the seven questions Slices 6 and 7 needed @@ -1140,7 +1164,7 @@ what the part can do to a household's data if it is wrong, not how likely that i | **4** | **SHIPPED — catalog `6db08a5`, night 2026-09-23** (`audits/DRILL-night-2026-09-23.md`): `update_ladder:` in `.felhom.yml`, one JSON entry per line (spiked on controller v0.266.0 and v0.267.0 first — the controller ignores the key); `check-test-record.py` (static, CI) + `check-test-record-move.py` (history + the registry for moved refs only, decision 23); the only writer `upgrade-test.py --write-ladder`; the 21 moves of 2026-09-22 backfilled from their records (21 proven). Not built: `steps/.yml` (part 5 needs it once an app has two steps); `CompareImageRefs` did NOT move to the gate — the gate asks for a proven test instead, which is decision 13's own test. **The test record + the catalog gate + the memory check.** The harness writes the ladder entry (below) from its verdict record, including the memory watch's peak and marks; the gate refuses an image move with no entry, an entry with a `failed` verdict, or one with no memory watch; `CompareImageRefs`' rule moves here as the push-time safety net. **Backfill:** one entry per current pin — the 21 proven moves from their records, every other pin `needs_person: "never tested"`, which is honest and keeps them manual. A version move re-checks `mem_limit` against the watch's peak (the RomM follow-up: gate, not checklist, because the watch now produces the number). | 13, R-635 follow-up | **2.5** | the memory watch (shipped 2026-09-23) | none on a box — catalog-side only | | **5** | **SHIPPED — controller v0.268.0 (`206b035`) + catalog `5ed599c`, proven live on 9202 2026-09-24** (`audits/ladder-2026-09-24/partD/`): romm 5.3.0/11.4 → 5.3.1/11.4 (its app step, from `steps/90dd9d68258286ef.yml`) → 5.3.1/11.8 (the engine step; `mariadb-upgrade` ran) in two presses, the seeded account read back after each, the page's „Hátralévő frissítési lépések" 2 → 1 → none. Step files: `templates//steps/.yml` (sha256 of `to` as canonical JSON, 16 hex; `stacks.StepKey` = `ladder.step_key`), refused absent or wrong by `check-test-record.py` rule 4, written by `--write-ladder` when a step is superseded, 8 backfilled from the NEWEST commit naming each step's images (romm's from `f4eb94f`, not the OOM-looping `15f9ebf`). The box reads them from its `--depth 1` clone (the whole tree is there — measured). One press = one step; a missing step file refuses before anything moves; an installed version matching no entry jumps, logged by name (measured live: vikunja 2.5.0). Limitation: a step has no `.felhom.yml` of its own (R-664). **The ladder on the box.** Read `update_ladder:` from the clone, find the installed step, apply ONE step with its OWN definition (`steps/.yml`, the last step the current template), repeat next night; a failed step stops the ladder for that app. `CatalogOrder` compares refs with the digest stripped (see part 7). | 14 | **2.5** | 1, 4 | medium — each step is the guarded update + undo; the new risk is rendering the wrong step's definition, pinned by a test per step shape | | **6** | **BOTH HALVES SHIPPED. BOX HALF — controller v0.269.0 + v0.269.1, proven live on 9202 2026-09-24** (`audits/night-2026-09-24/B/`): the compose that runs pins `tag@sha256` from the ladder entry that tested those refs; pins and records stay digest-free; the badge reads behind for a newer TESTED digest; **an installed app keeps its digest until a guarded Update moves it** (v0.269.0 let the sync move it — found live, fixed in v0.269.1, `CarryDigests`); Compose accepts the form for all 25 ladder apps. **CATALOG HALF SHIPPED — night 2026-09-23:** every ladder entry carries the digest per `to` ref (`scripts/image_digest.py`, stdlib; equals Docker's `RepoDigests` on a box), and the move gate refuses a digest the registry no longer serves. **The box half (compare, render `name:tag@sha256`) is not built.** **Digests.** The catalog records `sha256` per pin at push time (`check-image-resolvable.py` already resolves it); the box compares it for the badge and renders `name:tag@sha256:…` when present. **Measured 2026-09-23 on 9202:** Docker and Compose both pull and run `redis:7-alpine@sha256:858f…`, and refuse a digest that does not exist (`audits/update-rulings-2026-09-23/70-…`). **Build trap, read from source:** `splitImageRef` returns "unorderable" for ANY ref containing `@` (`updateorder.go:134`), so the digest must be split off before ordering or every digest-pinned app reads Unknown. A digest gone upstream fails the PULL — Scenario E, pin back, nothing ran. | 17, R-446 | **2** | 4 (the entry carries the digest) | low | -| **7** | **SPIKED 2026-09-24 (night, Part C) — the measured build brief is §6.4.2 below.** **The update leg in the chain + the automatic caller + the switch.** A leg that starts when the off-site leg has ENDED (on every exit path, skips included), one app at a time, one step per press, `app_update.unattended` default ON (decision 12), `stacks.update_window` removed, reads `UpdateRefusal.Reason`, remembers a failed step so it never re-presses it, and the full-system backup's gate waits for it until W+5h (decision 20). | 11, 12, 14, 15, 20 | **3** | 1, 2, 5 | medium — the only part that acts with nobody watching; everything above is what makes it safe | +| **7** | **SHIPPED — controller v0.271.0 (`9cf13a3`), proven on 9202 over four simulated nights 2026-09-24/25 and watched through the demo boxes' first real automatic night (`audits/DRILL-night-2026-09-25.md`).** Decisions 31–33 taken unattended. Gaps a scratch box cannot close: R-687; no resume after a restart: R-686. *(Before:)* **SPIKED 2026-09-24 (night, Part C) — the measured build brief is §6.4.2 below.** **The update leg in the chain + the automatic caller + the switch.** A leg that starts when the off-site leg has ENDED (on every exit path, skips included), one app at a time, one step per press, `app_update.unattended` default ON (decision 12), `stacks.update_window` removed, reads `UpdateRefusal.Reason`, remembers a failed step so it never re-presses it, and the full-system backup's gate waits for it until W+5h (decision 20). | 11, 12, 14, 15, 20 | **3** | 1, 2, 5 | medium — the only part that acts with nobody watching; everything above is what makes it safe | | **8** | **SHIPPED — controller v0.265.0 + hub v0.121.0, proven live on 9202 2026-09-23** (`audits/cleanup-2026-09-23/`; a kernel `oom_kill` counter, not the sticky flag — `08` §6.2). **R-636** — the same OOM key re-firing escalates instead of staying one `warning` for six hours. | R-636 | **1** | — | none | | **9** | **SHIPPED — controller v0.265.0, proven live on 9202 2026-09-23** (badge „Megállítva — visszaállítás szükséges" / "Stopped — restore needed", no Update button, 409 `held` unchanged). **R-625** — a held app stops offering an Update it will refuse. With the undo, holds become rarer; the lie on the page does not go away by itself. | R-625 | **0.5** | 1 | none | | **10** | **PostgreSQL majors converted by the box.** A guarded-update step: `pg_dumpall` from the old engine, a NEW datadir (the old one kept aside, never deleted, until the check passes), load, check; then each of the eleven apps proven on the bench before its catalog move. | 16, R-463 | **2 + 3** | 1 (the same load discipline), 4 | **HIGH** — it rebuilds the datadir; bounded by keeping the old datadir aside | @@ -1225,6 +1249,17 @@ per-box switch `app_update.unattended`, **default ON (decision 12)**, and `stack removed; (e) the live proof: this night's four apps, with the window moved forward through the product's own form (`POST /backups/window` — measured: the three legs reschedule at once). +**6.4.2 as BUILT (v0.271.0) — where the build differs from the brief above, named:** (a) the leg is +`stacks.RunUpdateLeg`, chained by `chainUpdateLeg` inside the `offbox-backup` job (every path, a panic included); +**one step per app per night** (decision 33 — not "rescan between steps" in one night); (b) the failed-step record is +`app.yaml` `failed_update_step` tied to the ladder's print (R-680); R-679's `current` refusal shipped in v0.270.0; +(c) the gate interlock is `quiesce.Options.UpdateLegFn` with decision 31's in-flight grace; (d) the switch is +`settings.json` `app_update.unattended`, absent = ON, a card on the settings page; `stacks.update_window` removed; +(e) the live proof: `audits/DRILL-night-2026-09-25.md` Part C (four nights on 9202) and Part D (the demo boxes). +**Found live, not in the brief:** the leg presses the next app in the same second the previous step ends, so "a +controller killed BETWEEN two apps" lands inside the next step's early phases — which the guarded update puts back +(Scenario G); the rest of the night is lost (R-686). + ### 6.4.1 (record) The update night — the drill brief that preceded the rulings, costed and re-costed **The ruling is decision 6: all 53 apps, through the nightly rotation.** This is an ORDER inside that diff --git a/documentation/audits/night-2026-09-25/A/A1-demo-felhom.txt b/documentation/audits/night-2026-09-25/A/A1-demo-felhom.txt new file mode 100644 index 00000000..4ef3c17e --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A1-demo-felhom.txt @@ -0,0 +1,24 @@ +# A1 demo-felhom-8363b5 agent_update 0.133.0 — sha256 verified against published package: 3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 +## 2026-09-24T19:25:30+00:00 before: felhom-agent 0.132.0 +signed: op=agent_update host=demo-felhom-8363b5 guest="" key_id=felhom-op-1 nonce=fe69abc5ffe7ec94d0d426f78dae4c8b expires=2026-09-24T20:10:31Z +wrote envelope to /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/c9309498-57c3-4b75-aa79-a24434763a8b/scratchpad/au-n100.json +uploaded signed op to the hub jobs queue +rc=0 +## 2026-09-24T19:34:20+00:00 felhom-pve now runs 0.133.0 - waiting 80 s for dwell/commit + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.465+02:00 level=INFO msg="audit: gate decision" class=agent_update host=demo-felhom-8363b5 guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=fe69abc5… durable_id="" + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.465+02:00 level=INFO msg="gate decision" class=agent_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.465+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=c4b592ff7c9ff307 op=agent_update key_id=felhom-op-1 nonce=fe69abc5ffe7ec94d0d426f78dae4c8b + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.466+02:00 level=WARN msg="agent_update: downloading operator-signed binary" version=0.133.0 url=https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.133.0/felhom-agent sha256=3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.747+02:00 level=WARN msg="agent_update: handing staged binary to the guarded wrapper" staged=/var/lib/felhom-agent/selfupdate/felhom-agent-0.133.0 version=0.133.0 + [felhom-pve] Sep 24 21:34:05 demo-felhom sudo[3858148]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded apply /var/lib/felhom-agent/selfupdate/felhom-agent-0.133.0 3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.890+02:00 level=WARN msg="agent_update: apply handed off; restart scheduled" version=0.133.0 wrapper="" + [felhom-pve] Sep 24 21:34:05 demo-felhom felhom-agent[3559458]: time=2026-09-24T21:34:05.890+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=c4b592ff7c9ff307 op=agent_update + [felhom-pve] Sep 24 21:34:08 demo-felhom felhom-agent[3858230]: time=2026-09-24T21:34:08.547+02:00 level=INFO msg="backup: restore-test cadence disabled" + [felhom-pve] Sep 24 21:34:09 demo-felhom felhom-agent[3858230]: time=2026-09-24T21:34:09.787+02:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.133.0 prev=0.132.0 dwell=1m0s + [felhom-pve] Sep 24 21:34:09 demo-felhom felhom-agent[3858230]: time=2026-09-24T21:34:09.788+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s + [felhom-pve] Sep 24 21:35:09 demo-felhom sudo[3859585]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit + [felhom-pve] Sep 24 21:35:09 demo-felhom felhom-selfupdate-guarded[3859596]: committed (pending cleared, .prev retained) + [felhom-pve] Sep 24 21:35:09 demo-felhom felhom-agent[3858230]: time=2026-09-24T21:35:09.821+02:00 level=WARN msg="selfupdate: update committed" version=0.133.0 wrapper="" + [felhom-pve] active + [felhom-pve] felhom-agent 0.133.0 +hub /hosts at 2026-09-24T19:36:36+00:00: demo-felhom-8363b5 agent 0.133.0 ONLINE; demo-hp-bb76ea 0.132.0 (job queued) diff --git a/documentation/audits/night-2026-09-25/A/A1-demo-hp.txt b/documentation/audits/night-2026-09-25/A/A1-demo-hp.txt new file mode 100644 index 00000000..d4d1fd70 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A1-demo-hp.txt @@ -0,0 +1,25 @@ +# A1 demo-hp-bb76ea agent_update 0.133.0 — sha256 verified against published package: 3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 +## 2026-09-24T19:35:55+00:00 before: felhom-agent 0.132.0 +signed: op=agent_update host=demo-hp-bb76ea guest="" key_id=felhom-op-1 nonce=e631a534fa2d9781f3c631631b8e8b96 expires=2026-09-24T20:20:56Z +wrote envelope to /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/c9309498-57c3-4b75-aa79-a24434763a8b/scratchpad/au-hp.json +uploaded signed op to the hub jobs queue +rc=0 +## 2026-09-24T19:49:16+00:00 hp now runs 0.133.0 - waiting 80 s for dwell/commit + [hp] Sep 24 21:48:57 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:57.876+02:00 level=INFO msg="audit: gate decision" class=agent_update host=demo-hp-bb76ea guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=e631a534… durable_id="" + [hp] Sep 24 21:48:57 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:57.876+02:00 level=INFO msg="gate decision" class=agent_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed + [hp] Sep 24 21:48:57 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:57.877+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=36aab1ed8ad00979 op=agent_update key_id=felhom-op-1 nonce=e631a534fa2d9781f3c631631b8e8b96 + [hp] Sep 24 21:48:57 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:57.877+02:00 level=WARN msg="agent_update: downloading operator-signed binary" version=0.133.0 url=https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.133.0/felhom-agent sha256=3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 + [hp] Sep 24 21:48:58 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:58.306+02:00 level=WARN msg="agent_update: handing staged binary to the guarded wrapper" staged=/var/lib/felhom-agent/selfupdate/felhom-agent-0.133.0 version=0.133.0 + [hp] Sep 24 21:48:58 demo-hp sudo[727939]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded apply /var/lib/felhom-agent/selfupdate/felhom-agent-0.133.0 3aa303452b8c6be58573d00af01a0ab4a0d97e4f885ffecd0144a18ac24e69b6 + [hp] Sep 24 21:48:58 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:58.501+02:00 level=WARN msg="agent_update: apply handed off; restart scheduled" version=0.133.0 wrapper="" + [hp] Sep 24 21:48:58 demo-hp felhom-agent[3717779]: time=2026-09-24T21:48:58.501+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=36aab1ed8ad00979 op=agent_update + [hp] Sep 24 21:49:01 demo-hp felhom-agent[728032]: time=2026-09-24T21:49:01.515+02:00 level=INFO msg="backup: restore-test cadence disabled" + [hp] Sep 24 21:49:02 demo-hp felhom-agent[728032]: time=2026-09-24T21:49:02.430+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s + [hp] Sep 24 21:49:02 demo-hp felhom-agent[728032]: time=2026-09-24T21:49:02.431+02:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.133.0 prev=0.132.0 dwell=1m0s + [hp] Sep 24 21:50:02 demo-hp sudo[731849]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit + [hp] Sep 24 21:50:02 demo-hp felhom-agent[728032]: time=2026-09-24T21:50:02.455+02:00 level=WARN msg="selfupdate: update committed" version=0.133.0 wrapper="" + [hp] active + [hp] felhom-agent 0.133.0 +hub /hosts at 2026-09-24T19:56:33+00:00: + demo-felhom-8363b5 Demo Ügyfél 0.133.0 floor: declared MinAg + demo-hp-bb76ea Demo HP 0.133.0 floor: declared MinAgent ONLI diff --git a/documentation/audits/night-2026-09-25/A/A2-demo-felhom.txt b/documentation/audits/night-2026-09-25/A/A2-demo-felhom.txt new file mode 100644 index 00000000..8e131168 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A2-demo-felhom.txt @@ -0,0 +1,15 @@ +# A2 demo-felhom: restore test back on — 2026-09-24T19:36:36+00:00 +31c31,32 +< ] +--- +> ], +> "restore_test_eval_interval_seconds": -1 +88c89 +< } +\ No newline at end of file +--- +> } +diff_rc=1 +active +Sep 24 21:36:37 demo-felhom felhom-agent[3860787]: time=2026-09-24T21:36:37.184+02:00 level=INFO msg="backup: restore-test scheduler starting (per-archive due-check)" eval_interval=6h0m0s settle=24h0m0s +Sep 24 21:36:38 demo-felhom felhom-agent[3860787]: time=2026-09-24T21:36:38.416+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s diff --git a/documentation/audits/night-2026-09-25/A/A2-demo-hp.txt b/documentation/audits/night-2026-09-25/A/A2-demo-hp.txt new file mode 100644 index 00000000..15926a84 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A2-demo-hp.txt @@ -0,0 +1,15 @@ +# A2 demo-hp: restore test back on — 2026-09-24T19:56:33+00:00 +31c31,32 +< ] +--- +> ], +> "restore_test_eval_interval_seconds": -1 +88c89 +< } +\ No newline at end of file +--- +> } +diff_rc=1 +active +Sep 24 21:56:35 demo-hp felhom-agent[752235]: time=2026-09-24T21:56:35.100+02:00 level=INFO msg="backup: restore-test scheduler starting (per-archive due-check)" eval_interval=6h0m0s settle=24h0m0s +Sep 24 21:56:35 demo-hp felhom-agent[752235]: time=2026-09-24T21:56:35.952+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s diff --git a/documentation/audits/night-2026-09-25/A/A3-demo-felhom-restore-test.txt b/documentation/audits/night-2026-09-25/A/A3-demo-felhom-restore-test.txt new file mode 100644 index 00000000..f369606e --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A3-demo-felhom-restore-test.txt @@ -0,0 +1,38 @@ +# A3 on-demand restore test on demo-felhom — 2026-09-24T19:37:17+00:00 +== pool before: 3.13 0.58 +=== felhom-agent 0.133.0 selftest=restore-test === + --- recover: reaping any leaked scratch from a prior crashed test --- + recover: examined=0 scratch_destroyed=0 scratch_clean=0 + restoring felhom-backup:backup/vzdump-lxc-9201-2026_09_24-07_27_27.tar.zst into scratch band [990000,990009] on local-lvm … +time=2026-09-24T21:37:18.968+02:00 level=INFO msg="restore-test: space preflight passed" storage=local-lvm required_bytes=12816613376 avail_bytes=362815700966 +time=2026-09-24T21:37:18.991+02:00 level=INFO msg="restore-test: full-fidelity restore params derived from the archive config" scratch=990000 params=4 +time=2026-09-24T21:38:36.605+02:00 level=INFO msg="audit: gate decision" class=guest_destroy host=demo-felhom-8363b5 guest=990000 source=one_shot_job disposition=benign allowed=true reason=benign key_id="" nonce="" durable_id="" +time=2026-09-24T21:38:36.605+02:00 level=INFO msg="gate decision" class=guest_destroy guest=990000 source=one_shot_job disposition=benign allowed=true reason=benign +time=2026-09-24T21:38:42.669+02:00 level=INFO msg="restore-test: scratch guest torn down" vmid=990000 + --- restore-test record --- + { + "source_archive": "felhom-backup:backup/vzdump-lxc-9201-2026_09_24-07_27_27.tar.zst", + "source_tier": "local", + "scratch_vmid": 990000, + "pass": true, + "verified": "boot+running", + "tested_at": "2026-09-24T19:38:42Z", + "duration_seconds": 84.540885524, + "mount_parity": "ok", + "mount_inventory": [ + "mp0=/var/lib/felhom (250G)", + "mp8=/mnt/felhom-drives (throwaway for the archived bind)", + "mp9=/etc/felhom-bootstrap (throwaway for the archived bind)" + ] + } + space preflight passed: storage=local-lvm required=12816613376 avail=362815700966 +=== selftest=restore-test OK (scratch 990000 restored+booted+verified+torn-down in 1m25s) === +selftest_rc=0 +== pool peak (data% meta%): +21:38:28 5.68 0.65 +samples: 9 +== pool after: 3.13 0.58 +== pct list after: +VMID Status Lock Name +9201 running demo-felhom +# end 2026-09-24T19:38:49+00:00 diff --git a/documentation/audits/night-2026-09-25/A/A3-demo-hp-refusal.txt b/documentation/audits/night-2026-09-25/A/A3-demo-hp-refusal.txt new file mode 100644 index 00000000..9a458a00 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A3-demo-hp-refusal.txt @@ -0,0 +1,26 @@ +# A3 demo-hp on-demand restore test (expected: REFUSED for space) — 2026-09-24T19:56:59+00:00 +pool before: 59.01 2.65 +=== felhom-agent 0.133.0 selftest=restore-test === + --- recover: reaping any leaked scratch from a prior crashed test --- + recover: examined=0 scratch_destroyed=0 scratch_clean=0 + restoring local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst into scratch band [990000,990009] on local-lvm … +time=2026-09-24T21:57:00.839+02:00 level=WARN msg="restore-test SKIPPED by the space preflight (R-672) — nothing was created" archive=local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst storage=local-lvm required_bytes=32497541120 avail_bytes=23710110686 reason="not enough space on local-lvm: restoring 21.1 GiB (vzdump log: total bytes written) needs 30.3 GiB free, has 22.1 GiB" + --- restore-test record --- + { + "source_archive": "local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst", + "source_tier": "local", + "scratch_vmid": 0, + "pass": false, + "verified": "", + "error": "skipped: not enough space on local-lvm: restoring 21.1 GiB (vzdump log: total bytes written) needs 30.3 GiB free, has 22.1 GiB", + "tested_at": "2026-09-24T19:57:00Z", + "duration_seconds": 1.127556031, + "skipped": true + } + space preflight: storage=local-lvm required=32497541120 avail=23710110686 +=== selftest=restore-test SKIPPED — skipped: not enough space on local-lvm: restoring 21.1 GiB (vzdump log: total bytes written) needs 30.3 GiB free, has 22.1 GiB === +selftest_rc=4 +pool after: 59.01 2.65 +VMID Status Lock Name +9201 running demo-hp +9202 running demo-hp-scratch diff --git a/documentation/audits/night-2026-09-25/A/A4-hp-backup-run.txt b/documentation/audits/night-2026-09-25/A/A4-hp-backup-run.txt new file mode 100644 index 00000000..c1802dc9 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A4-hp-backup-run.txt @@ -0,0 +1,77 @@ +21:58:58 root before: /dev/mapper/pve-root 41424257024 22494769152 16792281088 58% / +21:58:59 POST /api/guest-backup/trigger -> 200 {"data": {"started": true}, "ok": true} +21:59:21 status 200: {"data": {"backup": null, "error": "", "job_id": "", "phase": "idle"}, "ok": true} | root used 22494769152 +21:59:43 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 22944747520 +22:00:05 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 23394385920 +22:00:28 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 24145051648 +22:00:58 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 24465993728 +22:01:21 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 25055621120 +22:01:43 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 25315524608 +22:02:11 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 26160775168 +22:02:33 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 27060424704 +22:02:57 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 27382145024 +22:03:19 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 27497279488 +22:03:42 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 27847630848 +22:04:04 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 28009426944 +22:04:26 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 28253089792 +22:04:50 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 28856958976 +22:05:12 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 29454897152 +22:05:35 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 29824364544 +22:06:01 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 30421061632 +22:06:23 status 200: {"data": {"backup": null, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "snapshotted"}, "ok": true} | root used 23260041216 +22:06:45 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260098560 +22:07:08 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260106752 +22:07:30 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260110848 +22:07:53 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260106752 +22:08:16 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260114944 +22:08:39 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260119040 +22:09:01 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260119040 +22:09:23 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260123136 +22:09:45 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260127232 +22:10:07 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260131328 +22:10:30 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260139520 +22:10:51 status 200: {"data": {"backup": {"target_id": "local", "vmid": 9201, "archive": "local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst", "mode": "snapshot", "crash_consistent": true, "size_bytes": 8182759056, "success": true, "started_at": "2026-09-24T19:59:25Z", "duration_seconds": 417.875062313}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260147712 +22:11:13 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260176384 +22:11:35 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260188672 +22:11:57 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260192768 +22:12:19 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260196864 +22:12:41 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260192768 +22:13:03 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260200960 +22:13:25 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260200960 +22:13:46 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260205056 +22:14:08 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260209152 +22:14:30 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260213248 +22:14:51 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23260217344 +22:15:13 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268610048 +22:15:35 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268610048 +22:15:56 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268618240 +22:16:18 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268618240 +22:16:40 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268622336 +22:17:01 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268634624 +22:17:23 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268638720 +22:17:45 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268638720 +22:18:07 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268646912 +22:18:28 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268651008 +22:18:50 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268659200 +22:19:12 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268667392 +22:19:33 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268671488 +22:19:55 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268679680 +22:20:17 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268683776 +22:20:38 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268691968 +22:21:00 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268700160 +22:21:22 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268708352 +22:21:43 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268716544 +22:22:05 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268720640 +22:22:26 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268732928 +22:22:48 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268728832 +22:23:09 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268745216 +22:23:31 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268749312 +22:23:53 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268761600 +22:24:14 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268765696 +22:24:36 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268773888 +22:24:58 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268777984 +22:25:19 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268790272 +22:25:41 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268794368 +22:26:02 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268798464 +22:26:24 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268806656 +22:26:45 status 200: {"data": {"backup": {"target_id": "felhom-pbs", "vmid": 9201, "archive": "felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z", "mode": "snapshot", "crash_consistent": true, "size_bytes": 23236593106, "success": true, "started_at": "2026-09-24T20:06:25Z", "duration_seconds": 272.083235108}, "error": "", "job_id": "backup-9201-1790279965190292701", "phase": "done"}, "ok": true} | root used 23268814848 diff --git a/documentation/audits/night-2026-09-25/A/A4-hp-backup-space.txt b/documentation/audits/night-2026-09-25/A/A4-hp-backup-space.txt new file mode 100644 index 00000000..0bbea7d0 --- /dev/null +++ b/documentation/audits/night-2026-09-25/A/A4-hp-backup-space.txt @@ -0,0 +1,40 @@ +# A4 demo-hp whole-box backup space (operator option A) — 2026-09-24T19:57:34+00:00 +== before +/dev/mapper/pve-root 41424257024 34957815808 4329234432 89% / +18: "local_backup_retention": 3, +local:backup/vzdump-lxc-9201-2026_09_21-06_47_30.tar.zst tar.zst backup 6195767336 9201 +local:backup/vzdump-lxc-9201-2026_09_22-06_52_48.tar.zst tar.zst backup 6267269544 9201 +local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst tar.zst backup 7417540996 9201 +/var/lib/vz/dump/vzdump-lxc-9201-2026_09_21-06_47_30.log: 2026-09-21 06:52:59 INFO: Total bytes written: 19160750080 (18GiB, 56MiB/s) +/var/lib/vz/dump/vzdump-lxc-9201-2026_09_22-06_52_48.log: 2026-09-22 06:58:34 INFO: Total bytes written: 19284930560 (18GiB, 54MiB/s) +/var/lib/vz/dump/vzdump-lxc-9201-2026_09_23-06_55_25.log: 2026-09-23 07:02:47 INFO: Total bytes written: 22607360000 (22GiB, 49MiB/s) +== setting after +18c18 +< "local_backup_retention": 3, +--- +> "local_backup_retention": 1, +json ok +active +== prune dry-run (keep-last=1) +NOTE: this is only a preview and might not be what a subsequent +prune call does if backups are removed/added in the meantime. + +Backup Backup-ID Prune-Mark +local:backup/vzdump-lxc-9201-2026_09_21-06_47_30.tar.zst lxc/9201 remove +local:backup/vzdump-lxc-9201-2026_09_22-06_52_48.tar.zst lxc/9201 remove +local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst lxc/9201 keep +== prune (keep-last=1) 2026-09-24T21:57:53+02:00 +removing backup 'local:backup/vzdump-lxc-9201-2026_09_21-06_47_30.tar.zst' +removing backup 'local:backup/vzdump-lxc-9201-2026_09_22-06_52_48.tar.zst' +rc=0 +== after +/dev/mapper/pve-root 41424257024 22494752768 16792297472 58% / +local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst tar.zst backup 7417540996 9201 +== after (poller stopped by PID at 2026-09-24T20:26:57+00:00; both tiers complete per host journal) +Sep 24 22:06:23 demo-hp felhom-agent[756657]: time=2026-09-24T22:06:23.065+02:00 level=INFO msg="backup: completed" vmid=9201 target=local archive=local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst size_bytes=8182759056 uncovered_volumes=2 +Sep 24 22:06:23 demo-hp felhom-agent[756657]: time=2026-09-24T22:06:23.065+02:00 level=INFO msg="local-api: backup job complete" vmid=9201 target=local job=backup-9201-1790279965190292701 archive=local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst +Sep 24 22:10:57 demo-hp felhom-agent[756657]: time=2026-09-24T22:10:57.339+02:00 level=INFO msg="backup: completed" vmid=9201 target=felhom-pbs archive=felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z size_bytes=23236593106 uncovered_volumes=2 +Sep 24 22:10:57 demo-hp felhom-agent[756657]: time=2026-09-24T22:10:57.339+02:00 level=INFO msg="local-api: backup job complete" vmid=9201 target=felhom-pbs job=backup-9201-felhom-pbs-1790280385256228451 archive=felhom-pbs:backup/ct/9201/2026-09-24T20:06:25Z +/dev/mapper/pve-root 41424257024 23268814848 16018235392 60% / +local:backup/vzdump-lxc-9201-2026_09_24-21_59_25.tar.zst tar.zst backup 8182759056 9201 +2 diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D12-default-off.txt b/documentation/audits/night-2026-09-25/B/redproofs/D12-default-off.txt new file mode 100644 index 00000000..b607a3d6 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D12-default-off.txt @@ -0,0 +1,11 @@ +# red-proof D12-default-off: mutate internal/settings/settings.go +# - '\tif s.AppUpdate == nil || s.AppUpdate.Unattended == nil {\n\t\treturn true\n\t}' +# + '\tif s.AppUpdate == nil || s.AppUpdate.Unattended == nil {\n\t\treturn false\n\t}' +# run: go test ./internal/web/ -run ^TestAppUpdateSwitch_DefaultOnAndSaved$ -v -count=1 (rc=1) + +=== RUN TestAppUpdateSwitch_DefaultOnAndSaved + app_update_switch_test.go:23: a box that never saved the switch must show it ON (decision 12) +--- FAIL: TestAppUpdateSwitch_DefaultOnAndSaved (0.07s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/web 0.075s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D13-files-may-change-ignored.txt b/documentation/audits/night-2026-09-25/B/redproofs/D13-files-may-change-ignored.txt new file mode 100644 index 00000000..22669827 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D13-files-may-change-ignored.txt @@ -0,0 +1,19 @@ +# red-proof D13-files-may-change-ignored: mutate internal/stacks/unattended.go +# - 'if e.Marks.FilesMayChange {' +# + 'if false && e.Marks.FilesMayChange {' +# run: go test ./internal/stacks/ -run ^TestLeg_FilesMayChangeNeedsAWholeCopy$ -v -count=1 (rc=1) + +=== RUN TestLeg_FilesMayChangeNeedsAWholeCopy +2026/09/24 21:55:58 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_FilesMayChangeNeedsAWholeCopy1039822279/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:58 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:58 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_FilesMayChangeNeedsAWholeCopy1039822279/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:58 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:58 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_FilesMayChangeNeedsAWholeCopy1039822279/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:58 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:58 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_FilesMayChangeNeedsAWholeCopy1039822279/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:58 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:240: without a whole copy the step must be skipped: ups=[nextcloud:33.0.0-apache] summary=&{Trigger:test StartedAt:2026-09-25 04:15:00 +0000 UTC EndedAt:2026-09-25 04:15:00 +0000 UTC Deadline:2026-09-25 07:30:00 +0000 UTC Enabled:true Done:1 Undone:0 Held:0 Failed:0 Skipped:0 Stopped: Steps:[{App:nextcloud Outcome:done Reason: From:map[web:nextcloud:31.0.14-apache] To:map[web:nextcloud:33.0.0-apache] Seconds:0}]} +--- FAIL: TestLeg_FilesMayChangeNeedsAWholeCopy (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.015s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D13-needs-person-ignored.txt b/documentation/audits/night-2026-09-25/B/redproofs/D13-needs-person-ignored.txt new file mode 100644 index 00000000..e7cd7345 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D13-needs-person-ignored.txt @@ -0,0 +1,19 @@ +# red-proof D13-needs-person-ignored: mutate internal/stacks/unattended.go +# - 'if e.Marks.NeedsPerson != nil && strings.TrimSpace(*e.Marks.NeedsPerson) != "" {' +# + 'if false && e.Marks.NeedsPerson != nil {' +# run: go test ./internal/stacks/ -run ^TestLeg_NeedsPersonIsNeverTaken$ -v -count=1 (rc=1) + +=== RUN TestLeg_NeedsPersonIsNeverTaken +2026/09/24 21:55:55 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NeedsPersonIsNeverTaken894061079/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:55 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:55 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NeedsPersonIsNeverTaken894061079/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:55 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:55 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NeedsPersonIsNeverTaken894061079/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:55 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:55 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NeedsPersonIsNeverTaken894061079/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:55 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:220: a needs_person step was pressed by the leg (ups=[nextcloud:33.0.0-apache]) +--- FAIL: TestLeg_NeedsPersonIsNeverTaken (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.016s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D14-two-steps-a-night.txt b/documentation/audits/night-2026-09-25/B/redproofs/D14-two-steps-a-night.txt new file mode 100644 index 00000000..85c56290 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D14-two-steps-a-night.txt @@ -0,0 +1,23 @@ +# red-proof D14-two-steps-a-night: mutate internal/stacks/unattended.go +# - '\t\t\tsum.add(step)\n' +# + '\t\t\tsum.add(step)\n\t\t\tif outcome == LegOutcomeDone {\n\t\t\t\tif _, r2 := m.legCandidate(ctx, name, o); r2 == "" && m.StartGuardedUpdate(name) == nil {\n\t\t\t\t\tm.legWait(ctx, o, name)\n\t\t\t\t}\n\t\t\t}\n' +# run: go test ./internal/stacks/ -run ^TestLeg_OneStepPerAppPerNight$ -v -count=1 (rc=1) + +=== RUN TestLeg_OneStepPerAppPerNight +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:03 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_OneStepPerAppPerNight1986641340/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:03 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:108: the leg took 2 steps in one night (ups=[nextcloud:33.0.0-apache nextcloud:34.0.1-apache]), want exactly one +--- FAIL: TestLeg_OneStepPerAppPerNight (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.020s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D20-gate-does-not-wait.txt b/documentation/audits/night-2026-09-25/B/redproofs/D20-gate-does-not-wait.txt new file mode 100644 index 00000000..64234e2b --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D20-gate-does-not-wait.txt @@ -0,0 +1,11 @@ +# red-proof D20-gate-does-not-wait: mutate internal/quiesce/quiesce.go +# - 'if wait, why := updateLegDefers(l.updateLegFn, l.now().In(budapestLocation()), window); wait {' +# + 'if wait, why := updateLegDefers(l.updateLegFn, l.now().In(budapestLocation()), window); false && wait {' +# run: go test ./internal/quiesce/ -run ^TestD20_GateWaitsForTheLeg$ -v -count=1 (rc=1) + +=== RUN TestD20_GateWaitsForTheLeg + d20_update_leg_test.go:30: the backup started while the update leg was running (04:45, before W+5h): start=1 stopped=[nextcloud] +--- FAIL: TestD20_GateWaitsForTheLeg (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/quiesce 0.004s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D20-leg-ignores-W5h.txt b/documentation/audits/night-2026-09-25/B/redproofs/D20-leg-ignores-W5h.txt new file mode 100644 index 00000000..6c2cbd13 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D20-leg-ignores-W5h.txt @@ -0,0 +1,19 @@ +# red-proof D20-leg-ignores-W5h: mutate internal/stacks/unattended.go +# - 'case !o.Now().Before(sum.Deadline):' +# + 'case false && !o.Now().Before(sum.Deadline):' +# run: go test ./internal/stacks/ -run ^TestLeg_NoStepAtOrAfterW5h$ -v -count=1 (rc=1) + +=== RUN TestLeg_NoStepAtOrAfterW5h +2026/09/24 21:56:00 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NoStepAtOrAfterW5h3615223022/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:00 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:01 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NoStepAtOrAfterW5h3615223022/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:01 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:01 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NoStepAtOrAfterW5h3615223022/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:01 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:01 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestLeg_NoStepAtOrAfterW5h3615223022/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:01 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:301: a step was started at W+5h: [nextcloud:33.0.0-apache] +--- FAIL: TestLeg_NoStepAtOrAfterW5h (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.014s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/D28-updating-app-sampled.txt b/documentation/audits/night-2026-09-25/B/redproofs/D28-updating-app-sampled.txt new file mode 100644 index 00000000..6d1f7c2a --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/D28-updating-app-sampled.txt @@ -0,0 +1,19 @@ +# red-proof D28-updating-app-sampled: mutate internal/stacks/unhealthy.go +# - 'if st.Deploying || st.Updating || st.HoldReason' +# + 'if st.Deploying || st.HoldReason' +# run: go test ./internal/stacks/ -run ^TestD28_NoCrashLoopStopDuringAnAutomaticStep$ -v -count=1 (rc=1) + +=== RUN TestD28_NoCrashLoopStopDuringAnAutomaticStep +2026/09/24 21:56:04 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestD28_NoCrashLoopStopDuringAnAutomaticStep1296061368/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:04 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:04 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestD28_NoCrashLoopStopDuringAnAutomaticStep1296061368/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:04 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:04 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestD28_NoCrashLoopStopDuringAnAutomaticStep1296061368/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:04 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:56:04 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestD28_NoCrashLoopStopDuringAnAutomaticStep1296061368/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:56:04 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:429: a crash-loop verdict fired during an automatic step: [{Stack:nextcloud Kind:crash_loop Count:6 Window:10m0s}] +--- FAIL: TestD28_NoCrashLoopStopDuringAnAutomaticStep (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.017s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/R678-no-rescan-at-done.txt b/documentation/audits/night-2026-09-25/B/redproofs/R678-no-rescan-at-done.txt new file mode 100644 index 00000000..07f7db31 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/R678-no-rescan-at-done.txt @@ -0,0 +1,13 @@ +# red-proof R678-no-rescan-at-done: mutate internal/stacks/update.go +# - '\tif err := m.ScanStacks(); err != nil {\n\t\tm.logger.Printf("[WARN] [stacks] update %s: rescan after the update failed' +# + '\tif err := error(nil); err != nil {\n\t\tm.logger.Printf("[WARN] [stacks] update %s: rescan after the update failed' +# run: go test ./internal/stacks/ -run ^TestR678_StepsLeftFreshAtDone$ -v -count=1 (rc=1) + +=== RUN TestR678_StepsLeftFreshAtDone +2026/09/24 21:55:51 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR678_StepsLeftFreshAtDone3045165117/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:51 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:153: steps left read 2 right after the step ended done, want 1 +--- FAIL: TestR678_StepsLeftFreshAtDone (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.014s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/R680-no-failed-step-check.txt b/documentation/audits/night-2026-09-25/B/redproofs/R680-no-failed-step-check.txt new file mode 100644 index 00000000..8ec7cabe --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/R680-no-failed-step-check.txt @@ -0,0 +1,41 @@ +# red-proof R680-no-failed-step-check: mutate internal/stacks/unattended.go +# - 'if fs := st.AppConfig.FailedStep; fs != nil && sameRefs(fs.To, e.To)' +# + 'if fs := st.AppConfig.FailedStep; false && fs != nil && sameRefs(fs.To, e.To)' +# run: go test ./internal/stacks/ -run ^TestR680_FailedStepIsNotPressedAgain$ -v -count=1 (rc=1) + +=== RUN TestR680_FailedStepIsNotPressedAgain +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud +2026/09/24 21:55:53 [DEBUG] [stacks] SaveAppConfig: saving /tmp/TestR680_FailedStepIsNotPressedAgain582131369/001/stacks/nextcloud — 0 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 21:55:53 [INFO] [stacks] SaveAppConfig: saved config for nextcloud + unattended_test.go:185: night 2 pressed the step that was undone on night 1 (ups=[nextcloud:33.0.0-apache nextcloud:31.0.14-apache nextcloud:33.0.0-apache nextcloud:31.0.14-apache]) +--- FAIL: TestR680_FailedStepIsNotPressedAgain (0.02s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.031s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/SUMMARY.txt b/documentation/audits/night-2026-09-25/B/redproofs/SUMMARY.txt new file mode 100644 index 00000000..5c864acf --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/SUMMARY.txt @@ -0,0 +1,41 @@ +R678-no-rescan-at-done: test=TestR678_StepsLeftFreshAtDone ran=True FAILED-under-mutation=True rc=1 +R680-no-failed-step-check: test=TestR680_FailedStepIsNotPressedAgain ran=True FAILED-under-mutation=True rc=1 +D13-needs-person-ignored: test=TestLeg_NeedsPersonIsNeverTaken ran=True FAILED-under-mutation=True rc=1 +D13-files-may-change-ignored: test=TestLeg_FilesMayChangeNeedsAWholeCopy ran=True FAILED-under-mutation=True rc=1 +D20-leg-ignores-W5h: test=TestLeg_NoStepAtOrAfterW5h ran=True FAILED-under-mutation=True rc=1 +D14-two-steps-a-night: test=TestLeg_OneStepPerAppPerNight ran=True FAILED-under-mutation=True rc=1 +D28-updating-app-sampled: test=TestD28_NoCrashLoopStopDuringAnAutomaticStep ran=True FAILED-under-mutation=True rc=1 +D20-gate-does-not-wait: test=TestD20_GateWaitsForTheLeg ran=True FAILED-under-mutation=True rc=1 +chain-leg-not-deferred: test=TestChainUpdateLeg_EveryPath ran=True FAILED-under-mutation=True rc=1 +wiring-SetUpdateLeg-dropped: test=TestUpdateLegIsWiredAtStartup ran=True FAILED-under-mutation=True rc=1 +D12-default-off: test=TestAppUpdateSwitch_DefaultOnAndSaved ran=True FAILED-under-mutation=True rc=1 +tree after reverts: + M controller/cmd/controller/main.go + M controller/internal/backup/update_guard.go + M controller/internal/backupwindow/backupwindow.go + M controller/internal/i18n/locales/en.json + M controller/internal/i18n/locales/hu.json + M controller/internal/quiesce/quiesce.go + M controller/internal/report/builder.go + M controller/internal/report/types.go + M controller/internal/settings/settings.go + M controller/internal/stacks/deploy.go + M controller/internal/stacks/ladder.go + M controller/internal/stacks/manager.go + M controller/internal/stacks/undo.go + M controller/internal/stacks/update.go + M controller/internal/web/handlers.go + M controller/internal/web/server.go + M controller/internal/web/templates/app_info.html + M controller/internal/web/templates/settings_system.html + M controller/internal/web/testdata/i18n_parity/settings_system_failed.html + M controller/internal/web/testdata/i18n_parity/settings_system_full.html + M controller/internal/web/testdata/i18n_parity/settings_system_nomemory.html + M controller/internal/web/testdata/i18n_parity/settings_system_pending.html + M controller/scripts/i18n_go_keys.json +?? controller/cmd/controller/update_leg_wiring_test.go +?? controller/internal/quiesce/d20_update_leg_test.go +?? controller/internal/stacks/unattended.go +?? controller/internal/stacks/unattended_test.go +?? controller/internal/web/app_update_switch_test.go + diff --git a/documentation/audits/night-2026-09-25/B/redproofs/chain-leg-not-deferred.txt b/documentation/audits/night-2026-09-25/B/redproofs/chain-leg-not-deferred.txt new file mode 100644 index 00000000..8f6b55b6 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/chain-leg-not-deferred.txt @@ -0,0 +1,11 @@ +# red-proof chain-leg-not-deferred: mutate cmd/controller/main.go +# - '\t\tleg(ctx)\n\t}()\n\treturn offsite(ctx)\n}' +# + '\t}()\n\terr = offsite(ctx)\n\tleg(ctx)\n\treturn err\n}' +# run: go test ./cmd/controller/ -run ^TestChainUpdateLeg_EveryPath$ -v -count=1 (rc=1) + +=== RUN TestChainUpdateLeg_EveryPath + update_leg_wiring_test.go:35: off-site panicked: the leg did not run after the off-site half (ran 0 times) +--- FAIL: TestChainUpdateLeg_EveryPath (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/cmd/controller 0.007s +FAIL diff --git a/documentation/audits/night-2026-09-25/B/redproofs/wiring-SetUpdateLeg-dropped.txt b/documentation/audits/night-2026-09-25/B/redproofs/wiring-SetUpdateLeg-dropped.txt new file mode 100644 index 00000000..8e9491f5 --- /dev/null +++ b/documentation/audits/night-2026-09-25/B/redproofs/wiring-SetUpdateLeg-dropped.txt @@ -0,0 +1,11 @@ +# red-proof wiring-SetUpdateLeg-dropped: mutate cmd/controller/main.go +# - '\tstackMgr.SetUpdateLeg(stacks.UpdateLegOptions{' +# + '\t_ = (stacks.UpdateLegOptions{' +# run: go test ./cmd/controller/ -run ^TestUpdateLegIsWiredAtStartup$ -v -count=1 (rc=1) + +=== RUN TestUpdateLegIsWiredAtStartup + update_leg_wiring_test.go:55: main.go never calls SetUpdateLeg — the automatic update leg is not wired +--- FAIL: TestUpdateLegIsWiredAtStartup (0.01s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/cmd/controller 0.019s +FAIL diff --git a/documentation/audits/night-2026-09-25/C/C0-9202-window-before.txt b/documentation/audits/night-2026-09-25/C/C0-9202-window-before.txt new file mode 100644 index 00000000..8c0df390 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C0-9202-window-before.txt @@ -0,0 +1 @@ +9202 settings.json backup_window_start before Part C: "02:30" (read 2026-09-24 ~22:12 CEST) diff --git a/documentation/audits/night-2026-09-25/C/C0-deploy-9202.txt b/documentation/audits/night-2026-09-25/C/C0-deploy-9202.txt new file mode 100644 index 00000000..6a05b56f --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C0-deploy-9202.txt @@ -0,0 +1,8 @@ +# C0 9202 deploy 0.271.0 — 2026-09-24T20:09:12+00:00 +before: gitea.dooplex.hu/admin/felhom-controller:0.270.0 +gitea.dooplex.hu/admin/felhom-controller:0.271.0 +gitea.dooplex.hu/admin/felhom-controller:0.271.0 Up 26 seconds (healthy) +2026/09/24 20:09:17 undo.go:672: [INFO] [stacks] applied-meta backfill (R-646): recorded 0 []; skipped 0 [] — not current with the catalog, their pinned version's .felhom.yml is no longer on the box +2026/09/24 20:09:17 scheduler.go:132: [INFO] [scheduler] Daily job offbox-backup scheduled for 2026-09-25 04:15 CEST +2026/09/24 20:09:17 scheduler.go:67: [DEBUG] [scheduler] daily job registered: name="offbox-backup" schedule="04:15" nextRun=2026-09-25T04:15:00+02:00 totalJobs=11 +2026/09/24 20:09:17 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 04:15:00 CEST (waiting 6h5m42s) diff --git a/documentation/audits/night-2026-09-25/C/C0-repoint-drill.txt b/documentation/audits/night-2026-09-25/C/C0-repoint-drill.txt new file mode 100644 index 00000000..f2c89786 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C0-repoint-drill.txt @@ -0,0 +1,10 @@ +git: + branch: main + repo_url: https://gitea.dooplex.hu/admin/app-catalog-drill.git + sync_interval: 15m + token: + username: "admin" +hub: +update: + health_timeout: 90s + diff --git a/documentation/audits/night-2026-09-25/C/C1-setup.json b/documentation/audits/night-2026-09-25/C/C1-setup.json new file mode 100644 index 00000000..1c2926d1 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C1-setup.json @@ -0,0 +1,119 @@ +{ + "controller": "gitea.dooplex.hu/admin/felhom-controller:0.271.0", + "apps": { + "wishlist": { + "ladder_len": 1, + "installed_at": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "deploy_ok": true, + "deploy_s": 30, + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "mark": { + "files_may_change": true + }, + "steps_left": null, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "ladder_len": 2, + "installed_at": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "deploy_ok": true, + "deploy_s": 18, + "pinned": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "mark": { + "needs_person": "drill: a person must check the library scan after this step" + }, + "steps_left": null, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "romm": { + "ladder_len": 3, + "installed_at": { + "romm": "rommapp/romm:5.0.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "deploy_ok": true, + "deploy_s": 69, + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": null, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "vikunja": { + "ladder_len": 1, + "installed_at": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "deploy_ok": true, + "deploy_s": 5, + "pinned": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": null, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/C1-setup.log b/documentation/audits/night-2026-09-25/C/C1-setup.log new file mode 100644 index 00000000..1e3c1c55 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C1-setup.log @@ -0,0 +1,22 @@ +22:10:46 drill: wishlist/navidrome/romm/vikunja at the ladder's first from (+ step .felhom.yml files from live cf7cf84) +22:10:51 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +22:11:21 [1] deployed, controller state=running, pinned={'wishlist': 'ghcr.io/cmintey/wishlist:v0.67.1'} +22:11:21 wishlist: deployed=True in 30 s pinned={'wishlist': 'ghcr.io/cmintey/wishlist:v0.67.1'} +22:11:24 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/navidrome'] +22:11:24 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['HDD_PATH'] +22:11:24 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +22:11:39 [1] deployed, controller state=running, pinned={'navidrome': 'deluan/navidrome:0.64.1'} +22:11:39 navidrome: deployed=True in 18 s pinned={'navidrome': 'deluan/navidrome:0.64.1'} +22:11:42 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/romm'] +22:11:42 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['HDD_PATH'] +22:11:42 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +22:12:47 [1] deployed, controller state=running, pinned={'romm': 'rommapp/romm:5.3.1', 'romm-db': 'mariadb:11.8', 'romm-redis': 'redis:7-alpine'} +22:12:47 romm: deployed=True in 69 s pinned={'romm': 'rommapp/romm:5.3.1', 'romm-db': 'mariadb:11.8', 'romm-redis': 'redis:7-alpine'} +22:12:48 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +22:12:53 [1] deployed, controller state=running, pinned={'vikunja': 'vikunja/vikunja:2.6.0'} +22:12:53 vikunja: deployed=True in 5 s pinned={'vikunja': 'vikunja/vikunja:2.6.0'} +22:12:54 drill: heads back; vikunja's head step probes port 8999 (the failing step); navidrome step 1 needs_person; wishlist step files_may_change +22:13:08 wishlist: steps_left=None badge=[{'title': 'Ez az alkalmazás a legfrissebb elérhető változatot futtatja.', 'text': 'Naprakész'}] +22:13:08 navidrome: steps_left=None badge=[{'title': 'Ez az alkalmazás a legfrissebb elérhető változatot futtatja.', 'text': 'Naprakész'}] +22:13:08 romm: steps_left=None badge=[{'title': 'Ez az alkalmazás a legfrissebb elérhető változatot futtatja.', 'text': 'Naprakész'}] +22:13:08 vikunja: steps_left=None badge=[{'title': 'Ez az alkalmazás a legfrissebb elérhető változatot futtatja.', 'text': 'Naprakész'}] diff --git a/documentation/audits/night-2026-09-25/C/C1-state-before-night1.json b/documentation/audits/night-2026-09-25/C/C1-state-before-night1.json new file mode 100644 index 00000000..298aac1c --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/C1-state-before-night1.json @@ -0,0 +1,124 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": 1, + "failed_step": null, + "last_auto_update": null, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "installed": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": 2, + "failed_step": null, + "last_auto_update": null, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.0.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.0.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": 3, + "failed_step": null, + "last_auto_update": null, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "vikunja": { + "state": "unhealthy", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": null, + "last_auto_update": null, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/data-readback.txt b/documentation/audits/night-2026-09-25/C/data-readback.txt new file mode 100644 index 00000000..a47aeab7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/data-readback.txt @@ -0,0 +1,8 @@ +before night 1: seed wishlist=True navidrome=True romm=True vikunja=True +control right after seeding: read wishlist=True navidrome=True romm=True vikunja=True +night1 (after the leg): read wishlist=True navidrome=True romm=True vikunja=True +night2 (after the leg): read wishlist=True navidrome=True romm=True vikunja=True +night3-kill (after the kill): read wishlist=True navidrome=True romm=True vikunja=True +night4-power (after the cut and the resume): read wishlist=True navidrome=True romm=True vikunja=True +night5-off (after the leg): read wishlist=True navidrome=True romm=True vikunja=True +night6-on (after the leg): read wishlist=True navidrome=True romm=True vikunja=True diff --git a/documentation/audits/night-2026-09-25/C/night1/app-lines.txt b/documentation/audits/night-2026-09-25/C/night1/app-lines.txt new file mode 100644 index 00000000..2922c9e9 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night1/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Frissítés elérhető — 1 napja'] lines=[] +[hu] romm: badge=['Frissítés elérhető — 1 napja'] lines=['Automatikus frissítés 2026-09-24 22:31-kor — sikeres.'] +[hu] vikunja: badge=['Frissítés elérhető — 2 napja'] lines=['A(z) vikunja frissítése 2026-09-24 22:33-kor nem sikerült. A doboz automatikusan visszaállította az előző változatot és az adatokat — semmi nem veszett el.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Update available — 1 day ago'] lines=[] +[en] romm: badge=['Update available — 1 day ago'] lines=['Automatic update at 2026-09-24 22:31 — done.'] +[en] vikunja: badge=['Update available — 2 days ago'] lines=['The update of vikunja at 2026-09-24 22:33 did not succeed. The box put back the previous version and its data automatically — nothing was lost.'] diff --git a/documentation/audits/night-2026-09-25/C/night1/controller-full.log b/documentation/audits/night-2026-09-25/C/night1/controller-full.log new file mode 100644 index 00000000..1ebca909 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night1/controller-full.log @@ -0,0 +1,1235 @@ +2026/09/24 20:29:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:29:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:29:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:29:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:29:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:29:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:29:36 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:29:36 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:29:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:29:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:29:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:29:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:29:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:29:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:29:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:29:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:29:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:29:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:29:56 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:29:56 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:30:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:30:06 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:30:06 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:30:16 scheduler.go:346: [INFO] [scheduler] Running job: offsite-credential-retry +2026/09/24 20:30:16 scheduler.go:346: [INFO] [scheduler] Running job: backup-cache +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job backup-cache: execution starting +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:30:16 scheduler.go:346: [INFO] [scheduler] Running job: system-health +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job system-health: execution starting +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:30:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:30:16 scheduler.go:67: [DEBUG] [scheduler] job offsite-credential-retry: execution starting +2026/09/24 20:30:16 scheduler.go:363: [INFO] [scheduler] Job offsite-credential-retry completed (took 0s) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5428MB avail≈20470MB +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24276768 → total=25898MB avail=20470MB used=5428MB (21.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.24 0.94 1.27 9/1481 6980" → 1m=1.24 5m=0.94 15m=1.27 +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 49.9°C (hwmon1) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetInfo done in 81ms — mem=5428MB/25898MB (21.0%), rootDisk=23.2GB/68.4GB (34.0%), load=1.24/0.94/1.27, temp=49.9°C (hwmon1), cpu=19.7% +2026/09/24 20:30:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5436MB avail≈20462MB +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24267768 → total=25898MB avail=20462MB used=5436MB (21.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.24 0.94 1.27 10/1485 7010" → 1m=1.24 5m=0.94 15m=1.27 +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 56.0°C (hwmon2) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetInfo done in 72ms — mem=5436MB/25898MB (21.0%), rootDisk=23.2GB/68.4GB (34.0%), load=1.24/0.94/1.27, temp=56.0°C (hwmon2), cpu=19.7% +2026/09/24 20:30:16 healthcheck.go:86: [DEBUG] [monitor] Raw values: disk=34.0%, hdd=7.6% (configured=true), mem=21.0% (5436MB/25898MB), cpu=19.7%, temp=56.0°C (hwmon2) +2026/09/24 20:30:16 healthcheck.go:118: [DEBUG] [monitor] SSD disk: OK (34%) +2026/09/24 20:30:16 healthcheck.go:151: [DEBUG] [monitor] Memory: OK (21%) +2026/09/24 20:30:16 healthcheck.go:169: [DEBUG] [monitor] CPU: OK (20%) +2026/09/24 20:30:16 healthcheck.go:187: [DEBUG] [monitor] Temperature: OK (56°C) +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:30:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:30:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:30:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:30:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 113ms) +2026/09/24 20:30:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:30:16 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/navidrome → [navidrome] +2026/09/24 20:30:16 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx → [paperless-ngx] +2026/09/24 20:30:16 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/sys_drive → [privatebin, vikunja, wishlist] +2026/09/24 20:30:16 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/romm → [romm] +2026/09/24 20:30:16 backup.go:1119: [INFO] [backup] Found 1 DB dump files across drives +2026/09/24 20:30:16 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:30:16 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:30:16 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:30:16 healthcheck.go:201: [DEBUG] [monitor] Docker daemon: OK +2026/09/24 20:30:16 healthcheck.go:209: [DEBUG] [monitor] Checking 1 protected containers: [traefik] +2026/09/24 20:30:16 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: fddb22967556 vikunja vikunja vikunja/vikunja:2.3.0 +655e70b891f9 romm romm rommapp/romm:5.0.0 +14256bf0e07e romm-db romm mariadb:11.4 +bbc8b1d0b25a romm-redis romm redis:7-alpine +257abdec6373 navidrome navidrome deluan/navidrome:0.63.2 +f0dcacf87ee1 wishlist wishlist ghcr.io/cmintey/wishlist:v0.66.0 +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.0.0, not a database) +2026/09/24 20:30:16 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=14256bf0e07e) +2026/09/24 20:30:16 healthcheck.go:219: [DEBUG] [monitor] All protected containers running +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/romm" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/navidrome" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 healthcheck.go:238: [INFO] [monitor] Health check: status=warn +2026/09/24 20:30:16 healthcheck.go:242: [DEBUG] [monitor] Final status: warn (issues=0, warnings=4, info=5) +2026/09/24 20:30:16 notifier.go:1259: [DEBUG] notifier disabled: dropped event health_change (severity warning) +2026/09/24 20:30:16 scheduler.go:363: [INFO] [scheduler] Job system-health completed (took 161ms) +2026/09/24 20:30:16 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.63.2, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.66.0, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:30:16 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:30:16 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:30:16 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:30:16 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:30:16 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:30:16 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 2 HDD mounts for /opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 1 HDD mounts for /opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:30:16 [INFO] [backup] Discovered app data: 6 apps +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/navidrome" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/romm" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:30:16 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:30:16 backup.go:1189: [INFO] [backup] Backup status cache refreshed +2026/09/24 20:30:16 scheduler.go:363: [INFO] [scheduler] Job backup-cache completed (took 213ms) +2026/09/24 20:30:16 infra.go:67: [INFO] [infra] cloudflared skipped — no cf_tunnel_token configured (LAN-only node) +2026/09/24 20:30:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:30:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:30:36 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:30:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:36 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:30:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:30:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:30:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 3 targets (3 skipped not due, 0 skipped no container) +2026/09/24 20:30:46 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:30:46 healthprobe.go:181: [DEBUG] Health probe privatebin: HTTP GET :8080/ → 200 (6ms) +2026/09/24 20:30:46 healthprobe.go:181: [DEBUG] Health probe paperless-ngx: HTTP GET :8000/ → 302 (7ms) +2026/09/24 20:30:46 healthprobe.go:159: [WARN] Health probes: 2 ok, 1 unhealthy (of 3 probed) +2026/09/24 20:30:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:30:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:30:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:30:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:30:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:30:56 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:30:56 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:31:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:31:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:31:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:31:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:46, no step starts at or after 01:46 +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:31:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:31:00 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:31:00 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:31:00 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:31:00 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:00 unattended.go:300: [INFO] [update-leg] navidrome: skipped — needs_person +2026/09/24 20:31:00 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-2761957447: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-2761957447/docker-compose.yml: no such file or directory +2026/09/24 20:31:00 [ERROR] [stacks] .felhom.yml data_paths rejected in /tmp/felhom-step-meta-2761957447: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-2761957447/docker-compose.yml: no such file or directory +2026/09/24 20:31:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5601MB avail≈20297MB +2026/09/24 20:31:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24070480 → total=25898MB avail=20297MB used=5601MB (21.6%) +2026/09/24 20:31:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=750MB, new_req=300MB, remaining=24464MB +2026/09/24 20:31:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:31:00 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:31:00 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.0.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine +2026/09/24 20:31:00 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:31:00 update.go:776: [INFO] [stacks] update romm: ladder — step 1 of 3: romm=rommapp/romm:5.0.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine, from 7f9c6a74b5891f8e.yml (probe from 7f9c6a74b5891f8e.felhom.yml) +2026/09/24 20:31:00 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:31:00 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (11m0s old, limit 24h0m0s) +2026/09/24 20:31:00 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:31:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:31:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: fddb22967556 vikunja vikunja vikunja/vikunja:2.3.0 +655e70b891f9 romm romm rommapp/romm:5.0.0 +14256bf0e07e romm-db romm mariadb:11.4 +bbc8b1d0b25a romm-redis romm redis:7-alpine +257abdec6373 navidrome navidrome deluan/navidrome:0.63.2 +f0dcacf87ee1 wishlist wishlist ghcr.io/cmintey/wishlist:v0.66.0 +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.0.0, not a database) +2026/09/24 20:31:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=14256bf0e07e) +2026/09/24 20:31:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.63.2, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.66.0, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:31:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:31:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:31:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:31:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:31:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:31:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:31:00 dbdump.go:266: [DEBUG] DumpOne: starting dump for container=romm-db, stack=romm, dbType=mariadb, dest=/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203100Z-romm-mariadb.sql +2026/09/24 20:31:00 dbdump.go:321: [DEBUG] DumpOne: mariadb-dump command: docker exec 14256bf0e07e mariadb-dump -u root -p*** --single-transaction --routines --triggers romm +2026/09/24 20:31:00 dbdump.go:405: [DEBUG] DumpOne: completed romm-db → pre-restore-20260924T203100Z-romm-mariadb.sql (size=61.3 KB, valid=true, tables=24, duration=267ms) +2026/09/24 20:31:00 dbdump.go:410: [INFO] [backup] DB dump: romm-db → pre-restore-20260924T203100Z-romm-mariadb.sql (61.3 KB, 267ms, 24 tables) +2026/09/24 20:31:00 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T203100Z-romm-mariadb.sql (61.3 KB) +2026/09/24 20:31:00 update_guard.go:478: [INFO] [backup] update safety dump for romm: 1 file(s) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203100Z-romm-mariadb.sql] +2026/09/24 20:31:00 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203100Z-romm-mariadb.sql] +2026/09/24 20:31:01 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:31:01 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 178.8 MiB +2026/09/24 20:31:01 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:31:01 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:31:01 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:31:01 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/steps/7f9c6a74b5891f8e.yml (romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine) +2026/09/24 20:31:01 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:31:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, DOMAIN, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:31:01 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/romm) +2026/09/24 20:31:02 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:31:02 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:02 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, DOMAIN, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:31:02 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/romm) +2026/09/24 20:31:05 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 3.3s) +2026/09/24 20:31:05 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:06 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_config → romm_romm_config.pre-update-20260924T203105Z in 479ms +2026/09/24 20:31:06 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:31:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:31:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:07 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_db_data → romm_romm_db_data.pre-update-20260924T203105Z in 803ms +2026/09/24 20:31:07 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:07 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_redis_data → romm_romm_redis_data.pre-update-20260924T203105Z in 463ms +2026/09/24 20:31:07 update.go:1285: [INFO] [stacks] update romm: phase starting +2026/09/24 20:31:07 manager.go:418: [INFO] [stacks] userdata belt: drive /mnt/felhom-drives/scratch_hdd/userdata/romm not mounted — skipping ensure (held by drive gate) +2026/09/24 20:31:07 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, DOMAIN, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:31:07 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/romm) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5386MB avail≈20512MB +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24325204 → total=25898MB avail=20512MB used=5386MB (20.8%) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.4GB avail=41.5GB (34.2%) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.49 1.05 1.29 7/1473 7590" → 1m=1.49 5m=1.05 15m=1.29 +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 57.5°C (hwmon2) +2026/09/24 20:31:16 info.go:13: [DEBUG] [system] GetInfo done in 60ms — mem=5386MB/25898MB (20.8%), rootDisk=23.4GB/68.4GB (34.2%), load=1.49/1.05/1.29, temp=57.5°C (hwmon2), cpu=26.6% +2026/09/24 20:31:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:31:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:31:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:31:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:16 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:31:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:16 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:31:16 main.go:2017: [INFO] [deadapp] check alive: 40 scans since boot, 6 deployed app(s) evaluated, 0 currently down +2026/09/24 20:31:18 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 11.3s) +2026/09/24 20:31:18 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:31:18 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-3369487648: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3369487648/docker-compose.yml: no such file or directory +2026/09/24 20:31:18 [ERROR] [stacks] .felhom.yml data_paths rejected in /tmp/felhom-step-meta-3369487648: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3369487648/docker-compose.yml: no such file or directory +2026/09/24 20:31:18 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:31:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:31:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:31:36 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:31:36 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:31:38 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:44 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:31:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:31:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 10s ago, effective interval 10s, healthy=false +2026/09/24 20:31:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:31:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:49 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:54 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:31:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:31:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:31:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:31:56 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:31:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:56 healthprobe.go:159: [WARN] Health probes: 0 ok, 1 unhealthy (of 1 probed) +2026/09/24 20:31:59 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:59 healthprobe.go:181: [DEBUG] Health probe romm: HTTP GET :8080/ → 200 (1ms) +2026/09/24 20:31:59 update.go:942: [INFO] [stacks] update romm: healthy after 40s (the app's health check passed) +2026/09/24 20:31:59 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:31:59 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:31:59 installed.go:420: [INFO] [stacks] installed-images romm: recorded 3 service(s) (romm=rommapp/romm:5.3.0 (sha256:dc586cb3a2c7…), romm-db=mariadb:11.4 (sha256:70cc072b29b4…), romm-redis=redis:7-alpine (sha256:858f009f9709…)) +2026/09/24 20:31:59 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:31:59 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:31:59 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:31:59 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:31:59 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:31:59 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:31:59 update.go:958: [INFO] [stacks] update romm: DONE in 59s +2026/09/24 20:32:00 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:32:00 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:32:00 unattended.go:327: [INFO] [update-leg] romm: step ended done after 60.1 s +2026/09/24 20:32:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5392MB avail≈20506MB +2026/09/24 20:32:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24318992 → total=25898MB avail=20506MB used=5392MB (20.8%) +2026/09/24 20:32:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=1000MB, new_req=50MB, remaining=24464MB +2026/09/24 20:32:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:32:00 update.go:526: [INFO] [stacks] update vikunja: accepted — guarded update started +2026/09/24 20:32:00 unattended.go:320: [INFO] [update-leg] vikunja: step pressed vikunja=vikunja/vikunja:2.3.0 → vikunja=vikunja/vikunja:2.6.0 +2026/09/24 20:32:00 update.go:1285: [INFO] [stacks] update vikunja: phase checking +2026/09/24 20:32:00 update.go:776: [INFO] [stacks] update vikunja: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:32:00 update_guard.go:248: [DEBUG] [backup] update precondition for vikunja: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:32:00 update.go:789: [INFO] [stacks] update vikunja: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (12m0s old, limit 24h0m0s) +2026/09/24 20:32:00 update.go:1285: [INFO] [stacks] update vikunja: phase safety-dump +2026/09/24 20:32:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:32:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: e237099ca189 romm romm rommapp/romm:5.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +fddb22967556 vikunja vikunja vikunja/vikunja:2.3.0 +257abdec6373 navidrome navidrome deluan/navidrome:0.63.2 +f0dcacf87ee1 wishlist wishlist ghcr.io/cmintey/wishlist:v0.66.0 +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.0, not a database) +2026/09/24 20:32:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:32:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.63.2, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.66.0, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:32:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:32:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:32:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:32:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:32:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:32:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:32:00 update_guard.go:476: [INFO] [backup] update safety dump for vikunja: the app has no database — nothing to copy (no-op) +2026/09/24 20:32:00 update.go:821: [INFO] [stacks] update vikunja: safety dump done (0 file(s)) [] +2026/09/24 20:32:01 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:32:01 undo.go:285: [INFO] [stacks] update vikunja: the undo copy will hold 2 named volume(s), 2.2 MiB +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase pinning +2026/09/24 20:32:01 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:32:01 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:32:01 pin.go:373: [INFO] [stacks] update vikunja: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/vikunja/docker-compose.yml (vikunja=vikunja/vikunja:2.6.0) +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase pulling +2026/09/24 20:32:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:32:01 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/vikunja) +2026/09/24 20:32:01 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.5s) +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:32:01 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/vikunja) +2026/09/24 20:32:02 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 0.3s) +2026/09/24 20:32:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_data → vikunja_vikunja_data.pre-update-20260924T203202Z in 481ms +2026/09/24 20:32:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:03 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_db → vikunja_vikunja_db.pre-update-20260924T203202Z in 410ms +2026/09/24 20:32:03 update.go:1285: [INFO] [stacks] update vikunja: phase starting +2026/09/24 20:32:03 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:32:03 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/vikunja) +2026/09/24 20:32:03 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:32:03 update.go:1285: [INFO] [stacks] update vikunja: phase verifying +2026/09/24 20:32:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:03 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 7s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:32:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:08 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:13 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:13 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5563MB avail≈20335MB +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24116292 → total=25898MB avail=20335MB used=5563MB (21.5%) +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="2.44 1.42 1.40 6/1483 8193" → 1m=2.44 5m=1.42 15m=1.40 +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.0°C (hwmon2) +2026/09/24 20:32:16 info.go:13: [DEBUG] [system] GetInfo done in 69ms — mem=5563MB/25898MB (21.5%), rootDisk=23.2GB/68.4GB (34.0%), load=2.44/1.42/1.40, temp=54.0°C (hwmon2), cpu=9.3% +2026/09/24 20:32:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:32:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:32:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:32:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:32:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 17s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:32:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:32:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:32:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:32:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:32:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 90ms) +2026/09/24 20:32:18 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:18 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:23 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 5m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 27s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:32:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:28 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:33 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 37s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:32:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:36 healthprobe.go:181: [DEBUG] Health probe wishlist: HTTP GET :3000/ → 307 (4ms) +2026/09/24 20:32:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:36 healthprobe.go:161: [INFO] Health probes: 1 ok (of 1 probed) +2026/09/24 20:32:38 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:38 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:43 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:32:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:32:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 47s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:32:46 healthprobe.go:181: [DEBUG] Health probe navidrome: API GET :4533/ping → 200 (1ms) +2026/09/24 20:32:46 healthprobe.go:161: [INFO] Health probes: 1 ok (of 1 probed) +2026/09/24 20:32:48 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:48 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:53 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:32:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:32:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:32:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 57s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:32:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:32:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:32:58 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:32:58 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:03 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m7s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3s ago, effective interval 10s, healthy=false +2026/09/24 20:33:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:33:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:08 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:13 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:13 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:33:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2s ago, effective interval 10s, healthy=false +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m17s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:33:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:33:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5605MB avail≈20293MB +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24066396 → total=25898MB avail=20293MB used=5605MB (21.6%) +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.71 1.38 1.38 8/1488 8494" → 1m=1.71 5m=1.38 15m=1.38 +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.6°C (hwmon2) +2026/09/24 20:33:16 info.go:13: [DEBUG] [system] GetInfo done in 74ms — mem=5605MB/25898MB (21.6%), rootDisk=23.2GB/68.4GB (34.0%), load=1.71/1.38/1.38, temp=54.6°C (hwmon2), cpu=3.1% +2026/09/24 20:33:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:18 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:18 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:23 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2s ago, effective interval 10s, healthy=false +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m27s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:33:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:28 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:33 healthprobe.go:190: [WARN] Health probe vikunja: API GET :8999/api/v1/info → Get "http://vikunja:8999/api/v1/info": dial tcp 172.18.0.9:8999: connect: connection refused +2026/09/24 20:33:33 update.go:991: [ERROR] [stacks] update vikunja FAILED after the new version was started: not healthy: not healthy within 1m30s (last: health check failing) +2026/09/24 20:33:33 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:33:33 manager.go:1473: [DEBUG] Running: docker compose logs --no-color --tail 400 (in /opt/docker/stacks/vikunja) +2026/09/24 20:33:34 manager.go:1493: [DEBUG] Command completed: docker compose logs --no-color --tail 400 (took 0.1s) +2026/09/24 20:33:34 update.go:983: [INFO] [stacks] update vikunja: kept 1299 bytes of the app's own log at /opt/docker/stacks/vikunja/hold-logs/20260924T203333Z/compose-logs.txt before stopping it (R-621) +2026/09/24 20:33:34 update.go:1285: [INFO] [stacks] update vikunja: phase undoing +2026/09/24 20:33:34 undo.go:513: [WARN] [stacks] update vikunja: UNDO — putting back the previous version and its 2 volume copy(ies) (reason: not healthy: not healthy within 1m30s (last: health check failing)) +2026/09/24 20:33:34 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, VIKUNJA_SERVICE_JWTSECRET, DOMAIN, SUBDOMAIN, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:33:34 manager.go:1473: [DEBUG] Running: docker compose down (in /opt/docker/stacks/vikunja) +2026/09/24 20:33:34 manager.go:1493: [DEBUG] Command completed: docker compose down (took 0.3s) +2026/09/24 20:33:35 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:33:35 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:33:35 pin.go:93: [INFO] [stacks] pin vikunja: vikunja=vikunja/vikunja:2.3.0 +2026/09/24 20:33:35 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:33:35 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/vikunja) +2026/09/24 20:33:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2s ago, effective interval 10s, healthy=false +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m37s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:33:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:36 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:33:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:36 healthprobe.go:190: [WARN] Health probe vikunja: API GET :3456/api/v1/info → Get "http://vikunja:3456/api/v1/info": dial tcp 172.18.0.9:3456: connect: connection refused +2026/09/24 20:33:41 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:41 healthprobe.go:181: [DEBUG] Health probe vikunja: API GET :3456/api/v1/info → 200 (1ms) +2026/09/24 20:33:41 installed.go:409: [DEBUG] [stacks] installed-images vikunja: unchanged (1 service(s)) — app.yaml not rewritten +2026/09/24 20:33:41 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:33:41 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:33:41 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:33:41 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:33:41 unattended.go:490: [INFO] [stacks] update vikunja: step vikunja=vikunja/vikunja:2.6.0 recorded as undone — the automatic leg will not press it again until the catalog's ladder changes (ladder 5c632840c2e89eff) +2026/09/24 20:33:41 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:33:41 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:33:41 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:33:41 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:33:41 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:33:41 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:41 undo.go:637: [INFO] [stacks] update vikunja: event app_update_undone +2026/09/24 20:33:41 notifier.go:1255: [WARN] notifier disabled (no hub configured): DROPPED event app_update_undone (severity warning) — further app_update_undone events are logged at DEBUG only +2026/09/24 20:33:41 undo.go:565: [INFO] [stacks] update vikunja: UNDONE in 8s — the previous version is running on the data from before the update (the app's health check passed) +2026/09/24 20:33:45 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:33:45 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:33:45 unattended.go:327: [INFO] [update-leg] vikunja: step ended undone after 105.2 s +2026/09/24 20:33:45 update_guard.go:248: [DEBUG] [backup] update precondition for wishlist: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:33:45 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5579MB avail≈20319MB +2026/09/24 20:33:45 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24097420 → total=25898MB avail=20319MB used=5579MB (21.5%) +2026/09/24 20:33:45 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=930MB, new_req=120MB, remaining=24464MB +2026/09/24 20:33:45 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:33:45 update.go:526: [INFO] [stacks] update wishlist: accepted — guarded update started +2026/09/24 20:33:45 unattended.go:320: [INFO] [update-leg] wishlist: step pressed wishlist=ghcr.io/cmintey/wishlist:v0.66.0 → wishlist=ghcr.io/cmintey/wishlist:v0.67.1 +2026/09/24 20:33:45 update.go:1285: [INFO] [stacks] update wishlist: phase checking +2026/09/24 20:33:45 update.go:776: [INFO] [stacks] update wishlist: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:33:45 update_guard.go:248: [DEBUG] [backup] update precondition for wishlist: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:33:45 update.go:789: [INFO] [stacks] update wishlist: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (13m0s old, limit 24h0m0s) +2026/09/24 20:33:45 update.go:1285: [INFO] [stacks] update wishlist: phase safety-dump +2026/09/24 20:33:45 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:33:45 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +e237099ca189 romm romm rommapp/romm:5.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +257abdec6373 navidrome navidrome deluan/navidrome:0.63.2 +f0dcacf87ee1 wishlist wishlist ghcr.io/cmintey/wishlist:v0.66.0 +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.0, not a database) +2026/09/24 20:33:45 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:33:45 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.63.2, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.66.0, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:33:45 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:33:45 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:33:45 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:33:45 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:33:45 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:33:45 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:33:45 update_guard.go:476: [INFO] [backup] update safety dump for wishlist: the app has no database — nothing to copy (no-op) +2026/09/24 20:33:45 update.go:821: [INFO] [stacks] update wishlist: safety dump done (0 file(s)) [] +2026/09/24 20:33:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m47s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 5s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:33:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:33:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:33:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:46 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:33:46 undo.go:285: [INFO] [stacks] update wishlist: the undo copy will hold 2 named volume(s), 0.4 MiB +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase pinning +2026/09/24 20:33:46 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/wishlist — 2 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:33:46 [INFO] [stacks] SaveAppConfig: saved config for wishlist +2026/09/24 20:33:46 pin.go:373: [INFO] [stacks] update wishlist: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/wishlist/docker-compose.yml (wishlist=ghcr.io/cmintey/wishlist:v0.67.1) +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase pulling +2026/09/24 20:33:46 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, IMPORT_PATH] (8 app + 0 system) +2026/09/24 20:33:46 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/wishlist) +2026/09/24 20:33:46 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.4s) +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:46 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, IMPORT_PATH] (8 app + 0 system) +2026/09/24 20:33:46 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/wishlist) +2026/09/24 20:33:47 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 0.3s) +2026/09/24 20:33:47 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:47 undo.go:435: [INFO] [stacks] update wishlist: copied wishlist_wishlist_data → wishlist_wishlist_data.pre-update-20260924T203347Z in 485ms +2026/09/24 20:33:47 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:48 undo.go:435: [INFO] [stacks] update wishlist: copied wishlist_wishlist_uploads → wishlist_wishlist_uploads.pre-update-20260924T203347Z in 438ms +2026/09/24 20:33:48 update.go:1285: [INFO] [stacks] update wishlist: phase starting +2026/09/24 20:33:48 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, IMPORT_PATH] (8 app + 0 system) +2026/09/24 20:33:48 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/wishlist) +2026/09/24 20:33:48 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:33:48 update.go:1285: [INFO] [stacks] update wishlist: phase verifying +2026/09/24 20:33:48 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:33:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 15s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m57s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:33:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:33:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:33:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:33:58 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:03 healthprobe.go:181: [DEBUG] Health probe wishlist: HTTP GET :3000/ → 307 (6ms) +2026/09/24 20:34:03 update.go:942: [INFO] [stacks] update wishlist: healthy after 15s (the app's health check passed) +2026/09/24 20:34:03 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/wishlist — 2 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:34:03 [INFO] [stacks] SaveAppConfig: saved config for wishlist +2026/09/24 20:34:03 installed.go:420: [INFO] [stacks] installed-images wishlist: recorded 1 service(s) (wishlist=ghcr.io/cmintey/wishlist:v0.67.1 (sha256:4b89bfb5b57f…)) +2026/09/24 20:34:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:34:03 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:34:03 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:34:03 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:34:03 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:34:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:03 update.go:958: [INFO] [stacks] update wishlist: DONE in 18s +2026/09/24 20:34:05 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/wishlist — 2 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:34:05 [INFO] [stacks] SaveAppConfig: saved config for wishlist +2026/09/24 20:34:05 unattended.go:327: [INFO] [update-leg] wishlist: step ended done after 20.1 s +2026/09/24 20:34:05 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=2 undone=1 held=0 failed=0 skipped=1 in 3m6s [skipped: navidrome=needs_person] +2026/09/24 20:34:05 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 3m5.528s) +2026/09/24 20:34:05 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:31:00 CEST (waiting 23h56m54s) +2026/09/24 20:34:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m7s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 25s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:34:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:34:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:34:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:34:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:34:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:34:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:34:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:34:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5477MB avail≈20421MB +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24218252 → total=25898MB avail=20421MB used=5477MB (21.1%) +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.51 1.41 1.40 10/1488 9351" → 1m=1.51 5m=1.41 15m=1.40 +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 53.5°C (hwmon2) +2026/09/24 20:34:16 info.go:13: [DEBUG] [system] GetInfo done in 77ms — mem=5477MB/25898MB (21.1%), rootDisk=23.2GB/68.4GB (34.0%), load=1.51/1.41/1.40, temp=53.5°C (hwmon2), cpu=2.8% +2026/09/24 20:34:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m17s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 35s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 13s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:34:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:34:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:34:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:34:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:34:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:34:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 99ms) +2026/09/24 20:34:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:34:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m27s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 45s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 23s ago, effective interval 5m0s, healthy=true +2026/09/24 20:34:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:34:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks diff --git a/documentation/audits/night-2026-09-25/C/night1/controller-night.log b/documentation/audits/night-2026-09-25/C/night1/controller-night.log new file mode 100644 index 00000000..85a6ca9f --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night1/controller-night.log @@ -0,0 +1,80 @@ +2026/09/24 20:31:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:31:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:31:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:31:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:46, no step starts at or after 01:46 +2026/09/24 20:31:00 unattended.go:300: [INFO] [update-leg] navidrome: skipped — needs_person +2026/09/24 20:31:00 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:31:00 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.0.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine +2026/09/24 20:31:00 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:31:00 update.go:776: [INFO] [stacks] update romm: ladder — step 1 of 3: romm=rommapp/romm:5.0.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine, from 7f9c6a74b5891f8e.yml (probe from 7f9c6a74b5891f8e.felhom.yml) +2026/09/24 20:31:00 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (11m0s old, limit 24h0m0s) +2026/09/24 20:31:00 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:31:00 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T203100Z-romm-mariadb.sql (61.3 KB) +2026/09/24 20:31:00 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203100Z-romm-mariadb.sql] +2026/09/24 20:31:01 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 178.8 MiB +2026/09/24 20:31:01 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:31:01 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/steps/7f9c6a74b5891f8e.yml (romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine) +2026/09/24 20:31:01 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:31:02 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:05 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:06 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_config → romm_romm_config.pre-update-20260924T203105Z in 479ms +2026/09/24 20:31:06 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:07 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_db_data → romm_romm_db_data.pre-update-20260924T203105Z in 803ms +2026/09/24 20:31:07 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:31:07 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_redis_data → romm_romm_redis_data.pre-update-20260924T203105Z in 463ms +2026/09/24 20:31:07 update.go:1285: [INFO] [stacks] update romm: phase starting +2026/09/24 20:31:18 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:31:59 update.go:942: [INFO] [stacks] update romm: healthy after 40s (the app's health check passed) +2026/09/24 20:31:59 update.go:958: [INFO] [stacks] update romm: DONE in 59s +2026/09/24 20:32:00 unattended.go:327: [INFO] [update-leg] romm: step ended done after 60.1 s +2026/09/24 20:32:00 update.go:526: [INFO] [stacks] update vikunja: accepted — guarded update started +2026/09/24 20:32:00 unattended.go:320: [INFO] [update-leg] vikunja: step pressed vikunja=vikunja/vikunja:2.3.0 → vikunja=vikunja/vikunja:2.6.0 +2026/09/24 20:32:00 update.go:1285: [INFO] [stacks] update vikunja: phase checking +2026/09/24 20:32:00 update.go:776: [INFO] [stacks] update vikunja: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:32:00 update.go:789: [INFO] [stacks] update vikunja: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (12m0s old, limit 24h0m0s) +2026/09/24 20:32:00 update.go:1285: [INFO] [stacks] update vikunja: phase safety-dump +2026/09/24 20:32:00 update.go:821: [INFO] [stacks] update vikunja: safety dump done (0 file(s)) [] +2026/09/24 20:32:01 undo.go:285: [INFO] [stacks] update vikunja: the undo copy will hold 2 named volume(s), 2.2 MiB +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase pinning +2026/09/24 20:32:01 pin.go:373: [INFO] [stacks] update vikunja: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/vikunja/docker-compose.yml (vikunja=vikunja/vikunja:2.6.0) +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase pulling +2026/09/24 20:32:01 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_data → vikunja_vikunja_data.pre-update-20260924T203202Z in 481ms +2026/09/24 20:32:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:32:03 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_db → vikunja_vikunja_db.pre-update-20260924T203202Z in 410ms +2026/09/24 20:32:03 update.go:1285: [INFO] [stacks] update vikunja: phase starting +2026/09/24 20:32:03 update.go:1285: [INFO] [stacks] update vikunja: phase verifying +2026/09/24 20:33:33 update.go:991: [ERROR] [stacks] update vikunja FAILED after the new version was started: not healthy: not healthy within 1m30s (last: health check failing) +2026/09/24 20:33:34 update.go:983: [INFO] [stacks] update vikunja: kept 1299 bytes of the app's own log at /opt/docker/stacks/vikunja/hold-logs/20260924T203333Z/compose-logs.txt before stopping it (R-621) +2026/09/24 20:33:34 update.go:1285: [INFO] [stacks] update vikunja: phase undoing +2026/09/24 20:33:34 undo.go:513: [WARN] [stacks] update vikunja: UNDO — putting back the previous version and its 2 volume copy(ies) (reason: not healthy: not healthy within 1m30s (last: health check failing)) +2026/09/24 20:33:41 unattended.go:490: [INFO] [stacks] update vikunja: step vikunja=vikunja/vikunja:2.6.0 recorded as undone — the automatic leg will not press it again until the catalog's ladder changes (ladder 5c632840c2e89eff) +2026/09/24 20:33:41 undo.go:637: [INFO] [stacks] update vikunja: event app_update_undone +2026/09/24 20:33:41 notifier.go:1255: [WARN] notifier disabled (no hub configured): DROPPED event app_update_undone (severity warning) — further app_update_undone events are logged at DEBUG only +2026/09/24 20:33:41 undo.go:565: [INFO] [stacks] update vikunja: UNDONE in 8s — the previous version is running on the data from before the update (the app's health check passed) +2026/09/24 20:33:45 unattended.go:327: [INFO] [update-leg] vikunja: step ended undone after 105.2 s +2026/09/24 20:33:45 update.go:526: [INFO] [stacks] update wishlist: accepted — guarded update started +2026/09/24 20:33:45 unattended.go:320: [INFO] [update-leg] wishlist: step pressed wishlist=ghcr.io/cmintey/wishlist:v0.66.0 → wishlist=ghcr.io/cmintey/wishlist:v0.67.1 +2026/09/24 20:33:45 update.go:1285: [INFO] [stacks] update wishlist: phase checking +2026/09/24 20:33:45 update.go:776: [INFO] [stacks] update wishlist: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:33:45 update.go:789: [INFO] [stacks] update wishlist: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (13m0s old, limit 24h0m0s) +2026/09/24 20:33:45 update.go:1285: [INFO] [stacks] update wishlist: phase safety-dump +2026/09/24 20:33:45 update.go:821: [INFO] [stacks] update wishlist: safety dump done (0 file(s)) [] +2026/09/24 20:33:46 undo.go:285: [INFO] [stacks] update wishlist: the undo copy will hold 2 named volume(s), 0.4 MiB +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase pinning +2026/09/24 20:33:46 pin.go:373: [INFO] [stacks] update wishlist: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/wishlist/docker-compose.yml (wishlist=ghcr.io/cmintey/wishlist:v0.67.1) +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase pulling +2026/09/24 20:33:46 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:47 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:47 undo.go:435: [INFO] [stacks] update wishlist: copied wishlist_wishlist_data → wishlist_wishlist_data.pre-update-20260924T203347Z in 485ms +2026/09/24 20:33:47 update.go:1285: [INFO] [stacks] update wishlist: phase copying +2026/09/24 20:33:48 undo.go:435: [INFO] [stacks] update wishlist: copied wishlist_wishlist_uploads → wishlist_wishlist_uploads.pre-update-20260924T203347Z in 438ms +2026/09/24 20:33:48 update.go:1285: [INFO] [stacks] update wishlist: phase starting +2026/09/24 20:33:48 update.go:1285: [INFO] [stacks] update wishlist: phase verifying +2026/09/24 20:34:03 update.go:942: [INFO] [stacks] update wishlist: healthy after 15s (the app's health check passed) +2026/09/24 20:34:03 update.go:958: [INFO] [stacks] update wishlist: DONE in 18s +2026/09/24 20:34:05 unattended.go:327: [INFO] [update-leg] wishlist: step ended done after 20.1 s +2026/09/24 20:34:05 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=2 undone=1 held=0 failed=0 skipped=1 in 3m6s [skipped: navidrome=needs_person] +2026/09/24 20:34:05 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 3m5.528s) +2026/09/24 20:34:05 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:31:00 CEST (waiting 23h56m54s) diff --git a/documentation/audits/night-2026-09-25/C/night1/pages-after.json b/documentation/audits/night-2026-09-25/C/night1/pages-after.json new file mode 100644 index 00000000..10816856 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night1/pages-after.json @@ -0,0 +1,162 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "installed": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": 2, + "failed_step": null, + "last_auto_update": null, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": 2, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:31:00Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.0.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": { + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "ladder": "5c632840c2e89eff", + "at": "2026-09-24T20:33:41Z", + "outcome": "undone" + }, + "last_auto_update": { + "at": "2026-09-24T20:32:00Z", + "outcome": "undone", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night1/window-moved.txt b/documentation/audits/night-2026-09-25/C/night1/window-moved.txt new file mode 100644 index 00000000..4483b818 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night1/window-moved.txt @@ -0,0 +1,12 @@ +22:29:17 before: +2026/09/24 20:10:16 scheduler.go:132: [INFO] [scheduler] Daily job offsite-proof scheduled for 2026-09-25 05:30 CEST +2026/09/24 20:10:16 scheduler.go:132: [INFO] [scheduler] Daily job metrics-prune scheduled for 2026-09-25 04:00 CEST +2026/09/24 20:10:16 scheduler.go:132: [INFO] [scheduler] Daily job fill-watch scheduled for 2026-09-25 03:30 CEST +22:29:17 POST /backups/window window_start=20:46 -> 303 +22:29:23 after: +2026/09/24 20:29:17 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:29:17 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.2:35332 +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 02:30 → 20:46 (next run 2026-09-25 20:46 CEST) +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 03:30 → 21:46 (next run 2026-09-25 21:46 CEST) +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 04:15 → 22:31 (next run 2026-09-24 22:31 CEST) +2026/09/24 20:29:17 backup_handlers.go:64: [INFO] [web] backup window set to 20:46 (legs 20:46/21:46/22:31) diff --git a/documentation/audits/night-2026-09-25/C/night2/app-lines.txt b/documentation/audits/night-2026-09-25/C/night2/app-lines.txt new file mode 100644 index 00000000..20bcc3c7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Frissítés elérhető — 1 napja'] lines=['Automatikus frissítés 2026-09-24 22:37-kor — sikeres.'] +[hu] romm: badge=['Frissítés elérhető — 1 napja'] lines=['Automatikus frissítés 2026-09-24 22:37-kor — sikeres.'] +[hu] vikunja: badge=['Frissítés elérhető — 2 napja'] lines=['A(z) vikunja frissítése 2026-09-24 22:33-kor nem sikerült. A doboz automatikusan visszaállította az előző változatot és az adatokat — semmi nem veszett el.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Update available — 1 day ago'] lines=['Automatic update at 2026-09-24 22:37 — done.'] +[en] romm: badge=['Update available — 1 day ago'] lines=['Automatic update at 2026-09-24 22:37 — done.'] +[en] vikunja: badge=['Update available — 2 days ago'] lines=['The update of vikunja at 2026-09-24 22:33 did not succeed. The box put back the previous version and its data automatically — nothing was lost.'] diff --git a/documentation/audits/night-2026-09-25/C/night2/controller-full.log b/documentation/audits/night-2026-09-25/C/night2/controller-full.log new file mode 100644 index 00000000..77f92b0e --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/controller-full.log @@ -0,0 +1,702 @@ +2026/09/24 20:35:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:35:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:35:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:35:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:35:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:35:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m43s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m47s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m5s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 2 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:35:46 healthprobe.go:181: [DEBUG] Health probe privatebin: HTTP GET :8080/ → 200 (3ms) +2026/09/24 20:35:46 healthprobe.go:181: [DEBUG] Health probe paperless-ngx: HTTP GET :8000/ → 302 (6ms) +2026/09/24 20:35:46 healthprobe.go:161: [INFO] Health probes: 2 ok (of 2 probed) +2026/09/24 20:35:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:35:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m15s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m53s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m57s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:35:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:35:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m3s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m7s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m25s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:36:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:36:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:36:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:36:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5552MB avail≈20346MB +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24129336 → total=25898MB avail=20346MB used=5552MB (21.4%) +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.71 1.20 1.32 10/1488 9854" → 1m=0.71 5m=1.20 15m=1.32 +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.8°C (hwmon2) +2026/09/24 20:36:16 info.go:13: [DEBUG] [system] GetInfo done in 77ms — mem=5552MB/25898MB (21.4%), rootDisk=23.2GB/68.4GB (34.0%), load=0.71/1.20/1.32, temp=54.8°C (hwmon2), cpu=4.2% +2026/09/24 20:36:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m17s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m35s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m13s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:36:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:36:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:36:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:36:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:36:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 103ms) +2026/09/24 20:36:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m23s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m27s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m45s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m37s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m55s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m33s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:36:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:36:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m5s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m43s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m47s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m15s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m53s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m57s ago, effective interval 5m0s, healthy=true +2026/09/24 20:36:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:36:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:36:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:37:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:37:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:37:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:52, no step starts at or after 01:52 +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:37:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:37:00 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:37:00 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:37:00 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:37:00 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:00 update_guard.go:248: [DEBUG] [backup] update precondition for navidrome: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:37:00 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-162400737: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-162400737/docker-compose.yml: no such file or directory +2026/09/24 20:37:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5547MB avail≈20351MB +2026/09/24 20:37:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24135200 → total=25898MB avail=20351MB used=5547MB (21.4%) +2026/09/24 20:37:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=1000MB, new_req=50MB, remaining=24464MB +2026/09/24 20:37:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:37:00 update.go:526: [INFO] [stacks] update navidrome: accepted — guarded update started +2026/09/24 20:37:00 unattended.go:320: [INFO] [update-leg] navidrome: step pressed navidrome=deluan/navidrome:0.63.2 → navidrome=deluan/navidrome:0.64.0 +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase checking +2026/09/24 20:37:00 update.go:776: [INFO] [stacks] update navidrome: ladder — step 1 of 2: navidrome=deluan/navidrome:0.63.2 → navidrome=deluan/navidrome:0.64.0, from 755d09b083f757a0.yml (probe from 755d09b083f757a0.felhom.yml) +2026/09/24 20:37:00 update_guard.go:248: [DEBUG] [backup] update precondition for navidrome: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:37:00 update.go:789: [INFO] [stacks] update navidrome: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (17m0s old, limit 24h0m0s) +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase safety-dump +2026/09/24 20:37:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:37:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +e237099ca189 romm romm rommapp/romm:5.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +257abdec6373 navidrome navidrome deluan/navidrome:0.63.2 +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.0, not a database) +2026/09/24 20:37:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:37:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.63.2, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:37:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:37:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:37:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:37:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:37:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:37:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:37:00 update_guard.go:476: [INFO] [backup] update safety dump for navidrome: the app has no database — nothing to copy (no-op) +2026/09/24 20:37:00 update.go:821: [INFO] [stacks] update navidrome: safety dump done (0 file(s)) [] +2026/09/24 20:37:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:37:00 undo.go:285: [INFO] [stacks] update navidrome: the undo copy will hold 1 named volume(s), 4.7 MiB +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase pinning +2026/09/24 20:37:00 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:37:00 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:37:00 pin.go:373: [INFO] [stacks] update navidrome: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/navidrome/steps/755d09b083f757a0.yml (navidrome=deluan/navidrome:0.64.0) +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase pulling +2026/09/24 20:37:00 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, HDD_PATH, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:37:00 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/navidrome) +2026/09/24 20:37:01 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.5s) +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:37:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, HDD_PATH, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:37:01 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/navidrome) +2026/09/24 20:37:01 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 0.3s) +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:37:01 undo.go:435: [INFO] [stacks] update navidrome: copied navidrome_navidrome_data → navidrome_navidrome_data.pre-update-20260924T203701Z in 450ms +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase starting +2026/09/24 20:37:01 manager.go:418: [INFO] [stacks] userdata belt: drive /mnt/felhom-drives/scratch_hdd/userdata/navidrome not mounted — skipping ensure (held by drive gate) +2026/09/24 20:37:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DOMAIN, HDD_PATH, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:37:01 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/navidrome) +2026/09/24 20:37:02 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:37:02 update.go:1285: [INFO] [stacks] update navidrome: phase verifying +2026/09/24 20:37:02 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-3285006523: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3285006523/docker-compose.yml: no such file or directory +2026/09/24 20:37:02 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m25s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m3s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 1 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:37:06 healthprobe.go:181: [DEBUG] Health probe romm: HTTP GET :8080/ → 200 (1ms) +2026/09/24 20:37:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:06 healthprobe.go:161: [INFO] Health probes: 1 ok (of 1 probed) +2026/09/24 20:37:07 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:07 healthprobe.go:181: [DEBUG] Health probe navidrome: API GET :4533/ping → 200 (1ms) +2026/09/24 20:37:07 update.go:942: [INFO] [stacks] update navidrome: healthy after 5s (the app's health check passed) +2026/09/24 20:37:07 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:37:07 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:37:07 installed.go:420: [INFO] [stacks] installed-images navidrome: recorded 1 service(s) (navidrome=deluan/navidrome:0.64.0 (sha256:a384948b81bd…)) +2026/09/24 20:37:07 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:37:07 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:37:07 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:37:07 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:37:07 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:37:07 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:07 update.go:958: [INFO] [stacks] update navidrome: DONE in 7s +2026/09/24 20:37:10 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:37:10 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:37:10 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 10.1 s +2026/09/24 20:37:10 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-3526501500: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3526501500/docker-compose.yml: no such file or directory +2026/09/24 20:37:10 [ERROR] [stacks] .felhom.yml data_paths rejected in /tmp/felhom-step-meta-3526501500: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3526501500/docker-compose.yml: no such file or directory +2026/09/24 20:37:10 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5466MB avail≈20432MB +2026/09/24 20:37:10 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24231844 → total=25898MB avail=20432MB used=5466MB (21.1%) +2026/09/24 20:37:10 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=750MB, new_req=300MB, remaining=24464MB +2026/09/24 20:37:10 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:37:10 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:37:10 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine +2026/09/24 20:37:10 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:37:10 update.go:776: [INFO] [stacks] update romm: ladder — step 2 of 3: romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine, from 90dd9d68258286ef.yml (probe from 90dd9d68258286ef.felhom.yml) +2026/09/24 20:37:10 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:37:10 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:35:16Z (2m0s old, limit 24h0m0s) +2026/09/24 20:37:10 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:37:10 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:37:10 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: a0e7ad90f241 navidrome navidrome deluan/navidrome:0.64.0 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +e237099ca189 romm romm rommapp/romm:5.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.0, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.0, not a database) +2026/09/24 20:37:10 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:37:10 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:37:10 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:37:10 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:37:10 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:37:10 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:37:10 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:37:10 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:37:10 dbdump.go:266: [DEBUG] DumpOne: starting dump for container=romm-db, stack=romm, dbType=mariadb, dest=/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203710Z-romm-mariadb.sql +2026/09/24 20:37:10 dbdump.go:321: [DEBUG] DumpOne: mariadb-dump command: docker exec 43a089ab2dc8 mariadb-dump -u root -p*** --single-transaction --routines --triggers romm +2026/09/24 20:37:10 dbdump.go:405: [DEBUG] DumpOne: completed romm-db → pre-restore-20260924T203710Z-romm-mariadb.sql (size=110.2 KB, valid=true, tables=36, duration=292ms) +2026/09/24 20:37:10 dbdump.go:410: [INFO] [backup] DB dump: romm-db → pre-restore-20260924T203710Z-romm-mariadb.sql (110.2 KB, 292ms, 36 tables) +2026/09/24 20:37:10 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T203710Z-romm-mariadb.sql (110.2 KB) +2026/09/24 20:37:10 update_guard.go:478: [INFO] [backup] update safety dump for romm: 1 file(s) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203710Z-romm-mariadb.sql] +2026/09/24 20:37:10 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203710Z-romm-mariadb.sql] +2026/09/24 20:37:11 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:37:11 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 180.7 MiB +2026/09/24 20:37:11 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:37:11 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:37:11 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:37:11 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/steps/90dd9d68258286ef.yml (romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine) +2026/09/24 20:37:11 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:37:11 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:37:11 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/romm) +2026/09/24 20:37:12 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:37:12 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:12 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:37:12 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/romm) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5624MB avail≈20274MB +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24043888 → total=25898MB avail=20274MB used=5624MB (21.7%) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.79 1.13 1.29 11/1515 10615" → 1m=0.79 5m=1.13 15m=1.29 +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.1°C (hwmon2) +2026/09/24 20:37:16 info.go:13: [DEBUG] [system] GetInfo done in 57ms — mem=5624MB/25898MB (21.7%), rootDisk=23.2GB/68.4GB (34.0%), load=0.79/1.13/1.29, temp=54.1°C (hwmon2), cpu=8.7% +2026/09/24 20:37:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:37:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:37:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m35s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m13s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 9s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:37:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:20 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 7.7s) +2026/09/24 20:37:20 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:20 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_config → romm_romm_config.pre-update-20260924T203720Z in 468ms +2026/09/24 20:37:20 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:21 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_db_data → romm_romm_db_data.pre-update-20260924T203720Z in 771ms +2026/09/24 20:37:21 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:21 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_redis_data → romm_romm_redis_data.pre-update-20260924T203720Z in 467ms +2026/09/24 20:37:21 update.go:1285: [INFO] [stacks] update romm: phase starting +2026/09/24 20:37:21 manager.go:418: [INFO] [stacks] userdata belt: drive /mnt/felhom-drives/scratch_hdd/userdata/romm not mounted — skipping ensure (held by drive gate) +2026/09/24 20:37:21 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, SUBDOMAIN, DB_PASSWORD, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:37:21 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/romm) +2026/09/24 20:37:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m45s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m23s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 19s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:37:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:33 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 11.2s) +2026/09/24 20:37:33 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:37:33 [ERROR] [stacks] .felhom.yml backup block rejected in /tmp/felhom-step-meta-3586421514: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3586421514/docker-compose.yml: no such file or directory +2026/09/24 20:37:33 [ERROR] [stacks] .felhom.yml data_paths rejected in /tmp/felhom-step-meta-3586421514: docker-compose.yml unreadable: stat /tmp/felhom-step-meta-3586421514/docker-compose.yml: no such file or directory +2026/09/24 20:37:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 29s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m55s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m33s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:37:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:38 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:37:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:37:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m43s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 39s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m5s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:37:48 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:37:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:37:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 49s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m15s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m53s ago, effective interval 5m0s, healthy=true +2026/09/24 20:37:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:37:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:37:58 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:38:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 59s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m25s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m3s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:38:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:38:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:08 healthprobe.go:181: [DEBUG] Health probe romm: HTTP GET :8080/ → 200 (1ms) +2026/09/24 20:38:08 update.go:942: [INFO] [stacks] update romm: healthy after 35s (the app's health check passed) +2026/09/24 20:38:08 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:38:08 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:38:08 installed.go:420: [INFO] [stacks] installed-images romm: recorded 3 service(s) (romm=rommapp/romm:5.3.1 (sha256:0d66b4ea152a…), romm-db=mariadb:11.4 (sha256:70cc072b29b4…), romm-redis=redis:7-alpine (sha256:858f009f9709…)) +2026/09/24 20:38:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:38:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:38:08 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:38:08 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:38:08 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:38:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:08 update.go:958: [INFO] [stacks] update romm: DONE in 58s +2026/09/24 20:38:10 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:38:10 unattended.go:327: [INFO] [update-leg] romm: step ended done after 60.1 s +2026/09/24 20:38:10 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:38:10 unattended.go:300: [INFO] [update-leg] vikunja: skipped — failed_before +2026/09/24 20:38:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=2 undone=0 held=0 failed=0 skipped=1 in 1m10s [skipped: vikunja=failed_before] +2026/09/24 20:38:10 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 1m10.343s) +2026/09/24 20:38:10 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:37:00 CEST (waiting 23h58m50s) +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:38:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:38:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 8s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m9s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m35s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m13s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:38:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:38:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:38:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:38:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5463MB avail≈20435MB +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24234440 → total=25898MB avail=20435MB used=5463MB (21.1%) +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="2.04 1.39 1.37 12/1481 11044" → 1m=2.04 5m=1.39 15m=1.37 +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 63.0°C (hwmon2) +2026/09/24 20:38:16 info.go:13: [DEBUG] [system] GetInfo done in 75ms — mem=5463MB/25898MB (21.1%), rootDisk=23.2GB/68.4GB (34.0%), load=2.04/1.39/1.37, temp=63.0°C (hwmon2), cpu=37.9% +2026/09/24 20:38:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:38:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:38:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:38:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:38:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:38:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 101ms) +2026/09/24 20:38:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:38:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:38:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m45s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m23s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 18s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m19s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:38:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) diff --git a/documentation/audits/night-2026-09-25/C/night2/controller-night.log b/documentation/audits/night-2026-09-25/C/night2/controller-night.log new file mode 100644 index 00000000..87bfdce0 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/controller-night.log @@ -0,0 +1,51 @@ +2026/09/24 20:37:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:37:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:37:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:37:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:52, no step starts at or after 01:52 +2026/09/24 20:37:00 update.go:526: [INFO] [stacks] update navidrome: accepted — guarded update started +2026/09/24 20:37:00 unattended.go:320: [INFO] [update-leg] navidrome: step pressed navidrome=deluan/navidrome:0.63.2 → navidrome=deluan/navidrome:0.64.0 +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase checking +2026/09/24 20:37:00 update.go:776: [INFO] [stacks] update navidrome: ladder — step 1 of 2: navidrome=deluan/navidrome:0.63.2 → navidrome=deluan/navidrome:0.64.0, from 755d09b083f757a0.yml (probe from 755d09b083f757a0.felhom.yml) +2026/09/24 20:37:00 update.go:789: [INFO] [stacks] update navidrome: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (17m0s old, limit 24h0m0s) +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase safety-dump +2026/09/24 20:37:00 update.go:821: [INFO] [stacks] update navidrome: safety dump done (0 file(s)) [] +2026/09/24 20:37:00 undo.go:285: [INFO] [stacks] update navidrome: the undo copy will hold 1 named volume(s), 4.7 MiB +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase pinning +2026/09/24 20:37:00 pin.go:373: [INFO] [stacks] update navidrome: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/navidrome/steps/755d09b083f757a0.yml (navidrome=deluan/navidrome:0.64.0) +2026/09/24 20:37:00 update.go:1285: [INFO] [stacks] update navidrome: phase pulling +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:37:01 undo.go:435: [INFO] [stacks] update navidrome: copied navidrome_navidrome_data → navidrome_navidrome_data.pre-update-20260924T203701Z in 450ms +2026/09/24 20:37:01 update.go:1285: [INFO] [stacks] update navidrome: phase starting +2026/09/24 20:37:02 update.go:1285: [INFO] [stacks] update navidrome: phase verifying +2026/09/24 20:37:07 update.go:942: [INFO] [stacks] update navidrome: healthy after 5s (the app's health check passed) +2026/09/24 20:37:07 update.go:958: [INFO] [stacks] update navidrome: DONE in 7s +2026/09/24 20:37:10 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 10.1 s +2026/09/24 20:37:10 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:37:10 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine +2026/09/24 20:37:10 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:37:10 update.go:776: [INFO] [stacks] update romm: ladder — step 2 of 3: romm=rommapp/romm:5.3.0, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine, from 90dd9d68258286ef.yml (probe from 90dd9d68258286ef.felhom.yml) +2026/09/24 20:37:10 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:35:16Z (2m0s old, limit 24h0m0s) +2026/09/24 20:37:10 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:37:10 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T203710Z-romm-mariadb.sql (110.2 KB) +2026/09/24 20:37:10 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T203710Z-romm-mariadb.sql] +2026/09/24 20:37:11 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 180.7 MiB +2026/09/24 20:37:11 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:37:11 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/steps/90dd9d68258286ef.yml (romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine) +2026/09/24 20:37:11 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:37:12 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:20 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:20 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_config → romm_romm_config.pre-update-20260924T203720Z in 468ms +2026/09/24 20:37:20 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:21 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_db_data → romm_romm_db_data.pre-update-20260924T203720Z in 771ms +2026/09/24 20:37:21 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:37:21 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_redis_data → romm_romm_redis_data.pre-update-20260924T203720Z in 467ms +2026/09/24 20:37:21 update.go:1285: [INFO] [stacks] update romm: phase starting +2026/09/24 20:37:33 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:38:08 update.go:942: [INFO] [stacks] update romm: healthy after 35s (the app's health check passed) +2026/09/24 20:38:08 update.go:958: [INFO] [stacks] update romm: DONE in 58s +2026/09/24 20:38:10 unattended.go:327: [INFO] [update-leg] romm: step ended done after 60.1 s +2026/09/24 20:38:10 unattended.go:300: [INFO] [update-leg] vikunja: skipped — failed_before +2026/09/24 20:38:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=2 undone=0 held=0 failed=0 skipped=1 in 1m10s [skipped: vikunja=failed_before] +2026/09/24 20:38:10 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 1m10.343s) +2026/09/24 20:38:10 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:37:00 CEST (waiting 23h58m50s) diff --git a/documentation/audits/night-2026-09-25/C/night2/pages-after.json b/documentation/audits/night-2026-09-25/C/night2/pages-after.json new file mode 100644 index 00000000..de851e6a --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/pages-after.json @@ -0,0 +1,171 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "installed": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": 1, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:00Z", + "outcome": "done", + "from": { + "navidrome": "deluan/navidrome:0.63.2" + }, + "to": { + "navidrome": "deluan/navidrome:0.64.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": 1, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:10Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": { + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "ladder": "5c632840c2e89eff", + "at": "2026-09-24T20:33:41Z", + "outcome": "undone" + }, + "last_auto_update": { + "at": "2026-09-24T20:32:00Z", + "outcome": "undone", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night2/runner.log b/documentation/audits/night-2026-09-25/C/night2/runner.log new file mode 100644 index 00000000..fa0b7ca3 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/runner.log @@ -0,0 +1,7 @@ +22:35:34 night night2: W=20:52 -> off-site job + update leg at 22:37 +22:35:43 20:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 21:46 → 21:52 (next run 2026-09-25 21:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:31 → 22:37 (next run 2026-09-24 22:37 CEST) +2026/09/24 20:35:37 backup_handlers.go:64: [INFO] [web] backup window set to 20:52 (legs 20:52/21:52/22:37) + +22:38:27 SUMMARY: 2026/09/24 20:38:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=2 undone=0 held=0 failed=0 skipped=1 in 1m10s [skipped: vikunja=failed_before] diff --git a/documentation/audits/night-2026-09-25/C/night2/window-moved.txt b/documentation/audits/night-2026-09-25/C/night2/window-moved.txt new file mode 100644 index 00000000..d0ee1992 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night2/window-moved.txt @@ -0,0 +1,12 @@ +22:35:37 before: +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 02:30 → 20:46 (next run 2026-09-25 20:46 CEST) +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 03:30 → 21:46 (next run 2026-09-25 21:46 CEST) +2026/09/24 20:29:17 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 04:15 → 22:31 (next run 2026-09-24 22:31 CEST) +22:35:37 POST /backups/window window_start=20:52 -> 303 +22:35:43 after: +2026/09/24 20:35:37 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:35:37 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.2:39722 +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 20:46 → 20:52 (next run 2026-09-25 20:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 21:46 → 21:52 (next run 2026-09-25 21:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:31 → 22:37 (next run 2026-09-24 22:37 CEST) +2026/09/24 20:35:37 backup_handlers.go:64: [INFO] [web] backup window set to 20:52 (legs 20:52/21:52/22:37) diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/app-lines.txt b/documentation/audits/night-2026-09-25/C/night3-kill/app-lines.txt new file mode 100644 index 00000000..4daea896 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:42-kor — sikeres.'] +[hu] romm: badge=['Frissítés elérhető — 1 napja'] lines=['A frissítés megszakadt, mert a vezérlő újraindult, mielőtt az új verzió elindult volna. Az alkalmazás a korábbi verzióval fut tovább.'] +[hu] vikunja: badge=['Frissítés elérhető — 2 napja'] lines=['A(z) vikunja frissítése 2026-09-24 22:33-kor nem sikerült. A doboz automatikusan visszaállította az előző változatot és az adatokat — semmi nem veszett el.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:42 — done.'] +[en] romm: badge=['Update available — 1 day ago'] lines=['The update was interrupted, because the controller restarted before the new version started. The app keeps running on its previous version.'] +[en] vikunja: badge=['Update available — 2 days ago'] lines=['The update of vikunja at 2026-09-24 22:33 did not succeed. The box put back the previous version and its data automatically — nothing was lost.'] diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/controller-after-kill.log b/documentation/audits/night-2026-09-25/C/night3-kill/controller-after-kill.log new file mode 100644 index 00000000..21979b0c --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/controller-after-kill.log @@ -0,0 +1,22 @@ +2026/09/24 20:42:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:42:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:57, no step starts at or after 01:57 +2026/09/24 20:42:00 update.go:526: [INFO] [stacks] update navidrome: accepted — guarded update started +2026/09/24 20:42:00 unattended.go:320: [INFO] [update-leg] navidrome: step pressed navidrome=deluan/navidrome:0.64.0 → navidrome=deluan/navidrome:0.64.1 +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase checking +2026/09/24 20:42:00 update.go:776: [INFO] [stacks] update navidrome: ladder — the last step (2 of 2) — the catalog's current definition +2026/09/24 20:42:00 update.go:789: [INFO] [stacks] update navidrome: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:40:14Z (2m0s old, limit 24h0m0s) +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase safety-dump +2026/09/24 20:42:00 update.go:821: [INFO] [stacks] update navidrome: safety dump done (0 file(s)) [] +2026/09/24 20:42:00 undo.go:285: [INFO] [stacks] update navidrome: the undo copy will hold 1 named volume(s), 1.1 MiB +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pinning +2026/09/24 20:42:00 pin.go:373: [INFO] [stacks] update navidrome: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/navidrome/docker-compose.yml (navidrome=deluan/navidrome:0.64.1) +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pulling +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:23 undo.go:435: [INFO] [stacks] update navidrome: copied navidrome_navidrome_data → navidrome_navidrome_data.pre-update-20260924T204201Z in 21.498s +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase starting +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase verifying +2026/09/24 20:42:28 update.go:942: [INFO] [stacks] update navidrome: healthy after 5s (the app's health check passed) +2026/09/24 20:42:28 update.go:958: [INFO] [stacks] update navidrome: DONE in 29s +2026/09/24 20:42:30 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 30.1 s +2026/09/24 20:42:30 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/controller-before-kill.log b/documentation/audits/night-2026-09-25/C/night3-kill/controller-before-kill.log new file mode 100644 index 00000000..db4cd624 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/controller-before-kill.log @@ -0,0 +1,484 @@ +2026/09/24 20:40:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:40:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:40:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m18s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m19s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:40:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:40:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:40:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m28s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m29s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:40:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:40:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:40:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 5m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m38s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m39s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 5m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:40:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:40:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:40:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:40:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:40:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:40:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m48s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m49s ago, effective interval 5m0s, healthy=true +2026/09/24 20:40:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 2 targets (4 skipped not due, 0 skipped no container) +2026/09/24 20:40:56 healthprobe.go:181: [DEBUG] Health probe privatebin: HTTP GET :8080/ → 200 (3ms) +2026/09/24 20:40:56 healthprobe.go:181: [DEBUG] Health probe paperless-ngx: HTTP GET :8000/ → 302 (4ms) +2026/09/24 20:40:56 healthprobe.go:161: [INFO] Health probes: 2 ok (of 2 probed) +2026/09/24 20:41:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m58s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m59s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:41:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:41:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:41:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:41:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5831MB avail≈20067MB +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23798564 → total=25898MB avail=20067MB used=5831MB (22.5%) +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="2.30 1.58 1.43 14/1506 11766" → 1m=2.30 5m=1.58 15m=1.43 +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 57.2°C (hwmon2) +2026/09/24 20:41:16 info.go:13: [DEBUG] [system] GetInfo done in 72ms — mem=5831MB/25898MB (22.5%), rootDisk=23.2GB/68.4GB (34.0%), load=2.30/1.58/1.43, temp=57.2°C (hwmon2), cpu=22.6% +2026/09/24 20:41:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m8s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m9s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:41:16 main.go:2017: [INFO] [deadapp] check alive: 60 scans since boot, 6 deployed app(s) evaluated, 0 currently down +2026/09/24 20:41:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m18s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m19s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:41:36 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:36 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:36 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m28s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m29s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:36 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:41:46 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:46 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m38s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:41:46 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m39s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:46 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:41:46 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:56 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:41:56 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:41:56 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m48s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m49s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:41:56 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:42:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:42:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:42:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:42:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 20:57, no step starts at or after 01:57 +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:42:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:42:00 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:42:00 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:42:00 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:42:00 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈6081MB avail≈19817MB +2026/09/24 20:42:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23502068 → total=25898MB avail=19817MB used=6081MB (23.5%) +2026/09/24 20:42:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=1000MB, new_req=50MB, remaining=24464MB +2026/09/24 20:42:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:42:00 update.go:526: [INFO] [stacks] update navidrome: accepted — guarded update started +2026/09/24 20:42:00 unattended.go:320: [INFO] [update-leg] navidrome: step pressed navidrome=deluan/navidrome:0.64.0 → navidrome=deluan/navidrome:0.64.1 +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase checking +2026/09/24 20:42:00 update.go:776: [INFO] [stacks] update navidrome: ladder — the last step (2 of 2) — the catalog's current definition +2026/09/24 20:42:00 update_guard.go:248: [DEBUG] [backup] update precondition for navidrome: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:42:00 update.go:789: [INFO] [stacks] update navidrome: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:40:14Z (2m0s old, limit 24h0m0s) +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase safety-dump +2026/09/24 20:42:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:42:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 7a091a2055e4 romm romm rommapp/romm:5.3.1 +a0e7ad90f241 navidrome navidrome deluan/navidrome:0.64.0 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.1, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.0, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:42:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:42:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:42:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:42:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:42:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:42:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:42:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:42:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:42:00 update_guard.go:476: [INFO] [backup] update safety dump for navidrome: the app has no database — nothing to copy (no-op) +2026/09/24 20:42:00 update.go:821: [INFO] [stacks] update navidrome: safety dump done (0 file(s)) [] +2026/09/24 20:42:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:42:00 undo.go:285: [INFO] [stacks] update navidrome: the undo copy will hold 1 named volume(s), 1.1 MiB +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pinning +2026/09/24 20:42:00 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:42:00 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:42:00 pin.go:373: [INFO] [stacks] update navidrome: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/navidrome/docker-compose.yml (navidrome=deluan/navidrome:0.64.1) +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pulling +2026/09/24 20:42:00 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, SUBDOMAIN, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:42:00 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/navidrome) +2026/09/24 20:42:01 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, SUBDOMAIN, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:42:01 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/navidrome) +2026/09/24 20:42:01 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 0.3s) +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:06 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m59s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m58s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:06 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:42:06 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:42:16 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5825MB avail≈20073MB +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23805528 → total=25898MB avail=20073MB used=5825MB (22.5%) +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:42:16 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:42:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m8s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:16 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:16 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:42:16 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:42:16 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:42:16 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:42:16 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readLoadAvg: raw="4.00 2.13 1.63 10/1512 12096" → 1m=4.00 5m=2.13 15m=1.63 +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 53.8°C (hwmon2) +2026/09/24 20:42:16 info.go:13: [DEBUG] [system] GetInfo done in 72ms — mem=5825MB/25898MB (22.5%), rootDisk=23.2GB/68.4GB (34.0%), load=4.00/2.13/1.63, temp=53.8°C (hwmon2), cpu=14.0% +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:42:16 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:42:16 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:42:16 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:42:16 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:42:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:16 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 71ms) +2026/09/24 20:42:16 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:23 undo.go:435: [INFO] [stacks] update navidrome: copied navidrome_navidrome_data → navidrome_navidrome_data.pre-update-20260924T204201Z in 21.498s +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase starting +2026/09/24 20:42:23 manager.go:418: [INFO] [stacks] userdata belt: drive /mnt/felhom-drives/scratch_hdd/userdata/navidrome not mounted — skipping ensure (held by drive gate) +2026/09/24 20:42:23 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, HDD_PATH, SUBDOMAIN, USERDATA_PATH, IMPORT_PATH] (10 app + 0 system) +2026/09/24 20:42:23 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/navidrome) +2026/09/24 20:42:23 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase verifying +2026/09/24 20:42:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:26 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:42:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:26 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m18s ago, effective interval 5m0s, healthy=true +2026/09/24 20:42:26 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:42:26 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:42:26 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:28 healthprobe.go:181: [DEBUG] Health probe navidrome: API GET :4533/ping → 200 (1ms) +2026/09/24 20:42:28 update.go:942: [INFO] [stacks] update navidrome: healthy after 5s (the app's health check passed) +2026/09/24 20:42:28 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:42:28 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:42:28 installed.go:420: [INFO] [stacks] installed-images navidrome: recorded 1 service(s) (navidrome=deluan/navidrome:0.64.1 (sha256:df22d661b8d0…)) +2026/09/24 20:42:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:42:28 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:42:28 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:42:28 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:42:28 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:42:28 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:42:28 update.go:958: [INFO] [stacks] update navidrome: DONE in 29s +2026/09/24 20:42:30 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/navidrome — 3 env vars, 0 encrypted, 0 sensitive fields +2026/09/24 20:42:30 [INFO] [stacks] SaveAppConfig: saved config for navidrome +2026/09/24 20:42:30 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 30.1 s +2026/09/24 20:42:30 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:42:30 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5897MB avail≈20001MB +2026/09/24 20:42:30 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23720228 → total=25898MB avail=20001MB used=5897MB (22.8%) +2026/09/24 20:42:30 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=750MB, new_req=300MB, remaining=24464MB +2026/09/24 20:42:30 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:42:30 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:42:30 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine +2026/09/24 20:42:30 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:42:30 update.go:776: [INFO] [stacks] update romm: ladder — the last step (3 of 3) — the catalog's current definition +2026/09/24 20:42:30 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:42:30 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:40:14Z (2m0s old, limit 24h0m0s) +2026/09/24 20:42:30 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:42:30 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:42:30 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 09cc02b55482 navidrome navidrome deluan/navidrome:0.64.1 +7a091a2055e4 romm romm rommapp/romm:5.3.1 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.1, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.1, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:42:30 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:42:30 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:42:30 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:42:30 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:42:30 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:42:30 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:42:30 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:42:30 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:42:30 dbdump.go:266: [DEBUG] DumpOne: starting dump for container=romm-db, stack=romm, dbType=mariadb, dest=/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T204230Z-romm-mariadb.sql +2026/09/24 20:42:30 dbdump.go:321: [DEBUG] DumpOne: mariadb-dump command: docker exec 43a089ab2dc8 mariadb-dump -u root -p*** --single-transaction --routines --triggers romm +2026/09/24 20:42:30 dbdump.go:405: [DEBUG] DumpOne: completed romm-db → pre-restore-20260924T204230Z-romm-mariadb.sql (size=110.2 KB, valid=true, tables=36, duration=300ms) +2026/09/24 20:42:30 dbdump.go:410: [INFO] [backup] DB dump: romm-db → pre-restore-20260924T204230Z-romm-mariadb.sql (110.2 KB, 300ms, 36 tables) +2026/09/24 20:42:30 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T204230Z-romm-mariadb.sql (110.2 KB) +2026/09/24 20:42:30 update_guard.go:478: [INFO] [backup] update safety dump for romm: 1 file(s) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T204230Z-romm-mariadb.sql] +2026/09/24 20:42:30 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T204230Z-romm-mariadb.sql] +2026/09/24 20:42:31 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:42:31 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 180.9 MiB +2026/09/24 20:42:31 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:42:31 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:42:31 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:42:31 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/docker-compose.yml (romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine) +2026/09/24 20:42:31 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:42:31 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:42:31 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/romm) +2026/09/24 20:42:32 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:42:32 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:42:32 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:42:32 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/romm) diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/pages-after.json b/documentation/audits/night-2026-09-25/C/night3-kill/pages-after.json new file mode 100644 index 00000000..930bb387 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/pages-after.json @@ -0,0 +1,171 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "installed": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:42:00Z", + "outcome": "done", + "from": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "to": { + "navidrome": "deluan/navidrome:0.64.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": 1, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:10Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 1 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 1 day ago" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": { + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "ladder": "5c632840c2e89eff", + "at": "2026-09-24T20:33:41Z", + "outcome": "undone" + }, + "last_auto_update": { + "at": "2026-09-24T20:32:00Z", + "outcome": "undone", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/runner.log b/documentation/audits/night-2026-09-25/C/night3-kill/runner.log new file mode 100644 index 00000000..f1514ca7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/runner.log @@ -0,0 +1,19 @@ +22:40:10 night3: W=20:57, leg at 22:42 +22:42:34 ARMED: 2026/09/24 20:42:30 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 30.1 s +22:42:40 KILL: pid=345594 kill_rc=0 2026-09-24T20:42:39Z +22:43:23 controller after kill: running healthy 1 2026-09-24T20:42:39.737174063Z (43 s) +22:45:56 after-kill leg lines: +undo.go:285: [INFO] [stacks] update navidrome: the undo copy will hold 1 named volume(s), 1.1 MiB +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pinning +2026/09/24 20:42:00 pin.go:373: [INFO] [stacks] update navidrome: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/navidrome/docker-compose.yml (navidrome=deluan/navidrome:0.64.1) +2026/09/24 20:42:00 update.go:1285: [INFO] [stacks] update navidrome: phase pulling +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:01 update.go:1285: [INFO] [stacks] update navidrome: phase copying +2026/09/24 20:42:23 undo.go:435: [INFO] [stacks] update navidrome: copied navidrome_navidrome_data → navidrome_navidrome_data.pre-update-20260924T204201Z in 21.498s +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase starting +2026/09/24 20:42:23 update.go:1285: [INFO] [stacks] update navidrome: phase verifying +2026/09/24 20:42:28 update.go:942: [INFO] [stacks] update navidrome: healthy after 5s (the app's health check passed) +2026/09/24 20:42:28 update.go:958: [INFO] [stacks] update navidrome: DONE in 29s +2026/09/24 20:42:30 unattended.go:327: [INFO] [update-leg] navidrome: step ended done after 30.1 s +2026/09/24 20:42:30 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine + diff --git a/documentation/audits/night-2026-09-25/C/night3-kill/window-moved.txt b/documentation/audits/night-2026-09-25/C/night3-kill/window-moved.txt new file mode 100644 index 00000000..d81a98e9 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night3-kill/window-moved.txt @@ -0,0 +1,12 @@ +22:40:13 before: +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 20:46 → 20:52 (next run 2026-09-25 20:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 21:46 → 21:52 (next run 2026-09-25 21:52 CEST) +2026/09/24 20:35:37 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:31 → 22:37 (next run 2026-09-24 22:37 CEST) +22:40:14 POST /backups/window window_start=20:57 -> 303 +22:40:20 after: +2026/09/24 20:40:13 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:40:13 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.2:51902 +2026/09/24 20:40:13 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 20:52 → 20:57 (next run 2026-09-25 20:57 CEST) +2026/09/24 20:40:13 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 21:52 → 21:57 (next run 2026-09-25 21:57 CEST) +2026/09/24 20:40:13 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:37 → 22:42 (next run 2026-09-24 22:42 CEST) +2026/09/24 20:40:14 backup_handlers.go:64: [INFO] [web] backup window set to 20:57 (legs 20:57/21:57/22:42) diff --git a/documentation/audits/night-2026-09-25/C/night4-power/app-lines.txt b/documentation/audits/night-2026-09-25/C/night4-power/app-lines.txt new file mode 100644 index 00000000..1d4929c7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:42-kor — sikeres.'] +[hu] romm: badge=['Naprakész'] lines=[] +[hu] vikunja: badge=['Frissítés elérhető — 2 napja'] lines=['A(z) vikunja frissítése 2026-09-24 22:33-kor nem sikerült. A doboz automatikusan visszaállította az előző változatot és az adatokat — semmi nem veszett el.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:42 — done.'] +[en] romm: badge=['Up to date'] lines=[] +[en] vikunja: badge=['Update available — 2 days ago'] lines=['The update of vikunja at 2026-09-24 22:33 did not succeed. The box put back the previous version and its data automatically — nothing was lost.'] diff --git a/documentation/audits/night-2026-09-25/C/night4-power/controller-after-boot.log b/documentation/audits/night-2026-09-25/C/night4-power/controller-after-boot.log new file mode 100644 index 00000000..faffb467 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/controller-after-boot.log @@ -0,0 +1,8 @@ +2026/09/24 20:52:53 update.go:1360: [WARN] [stacks] update recovery: romm was interrupted in verifying (started 2026-09-24T20:52:00Z) — the new version may have run; marking it Updating and RESUMING the health wait +2026/09/24 20:52:53 main.go:496: [WARN] [update] 1 interrupted update(s) will resume after the backup side is wired: [romm] +2026/09/24 20:52:53 restore_record_wiring.go:35: [INFO] [backup] restore record wired: /opt/docker/felhom-controller/data/restore-status.json (interrupted at startup: false) +2026/09/24 20:52:53 main.go:594: [WARN] [update] resumed 1 interrupted update(s) +2026/09/24 20:52:53 update.go:1407: [INFO] [stacks] update romm: resuming after a controller restart — `up -d` then the health wait +2026/09/24 20:53:01 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:53:31 update.go:942: [INFO] [stacks] update romm: healthy after 30s (the app's health check passed) +2026/09/24 20:53:32 update.go:958: [INFO] [stacks] update romm: DONE in 1m32s diff --git a/documentation/audits/night-2026-09-25/C/night4-power/controller-before-cut.log b/documentation/audits/night-2026-09-25/C/night4-power/controller-before-cut.log new file mode 100644 index 00000000..362f9301 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/controller-before-cut.log @@ -0,0 +1,372 @@ +2026/09/24 20:50:21 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:50:21 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:50:21 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:21 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:50:21 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:50:31 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:31 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:50:31 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:50:31 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:50:41 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:41 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:50:41 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:50:41 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5860MB avail≈20038MB +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23764672 → total=25898MB avail=20038MB used=5860MB (22.6%) +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.17 1.52 1.66 4/1479 1718" → 1m=1.17 5m=1.52 15m=1.66 +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 56.4°C (hwmon2) +2026/09/24 20:50:51 info.go:13: [DEBUG] [system] GetInfo done in 66ms — mem=5860MB/25898MB (22.6%), rootDisk=23.2GB/68.4GB (34.0%), load=1.17/1.52/1.66, temp=56.4°C (hwmon2), cpu=24.7% +2026/09/24 20:50:51 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:50:51 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:50:51 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:50:51 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:50:51 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:50:51 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:50:51 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:50:51 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:50:51 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:50:51 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:50:51 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:50:51 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:50:51 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 85ms) +2026/09/24 20:51:01 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:01 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:01 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:01 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:51:11 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:11 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:11 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:11 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:21 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:21 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:51:21 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:21 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:51:31 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:31 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:31 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:31 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:41 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:41 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:41 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:41 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5856MB avail≈20042MB +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23768292 → total=25898MB avail=20042MB used=5856MB (22.6%) +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.99 1.63 1.68 7/1480 1884" → 1m=1.99 5m=1.63 15m=1.68 +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 56.2°C (hwmon2) +2026/09/24 20:51:51 info.go:13: [DEBUG] [system] GetInfo done in 63ms — mem=5856MB/25898MB (22.6%), rootDisk=23.2GB/68.4GB (34.0%), load=1.99/1.63/1.68, temp=56.2°C (hwmon2), cpu=23.8% +2026/09/24 20:51:51 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:51:51 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:51:51 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:51:51 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:51:51 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:52:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:52:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:52:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 21:07, no step starts at or after 02:07 +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:52:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:52:00 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:52:00 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:52:00 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:52:00 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:00 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:52:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5839MB avail≈20059MB +2026/09/24 20:52:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23788580 → total=25898MB avail=20059MB used=5839MB (22.5%) +2026/09/24 20:52:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=750MB, new_req=300MB, remaining=24464MB +2026/09/24 20:52:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:52:00 update.go:526: [INFO] [stacks] update romm: accepted — guarded update started +2026/09/24 20:52:00 unattended.go:320: [INFO] [update-leg] romm: step pressed romm=rommapp/romm:5.3.1, romm-db=mariadb:11.4, romm-redis=redis:7-alpine → romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine +2026/09/24 20:52:00 update.go:1285: [INFO] [stacks] update romm: phase checking +2026/09/24 20:52:00 update.go:776: [INFO] [stacks] update romm: ladder — the last step (3 of 3) — the catalog's current definition +2026/09/24 20:52:00 update_guard.go:248: [DEBUG] [backup] update precondition for romm: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:52:00 update.go:789: [INFO] [stacks] update romm: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:42:30Z (10m0s old, limit 24h0m0s) +2026/09/24 20:52:00 update.go:1285: [INFO] [stacks] update romm: phase safety-dump +2026/09/24 20:52:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:52:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 09cc02b55482 navidrome navidrome deluan/navidrome:0.64.1 +7a091a2055e4 romm romm rommapp/romm:5.3.1 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +43a089ab2dc8 romm-db romm mariadb:11.4 +e8b0b31b8cab romm-redis romm redis:7-alpine +ca88824b7c2c felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.1, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.1, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:52:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=43a089ab2dc8) +2026/09/24 20:52:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:52:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:52:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:52:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:52:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:52:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:52:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:52:00 dbdump.go:266: [DEBUG] DumpOne: starting dump for container=romm-db, stack=romm, dbType=mariadb, dest=/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T205200Z-romm-mariadb.sql +2026/09/24 20:52:00 dbdump.go:321: [DEBUG] DumpOne: mariadb-dump command: docker exec 43a089ab2dc8 mariadb-dump -u root -p*** --single-transaction --routines --triggers romm +2026/09/24 20:52:00 dbdump.go:405: [DEBUG] DumpOne: completed romm-db → pre-restore-20260924T205200Z-romm-mariadb.sql (size=110.2 KB, valid=true, tables=36, duration=298ms) +2026/09/24 20:52:00 dbdump.go:410: [INFO] [backup] DB dump: romm-db → pre-restore-20260924T205200Z-romm-mariadb.sql (110.2 KB, 298ms, 36 tables) +2026/09/24 20:52:00 offbox_reconstitute.go:255: [INFO] [offbox] romm: pre-restore safety dump written → pre-restore-20260924T205200Z-romm-mariadb.sql (110.2 KB) +2026/09/24 20:52:00 update_guard.go:478: [INFO] [backup] update safety dump for romm: 1 file(s) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T205200Z-romm-mariadb.sql] +2026/09/24 20:52:00 update.go:821: [INFO] [stacks] update romm: safety dump done (1 file(s)) [/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/primary/romm/db-dumps/pre-restore-20260924T205200Z-romm-mariadb.sql] +2026/09/24 20:52:01 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:52:01 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:52:01 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:01 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:52:01 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:52:01 undo.go:285: [INFO] [stacks] update romm: the undo copy will hold 3 named volume(s), 181.3 MiB +2026/09/24 20:52:01 update.go:1285: [INFO] [stacks] update romm: phase pinning +2026/09/24 20:52:01 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/romm — 6 env vars, 0 encrypted, 3 sensitive fields +2026/09/24 20:52:01 [INFO] [stacks] SaveAppConfig: saved config for romm +2026/09/24 20:52:01 pin.go:373: [INFO] [stacks] update romm: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/romm/docker-compose.yml (romm=rommapp/romm:5.3.1, romm-db=mariadb:11.8, romm-redis=redis:7-alpine) +2026/09/24 20:52:01 update.go:1285: [INFO] [stacks] update romm: phase pulling +2026/09/24 20:52:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:52:01 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/romm) +2026/09/24 20:52:02 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:52:02 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:52:02 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:52:02 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/romm) +2026/09/24 20:52:09 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 7.0s) +2026/09/24 20:52:09 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:52:09 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_config → romm_romm_config.pre-update-20260924T205209Z in 462ms +2026/09/24 20:52:09 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:52:10 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_db_data → romm_romm_db_data.pre-update-20260924T205209Z in 927ms +2026/09/24 20:52:10 update.go:1285: [INFO] [stacks] update romm: phase copying +2026/09/24 20:52:11 undo.go:435: [INFO] [stacks] update romm: copied romm_romm_redis_data → romm_romm_redis_data.pre-update-20260924T205209Z in 456ms +2026/09/24 20:52:11 update.go:1285: [INFO] [stacks] update romm: phase starting +2026/09/24 20:52:11 manager.go:418: [INFO] [stacks] userdata belt: drive /mnt/felhom-drives/scratch_hdd/userdata/romm not mounted — skipping ensure (held by drive gate) +2026/09/24 20:52:11 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, SUBDOMAIN, DB_PASSWORD, DOMAIN, HDD_PATH, MYSQL_ROOT_PASSWORD, ROMM_AUTH_SECRET_KEY, USERDATA_PATH, IMPORT_PATH] (13 app + 0 system) +2026/09/24 20:52:11 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/romm) +2026/09/24 20:52:11 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:52:11 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:52:11 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:11 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:11 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:52:21 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:52:21 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:52:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:21 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:21 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:52:21 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:52:21 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:52:21 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:21 manager.go:861: [DEBUG] [stacks] restart-policy of down member "romm" = "unless-stopped" +2026/09/24 20:52:27 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 16.2s) +2026/09/24 20:52:27 update.go:1285: [INFO] [stacks] update romm: phase verifying +2026/09/24 20:52:27 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:31 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:52:31 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:52:31 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:52:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:31 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:52:31 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (5 skipped not due, 0 skipped no container) +2026/09/24 20:52:32 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks diff --git a/documentation/audits/night-2026-09-25/C/night4-power/pages-after.json b/documentation/audits/night-2026-09-25/C/night4-power/pages-after.json new file mode 100644 index 00000000..2f7479f3 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/pages-after.json @@ -0,0 +1,171 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "installed": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:42:00Z", + "outcome": "done", + "from": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "to": { + "navidrome": "deluan/navidrome:0.64.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:10Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": { + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "ladder": "5c632840c2e89eff", + "at": "2026-09-24T20:33:41Z", + "outcome": "undone" + }, + "last_auto_update": { + "at": "2026-09-24T20:32:00Z", + "outcome": "undone", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night4-power/runner.log b/documentation/audits/night-2026-09-25/C/night4-power/runner.log new file mode 100644 index 00000000..62166181 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/runner.log @@ -0,0 +1,5 @@ +22:50:03 night4: W=21:07, leg at 22:52 +22:52:31 ARMED: 2026/09/24 20:52:27 update.go:1285: [INFO] [stacks] update romm: phase verifying +22:52:46 POWER CUT: 2026-09-24T20:52:34Z stop_rc=0 start_rc=0 2026-09-24T20:52:46Z +22:53:00 controller after boot: running healthy (13 s) +22:53:40 romm after resume: 2026/09/24 20:53:32 update.go:958: [INFO] [stacks] update romm: DONE in 1m32s diff --git a/documentation/audits/night-2026-09-25/C/night4-power/window-moved.txt b/documentation/audits/night-2026-09-25/C/night4-power/window-moved.txt new file mode 100644 index 00000000..fbea68ea --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night4-power/window-moved.txt @@ -0,0 +1,12 @@ +22:50:06 before: +2026/09/24 20:42:51 scheduler.go:132: [INFO] [scheduler] Daily job offsite-proof scheduled for 2026-09-25 05:30 CEST +2026/09/24 20:42:51 scheduler.go:132: [INFO] [scheduler] Daily job metrics-prune scheduled for 2026-09-25 04:00 CEST +2026/09/24 20:42:51 scheduler.go:132: [INFO] [scheduler] Daily job fill-watch scheduled for 2026-09-25 03:30 CEST +22:50:06 POST /backups/window window_start=21:07 -> 303 +22:50:12 after: +2026/09/24 20:50:06 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:50:06 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.2:35970 +2026/09/24 20:50:06 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 20:57 → 21:07 (next run 2026-09-25 21:07 CEST) +2026/09/24 20:50:06 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 21:57 → 22:07 (next run 2026-09-25 22:07 CEST) +2026/09/24 20:50:06 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:42 → 22:52 (next run 2026-09-24 22:52 CEST) +2026/09/24 20:50:06 backup_handlers.go:64: [INFO] [web] backup window set to 21:07 (legs 21:07/22:07/22:52) diff --git a/documentation/audits/night-2026-09-25/C/night5-off/app-lines.txt b/documentation/audits/night-2026-09-25/C/night5-off/app-lines.txt new file mode 100644 index 00000000..1d4929c7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:42-kor — sikeres.'] +[hu] romm: badge=['Naprakész'] lines=[] +[hu] vikunja: badge=['Frissítés elérhető — 2 napja'] lines=['A(z) vikunja frissítése 2026-09-24 22:33-kor nem sikerült. A doboz automatikusan visszaállította az előző változatot és az adatokat — semmi nem veszett el.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:42 — done.'] +[en] romm: badge=['Up to date'] lines=[] +[en] vikunja: badge=['Update available — 2 days ago'] lines=['The update of vikunja at 2026-09-24 22:33 did not succeed. The box put back the previous version and its data automatically — nothing was lost.'] diff --git a/documentation/audits/night-2026-09-25/C/night5-off/controller-full.log b/documentation/audits/night-2026-09-25/C/night5-off/controller-full.log new file mode 100644 index 00000000..cfbf5e8b --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/controller-full.log @@ -0,0 +1,186 @@ +2026/09/24 20:54:43 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m11s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:43 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:54:43 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:54:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5736MB avail≈20162MB +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23910864 → total=25898MB avail=20162MB used=5736MB (22.1%) +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readLoadAvg: raw="2.01 2.00 1.82 2/1508 823" → 1m=2.01 5m=2.00 15m=1.82 +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 50.6°C (hwmon2) +2026/09/24 20:54:53 info.go:13: [DEBUG] [system] GetInfo done in 63ms — mem=5736MB/25898MB (22.1%), rootDisk=23.2GB/68.4GB (34.0%), load=2.01/2.00/1.82, temp=50.6°C (hwmon2), cpu=6.8% +2026/09/24 20:54:53 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:54:53 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:54:53 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:54:53 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:54:53 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m21s ago, effective interval 5m0s, healthy=true +2026/09/24 20:54:53 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:54:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:54:53 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:54:53 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:54:53 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:54:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:54:53 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 64ms) +2026/09/24 20:54:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:03 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:03 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m31s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:03 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:55:13 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m41s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:13 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:13 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:55:13 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:23 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:23 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:23 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:55:23 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:55:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 1m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 1m51s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:23 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:55:33 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:33 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m1s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:33 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:55:43 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:43 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m11s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:43 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:55:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5642MB avail≈20256MB +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24022284 → total=25898MB avail=20256MB used=5642MB (21.8%) +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readLoadAvg: raw="1.01 1.72 1.74 3/1507 993" → 1m=1.01 5m=1.72 15m=1.74 +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 49.9°C (hwmon1) +2026/09/24 20:55:53 info.go:13: [DEBUG] [system] GetInfo done in 59ms — mem=5642MB/25898MB (21.8%), rootDisk=23.2GB/68.4GB (34.0%), load=1.01/1.72/1.74, temp=49.9°C (hwmon1), cpu=7.0% +2026/09/24 20:55:53 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:55:53 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:55:53 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:55:53 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:55:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 2m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m21s ago, effective interval 5m0s, healthy=true +2026/09/24 20:55:53 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:56:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:56:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:56:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:56:00 unattended.go:244: [INFO] [update-leg] automatic app updates are switched OFF on this box (app_update.unattended=false) — no app is pressed tonight +2026/09/24 20:56:00 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=0 undone=0 held=0 failed=0 skipped=0 stopped=switched_off in 0s [skipped: ] +2026/09/24 20:56:00 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 0s) +2026/09/24 20:56:00 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:56:00 CEST (waiting 24h0m0s) +2026/09/24 20:56:03 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:56:03 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:56:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 2m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 2m31s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:03 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) diff --git a/documentation/audits/night-2026-09-25/C/night5-off/controller-night.log b/documentation/audits/night-2026-09-25/C/night5-off/controller-night.log new file mode 100644 index 00000000..93fe56c5 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/controller-night.log @@ -0,0 +1,7 @@ +2026/09/24 20:56:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:56:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:56:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:56:00 unattended.go:244: [INFO] [update-leg] automatic app updates are switched OFF on this box (app_update.unattended=false) — no app is pressed tonight +2026/09/24 20:56:00 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=0 undone=0 held=0 failed=0 skipped=0 stopped=switched_off in 0s [skipped: ] +2026/09/24 20:56:00 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 0s) +2026/09/24 20:56:00 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:56:00 CEST (waiting 24h0m0s) diff --git a/documentation/audits/night-2026-09-25/C/night5-off/pages-after.json b/documentation/audits/night-2026-09-25/C/night5-off/pages-after.json new file mode 100644 index 00000000..2f7479f3 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/pages-after.json @@ -0,0 +1,171 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "installed": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:42:00Z", + "outcome": "done", + "from": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "to": { + "navidrome": "deluan/navidrome:0.64.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:10Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": 1, + "failed_step": { + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "ladder": "5c632840c2e89eff", + "at": "2026-09-24T20:33:41Z", + "outcome": "undone" + }, + "last_auto_update": { + "at": "2026-09-24T20:32:00Z", + "outcome": "undone", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — 2 napja" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — 2 days ago" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night5-off/runner.log b/documentation/audits/night-2026-09-25/C/night5-off/runner.log new file mode 100644 index 00000000..56292c7e --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/runner.log @@ -0,0 +1,7 @@ +22:54:24 night night5-off: W=21:11 -> off-site job + update leg at 22:56 +22:54:34 21:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 22:07 → 22:11 (next run 2026-09-25 22:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:52 → 22:56 (next run 2026-09-24 22:56 CEST) +2026/09/24 20:54:28 backup_handlers.go:64: [INFO] [web] backup window set to 21:11 (legs 21:11/22:11/22:56) + +22:56:07 SUMMARY: 2026/09/24 20:56:00 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=0 undone=0 held=0 failed=0 skipped=0 stopped=switched_off in 0s [skipped: ] diff --git a/documentation/audits/night-2026-09-25/C/night5-off/window-moved.txt b/documentation/audits/night-2026-09-25/C/night5-off/window-moved.txt new file mode 100644 index 00000000..2b743951 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-off/window-moved.txt @@ -0,0 +1,12 @@ +22:54:27 before: +2026/09/24 20:52:53 scheduler.go:132: [INFO] [scheduler] Daily job offsite-proof scheduled for 2026-09-25 05:30 CEST +2026/09/24 20:52:53 scheduler.go:132: [INFO] [scheduler] Daily job metrics-prune scheduled for 2026-09-25 04:00 CEST +2026/09/24 20:52:53 scheduler.go:132: [INFO] [scheduler] Daily job fill-watch scheduled for 2026-09-25 03:30 CEST +22:54:28 POST /backups/window window_start=21:11 -> 303 +22:54:34 after: +2026/09/24 20:54:27 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:54:27 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.6:48942 +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 21:07 → 21:11 (next run 2026-09-25 21:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 22:07 → 22:11 (next run 2026-09-25 22:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:52 → 22:56 (next run 2026-09-24 22:56 CEST) +2026/09/24 20:54:28 backup_handlers.go:64: [INFO] [web] backup window set to 21:11 (legs 21:11/22:11/22:56) diff --git a/documentation/audits/night-2026-09-25/C/night5-switch-off.txt b/documentation/audits/night-2026-09-25/C/night5-switch-off.txt new file mode 100644 index 00000000..9100dd2b --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night5-switch-off.txt @@ -0,0 +1,2 @@ +POST /settings/app-update off -> 200; page reads unattended=false +settings.json: diff --git a/documentation/audits/night-2026-09-25/C/night6-on/app-lines.txt b/documentation/audits/night-2026-09-25/C/night6-on/app-lines.txt new file mode 100644 index 00000000..5d3d7fd9 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/app-lines.txt @@ -0,0 +1,8 @@ +[hu] wishlist: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:33-kor — sikeres.'] +[hu] navidrome: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:42-kor — sikeres.'] +[hu] romm: badge=['Naprakész'] lines=[] +[hu] vikunja: badge=['Naprakész'] lines=['Automatikus frissítés 2026-09-24 22:58-kor — sikeres.'] +[en] wishlist: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:33 — done.'] +[en] navidrome: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:42 — done.'] +[en] romm: badge=['Up to date'] lines=[] +[en] vikunja: badge=['Up to date'] lines=['Automatic update at 2026-09-24 22:58 — done.'] diff --git a/documentation/audits/night-2026-09-25/C/night6-on/controller-full.log b/documentation/audits/night-2026-09-25/C/night6-on/controller-full.log new file mode 100644 index 00000000..4c078c84 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/controller-full.log @@ -0,0 +1,495 @@ +2026/09/24 20:56:43 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m11s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:43 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:56:43 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:56:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5655MB avail≈20243MB +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=24007020 → total=25898MB avail=20243MB used=5655MB (21.8%) +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.90 1.57 1.68 4/1510 1247" → 1m=0.90 5m=1.57 15m=1.68 +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 50.9°C (hwmon1) +2026/09/24 20:56:53 info.go:13: [DEBUG] [system] GetInfo done in 60ms — mem=5655MB/25898MB (21.8%), rootDisk=23.2GB/68.4GB (34.0%), load=0.90/1.57/1.68, temp=50.9°C (hwmon1), cpu=19.6% +2026/09/24 20:56:53 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 scheduler.go:346: [INFO] [scheduler] Running job: stack-scan +2026/09/24 20:56:53 scheduler.go:67: [DEBUG] [scheduler] job stack-scan: execution starting +2026/09/24 20:56:53 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:56:53 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:56:53 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:56:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m21s ago, effective interval 5m0s, healthy=true +2026/09/24 20:56:53 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:56:53 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:56:53 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:56:53 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:56:53 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:56:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:56:53 scheduler.go:363: [INFO] [scheduler] Job stack-scan completed (took 61ms) +2026/09/24 20:56:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:03 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:03 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m31s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:03 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:13 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m41s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:13 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:13 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:13 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:23 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:23 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:57:23 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:23 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:57:23 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 3m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 3m51s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:23 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:33 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m0s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m1s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:33 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:33 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:43 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m11s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:43 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:43 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:43 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5762MB avail≈20136MB +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23879784 → total=25898MB avail=20136MB used=5762MB (22.2%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.97 1.45 1.63 2/1480 1409" → 1m=0.97 5m=1.45 15m=1.63 +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.6°C (hwmon2) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetInfo done in 56ms — mem=5762MB/25898MB (22.2%), rootDisk=23.2GB/68.4GB (34.0%), load=0.97/1.45/1.63, temp=54.6°C (hwmon2), cpu=22.3% +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/09/24 20:57:53 scheduler.go:346: [INFO] [scheduler] Running job: offsite-credential-retry +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job offsite-credential-retry: execution starting +2026/09/24 20:57:53 scheduler.go:363: [INFO] [scheduler] Job offsite-credential-retry completed (took 0s) +2026/09/24 20:57:53 scheduler.go:346: [INFO] [scheduler] Running job: backup-cache +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job backup-cache: execution starting +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m20s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m21s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 4m50s ago, effective interval 5m0s, healthy=true +2026/09/24 20:57:53 scheduler.go:346: [INFO] [scheduler] Running job: system-health +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job system-health: execution starting +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:57:53 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/09/24 20:57:53 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/09/24 20:57:53 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:57:53 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/romm → [romm] +2026/09/24 20:57:53 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/navidrome → [navidrome] +2026/09/24 20:57:53 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx → [paperless-ngx] +2026/09/24 20:57:53 backup.go:467: [DEBUG] groupStacksByDrive: /mnt/sys_drive → [privatebin, vikunja, wishlist] +2026/09/24 20:57:53 backup.go:1119: [INFO] [backup] Found 4 DB dump files across drives +2026/09/24 20:57:53 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:57:53 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 73681e40b7d2 felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +b8a0bdf6d8b5 romm-db romm mariadb:11.8 +09cc02b55482 navidrome navidrome deluan/navidrome:0.64.1 +7a091a2055e4 romm romm rommapp/romm:5.3.1 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +e8b0b31b8cab romm-redis romm redis:7-alpine +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:57:53 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=b8a0bdf6d8b5) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5764MB avail≈20134MB +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23878648 → total=25898MB avail=20134MB used=5764MB (22.3%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=23.2GB avail=41.6GB (34.0%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=71.2GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.97 1.45 1.63 7/1512 1471" → 1m=0.97 5m=1.45 15m=1.63 +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 54.2°C (hwmon2) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetInfo done in 66ms — mem=5764MB/25898MB (22.3%), rootDisk=23.2GB/68.4GB (34.0%), load=0.97/1.45/1.63, temp=54.2°C (hwmon2), cpu=26.8% +2026/09/24 20:57:53 healthcheck.go:86: [DEBUG] [monitor] Raw values: disk=34.0%, hdd=7.6% (configured=true), mem=22.3% (5764MB/25898MB), cpu=26.8%, temp=54.2°C (hwmon2) +2026/09/24 20:57:53 healthcheck.go:118: [DEBUG] [monitor] SSD disk: OK (34%) +2026/09/24 20:57:53 healthcheck.go:151: [DEBUG] [monitor] Memory: OK (22%) +2026/09/24 20:57:53 healthcheck.go:169: [DEBUG] [monitor] CPU: OK (27%) +2026/09/24 20:57:53 healthcheck.go:187: [DEBUG] [monitor] Temperature: OK (54°C) +2026/09/24 20:57:53 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.1, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.1, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:57:53 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:57:53 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:57:53 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:57:53 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:57:53 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:57:53 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 2 HDD mounts for /opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 1 HDD mounts for /opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:57:53 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:57:53 [INFO] [backup] Discovered app data: 6 apps +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/navidrome" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/romm" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/sys_drive" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:57:53 backup.go:1189: [INFO] [backup] Backup status cache refreshed +2026/09/24 20:57:53 scheduler.go:363: [INFO] [scheduler] Job backup-cache completed (took 132ms) +2026/09/24 20:57:53 healthcheck.go:201: [DEBUG] [monitor] Docker daemon: OK +2026/09/24 20:57:53 healthcheck.go:209: [DEBUG] [monitor] Checking 1 protected containers: [traefik] +2026/09/24 20:57:53 healthcheck.go:219: [DEBUG] [monitor] All protected containers running +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/romm" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 info.go:13: [DEBUG] [system] GetDiskUsage: path="/mnt/felhom-drives/scratch_hdd/userdata/navidrome" total=937.8 GB used=71.2 GB avail=818.9GB (7.6%) +2026/09/24 20:57:53 healthcheck.go:238: [INFO] [monitor] Health check: status=warn +2026/09/24 20:57:53 healthcheck.go:242: [DEBUG] [monitor] Final status: warn (issues=0, warnings=4, info=5) +2026/09/24 20:57:53 notifier.go:1255: [WARN] notifier disabled (no hub configured): DROPPED event health_change (severity warning) — further health_change events are logged at DEBUG only +2026/09/24 20:57:53 scheduler.go:363: [INFO] [scheduler] Job system-health completed (took 155ms) +2026/09/24 20:57:53 infra.go:67: [INFO] [infra] cloudflared skipped — no cf_tunnel_token configured (LAN-only node) +2026/09/24 20:58:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:58:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:58:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:58:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 21:13, no step starts at or after 02:13 +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:58:00 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:58:00 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:58:00 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:58:00 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:58:00 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:00 info.go:13: [DEBUG] [system] readMemInfo: guest cap=25898MB (host total was 30714460KB) → used≈5873MB avail≈20025MB +2026/09/24 20:58:00 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714460 availKB=23748460 → total=25898MB avail=20025MB used=5873MB (22.7%) +2026/09/24 20:58:00 deploy.go:1272: [INFO] [stacks] Memory check: total=25898MB, reserved=384MB, usable=25514MB, committed_used=1000MB, new_req=50MB, remaining=24464MB +2026/09/24 20:58:00 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:58:00 update.go:526: [INFO] [stacks] update vikunja: accepted — guarded update started +2026/09/24 20:58:00 unattended.go:320: [INFO] [update-leg] vikunja: step pressed vikunja=vikunja/vikunja:2.3.0 → vikunja=vikunja/vikunja:2.6.0 +2026/09/24 20:58:00 update.go:1285: [INFO] [stacks] update vikunja: phase checking +2026/09/24 20:58:00 update.go:776: [INFO] [stacks] update vikunja: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:58:00 update_guard.go:248: [DEBUG] [backup] update precondition for vikunja: no Tier-2 copy (nincs másodlagos fájlmásolat ehhez az alkalmazáshoz) +2026/09/24 20:58:00 update.go:789: [INFO] [stacks] update vikunja: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (38m0s old, limit 24h0m0s) +2026/09/24 20:58:00 update.go:1285: [INFO] [stacks] update vikunja: phase safety-dump +2026/09/24 20:58:00 dbdump.go:130: [DEBUG] DiscoverDatabases: running docker ps to find database containers +2026/09/24 20:58:00 dbdump.go:149: [DEBUG] DiscoverDatabases: docker ps output: 73681e40b7d2 felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.271.0 +b8a0bdf6d8b5 romm-db romm mariadb:11.8 +09cc02b55482 navidrome navidrome deluan/navidrome:0.64.1 +7a091a2055e4 romm romm rommapp/romm:5.3.1 +1f408c3d379a wishlist wishlist ghcr.io/cmintey/wishlist:v0.67.1 +6b1598990997 vikunja vikunja vikunja/vikunja:2.3.0 +e8b0b31b8cab romm-redis romm redis:7-alpine +361a05346765 privatebin privatebin privatebin/pdo:2.0.6 +92b06fef42c0 paperless-webserver paperless-ngx ghcr.io/paperless-n... +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container felhom-controller (image=gitea.dooplex.hu/admin/felhom-controller:0.271.0, not a database) +2026/09/24 20:58:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found mariadb container: romm-db (id=b8a0bdf6d8b5) +2026/09/24 20:58:00 dbdump.go:198: [DEBUG] DiscoverDatabases: romm-db → stack=romm, dbUser=root, dbName=romm +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container navidrome (image=deluan/navidrome:0.64.1, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm (image=rommapp/romm:5.3.1, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container wishlist (image=ghcr.io/cmintey/wishlist:v0.67.1, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container vikunja (image=vikunja/vikunja:2.3.0, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container romm-redis (image=redis:7-alpine, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container privatebin (image=privatebin/pdo:2.0.6, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-webserver (image=ghcr.io/paperless-ngx/paperless-ngx:2.20.15, not a database) +2026/09/24 20:58:00 dbdump.go:178: [DEBUG] DiscoverDatabases: found postgres container: paperless-postgres (id=5e5c371b82fa) +2026/09/24 20:58:00 dbdump.go:896: [DEBUG] DiscoverDatabases: paperless-postgres → stack "paperless-ngx" from the compose project label (the container name would have given "paperless") +2026/09/24 20:58:00 dbdump.go:198: [DEBUG] DiscoverDatabases: paperless-postgres → stack=paperless-ngx, dbUser=paperless, dbName=paperless +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container paperless-redis (image=redis:7-alpine, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container filebrowser (image=gtstef/filebrowser:1.3.3-stable, not a database) +2026/09/24 20:58:00 dbdump.go:171: [DEBUG] DiscoverDatabases: skipping container traefik (image=traefik:v3.6.7, not a database) +2026/09/24 20:58:00 dbdump.go:205: [DEBUG] DiscoverDatabases: found 2 database(s), skipped 11 non-DB container(s) +2026/09/24 20:58:00 dbdump.go:208: [INFO] [backup] Discovered 2 databases +2026/09/24 20:58:00 update_guard.go:476: [INFO] [backup] update safety dump for vikunja: the app has no database — nothing to copy (no-op) +2026/09/24 20:58:00 update.go:821: [INFO] [stacks] update vikunja: safety dump done (0 file(s)) [] +2026/09/24 20:58:01 info.go:13: [DEBUG] [system] GetDiskUsage: path="/" total=68.4 GB used=23.2 GB avail=41.6GB (34.0%) +2026/09/24 20:58:01 undo.go:285: [INFO] [stacks] update vikunja: the undo copy will hold 2 named volume(s), 2.3 MiB +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase pinning +2026/09/24 20:58:01 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:58:01 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:58:01 pin.go:373: [INFO] [stacks] update vikunja: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/vikunja/docker-compose.yml (vikunja=vikunja/vikunja:2.6.0) +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase pulling +2026/09/24 20:58:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:58:01 manager.go:1473: [DEBUG] Running: docker compose pull (in /opt/docker/stacks/vikunja) +2026/09/24 20:58:01 manager.go:1493: [DEBUG] Command completed: docker compose pull (took 0.6s) +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:01 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:58:01 manager.go:1473: [DEBUG] Running: docker compose stop (in /opt/docker/stacks/vikunja) +2026/09/24 20:58:02 manager.go:1493: [DEBUG] Command completed: docker compose stop (took 0.3s) +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_data → vikunja_vikunja_data.pre-update-20260924T205802Z in 490ms +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_db → vikunja_vikunja_db.pre-update-20260924T205802Z in 412ms +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase starting +2026/09/24 20:58:02 manager.go:1464: [DEBUG] Env vars for compose: [PATH, HOSTNAME, FELHOM_BOOTSTRAP_PATH, HOME, DOMAIN, DOMAIN, SUBDOMAIN, VIKUNJA_SERVICE_JWTSECRET, IMPORT_PATH] (9 app + 0 system) +2026/09/24 20:58:02 manager.go:1473: [DEBUG] Running: docker compose up -d --remove-orphans (in /opt/docker/stacks/vikunja) +2026/09/24 20:58:03 manager.go:1493: [DEBUG] Command completed: docker compose up -d --remove-orphans (took 0.4s) +2026/09/24 20:58:03 update.go:1285: [INFO] [stacks] update vikunja: phase verifying +2026/09/24 20:58:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:03 healthprobe.go:190: [WARN] Health probe vikunja: API GET :3456/api/v1/info → Get "http://vikunja:3456/api/v1/info": dial tcp 172.18.0.5:3456: connect: connection refused +2026/09/24 20:58:03 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:58:03 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:58:03 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m30s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m31s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:03 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 0s ago, effective interval 10s, healthy=false +2026/09/24 20:58:03 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 3 targets (3 skipped not due, 0 skipped no container) +2026/09/24 20:58:03 healthprobe.go:181: [DEBUG] Health probe navidrome: API GET :4533/ping → 200 (1ms) +2026/09/24 20:58:03 healthprobe.go:181: [DEBUG] Health probe privatebin: HTTP GET :8080/ → 200 (3ms) +2026/09/24 20:58:03 healthprobe.go:181: [DEBUG] Health probe wishlist: HTTP GET :3000/ → 307 (3ms) +2026/09/24 20:58:03 healthprobe.go:161: [INFO] Health probes: 3 ok (of 3 probed) +2026/09/24 20:58:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:08 healthprobe.go:181: [DEBUG] Health probe vikunja: API GET :3456/api/v1/info → 200 (4ms) +2026/09/24 20:58:08 update.go:942: [INFO] [stacks] update vikunja: healthy after 5s (the app's health check passed) +2026/09/24 20:58:08 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:58:08 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:58:08 installed.go:420: [INFO] [stacks] installed-images vikunja: recorded 1 service(s) (vikunja=vikunja/vikunja:2.6.0 (sha256:417ada6f94e8…)) +2026/09/24 20:58:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:08 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:58:08 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:58:08 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:58:08 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "actualbudget" deployed=false composePath=/opt/docker/stacks/actualbudget/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "adventurelog" deployed=false composePath=/opt/docker/stacks/adventurelog/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "audiobookshelf" deployed=false composePath=/opt/docker/stacks/audiobookshelf/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bentopdf" deployed=false composePath=/opt/docker/stacks/bentopdf/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "bookstack" deployed=false composePath=/opt/docker/stacks/bookstack/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calcom" deployed=false composePath=/opt/docker/stacks/calcom/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "calibre-web" deployed=false composePath=/opt/docker/stacks/calibre-web/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaoscrash" deployed=false composePath=/opt/docker/stacks/chaoscrash/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoom" deployed=false composePath=/opt/docker/stacks/chaosoom/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "chaosoomb" deployed=false composePath=/opt/docker/stacks/chaosoomb/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "claper" deployed=false composePath=/opt/docker/stacks/claper/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "code-server" deployed=false composePath=/opt/docker/stacks/code-server/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "crafty-controller" deployed=false composePath=/opt/docker/stacks/crafty-controller/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "docmost" deployed=false composePath=/opt/docker/stacks/docmost/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "emby" deployed=false composePath=/opt/docker/stacks/emby/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "filebrowser" deployed=false composePath=/opt/docker/stacks/filebrowser/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "ghost" deployed=false composePath=/opt/docker/stacks/ghost/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gitea" deployed=false composePath=/opt/docker/stacks/gitea/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "glance" deployed=false composePath=/opt/docker/stacks/glance/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gokapi" deployed=false composePath=/opt/docker/stacks/gokapi/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "grafana" deployed=false composePath=/opt/docker/stacks/grafana/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "gramps-web" deployed=false composePath=/opt/docker/stacks/gramps-web/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "home-assistant" deployed=false composePath=/opt/docker/stacks/home-assistant/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homebox" deployed=false composePath=/opt/docker/stacks/homebox/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "homepage" deployed=false composePath=/opt/docker/stacks/homepage/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "immich" deployed=false composePath=/opt/docker/stacks/immich/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "jellyfin" deployed=false composePath=/opt/docker/stacks/jellyfin/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "kimai" deployed=false composePath=/opt/docker/stacks/kimai/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "komga" deployed=false composePath=/opt/docker/stacks/komga/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "mealie" deployed=false composePath=/opt/docker/stacks/mealie/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "n8n" deployed=false composePath=/opt/docker/stacks/n8n/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "navidrome" deployed=true composePath=/opt/docker/stacks/navidrome/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "nextcloud" deployed=false composePath=/opt/docker/stacks/nextcloud/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "onlyoffice" deployed=false composePath=/opt/docker/stacks/onlyoffice/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "opengist" deployed=false composePath=/opt/docker/stacks/opengist/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "outline" deployed=false composePath=/opt/docker/stacks/outline/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "paperless-ngx" deployed=true composePath=/opt/docker/stacks/paperless-ngx/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "papra" deployed=false composePath=/opt/docker/stacks/papra/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plant-it" deployed=false composePath=/opt/docker/stacks/plant-it/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "plex" deployed=false composePath=/opt/docker/stacks/plex/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "privatebin" deployed=true composePath=/opt/docker/stacks/privatebin/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "radarr" deployed=false composePath=/opt/docker/stacks/radarr/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "rallly" deployed=false composePath=/opt/docker/stacks/rallly/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "recipe-importer" deployed=false composePath=/opt/docker/stacks/recipe-importer/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "romm" deployed=true composePath=/opt/docker/stacks/romm/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "seerr" deployed=false composePath=/opt/docker/stacks/seerr/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sonarr" deployed=false composePath=/opt/docker/stacks/sonarr/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "sparkyfitness" deployed=false composePath=/opt/docker/stacks/sparkyfitness/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "tandoor" deployed=false composePath=/opt/docker/stacks/tandoor/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "termix" deployed=false composePath=/opt/docker/stacks/termix/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "traefik" deployed=false composePath=/opt/docker/stacks/traefik/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "uptime-kuma" deployed=false composePath=/opt/docker/stacks/uptime-kuma/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vaultwarden" deployed=false composePath=/opt/docker/stacks/vaultwarden/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "vikunja" deployed=true composePath=/opt/docker/stacks/vikunja/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wanderer" deployed=false composePath=/opt/docker/stacks/wanderer/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wger" deployed=false composePath=/opt/docker/stacks/wger/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "wishlist" deployed=true composePath=/opt/docker/stacks/wishlist/docker-compose.yml +2026/09/24 20:58:08 manager.go:616: [DEBUG] [stacks] ScanStacks: found stack "zipline" deployed=false composePath=/opt/docker/stacks/zipline/docker-compose.yml +2026/09/24 20:58:08 manager.go:1700: [DEBUG] [stacks] getCatalogTemplateSlugs: found 53 template slugs in /opt/docker/felhom-controller/data/catalog-cache/templates +2026/09/24 20:58:08 manager.go:697: [DEBUG] [stacks] ScanStacks: catalog has 53 template slugs for orphan detection +2026/09/24 20:58:08 manager.go:728: [INFO] [stacks] ScanStacks complete: 58 stacks found (6 deployed, 52 available) +2026/09/24 20:58:08 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks +2026/09/24 20:58:08 update.go:958: [INFO] [stacks] update vikunja: DONE in 9s +2026/09/24 20:58:10 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/vikunja — 3 env vars, 0 encrypted, 1 sensitive fields +2026/09/24 20:58:10 [INFO] [stacks] SaveAppConfig: saved config for vikunja +2026/09/24 20:58:10 unattended.go:327: [INFO] [update-leg] vikunja: step ended done after 10.1 s +2026/09/24 20:58:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=1 undone=0 held=0 failed=0 skipped=0 in 10s [skipped: ] +2026/09/24 20:58:10 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 10.192s) +2026/09/24 20:58:10 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:58:00 CEST (waiting 23h59m50s) +2026/09/24 20:58:13 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping paperless-ngx — last check 4m40s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping wishlist — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping romm — last check 4m41s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping vikunja — last check 5s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping privatebin — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 healthprobe.go:54: [DEBUG] [stacks] RunHealthProbes: skipping navidrome — last check 10s ago, effective interval 5m0s, healthy=true +2026/09/24 20:58:13 healthprobe.go:104: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (6 skipped not due, 0 skipped no container) +2026/09/24 20:58:13 manager.go:779: [INFO] [stacks] Status refresh: 12 containers across 58 stacks diff --git a/documentation/audits/night-2026-09-25/C/night6-on/controller-night.log b/documentation/audits/night-2026-09-25/C/night6-on/controller-night.log new file mode 100644 index 00000000..3beb179f --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/controller-night.log @@ -0,0 +1,28 @@ +2026/09/24 20:58:00 scheduler.go:346: [INFO] [scheduler] Running job: offbox-backup +2026/09/24 20:58:00 scheduler.go:67: [DEBUG] [scheduler] job offbox-backup: execution starting +2026/09/24 20:58:00 main.go:1299: [INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now +2026/09/24 20:58:00 unattended.go:254: [INFO] [update-leg] started (after-offsite): window 21:13, no step starts at or after 02:13 +2026/09/24 20:58:00 update.go:526: [INFO] [stacks] update vikunja: accepted — guarded update started +2026/09/24 20:58:00 unattended.go:320: [INFO] [update-leg] vikunja: step pressed vikunja=vikunja/vikunja:2.3.0 → vikunja=vikunja/vikunja:2.6.0 +2026/09/24 20:58:00 update.go:1285: [INFO] [stacks] update vikunja: phase checking +2026/09/24 20:58:00 update.go:776: [INFO] [stacks] update vikunja: ladder — the last step (1 of 1) — the catalog's current definition +2026/09/24 20:58:00 update.go:789: [INFO] [stacks] update vikunja: precondition met — Tier 1 (own recovery unit) copy from 2026-09-24T20:20:16Z (38m0s old, limit 24h0m0s) +2026/09/24 20:58:00 update.go:1285: [INFO] [stacks] update vikunja: phase safety-dump +2026/09/24 20:58:00 update.go:821: [INFO] [stacks] update vikunja: safety dump done (0 file(s)) [] +2026/09/24 20:58:01 undo.go:285: [INFO] [stacks] update vikunja: the undo copy will hold 2 named volume(s), 2.3 MiB +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase pinning +2026/09/24 20:58:01 pin.go:373: [INFO] [stacks] update vikunja: pin advanced to /opt/docker/felhom-controller/data/catalog-cache/templates/vikunja/docker-compose.yml (vikunja=vikunja/vikunja:2.6.0) +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase pulling +2026/09/24 20:58:01 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_data → vikunja_vikunja_data.pre-update-20260924T205802Z in 490ms +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase copying +2026/09/24 20:58:02 undo.go:435: [INFO] [stacks] update vikunja: copied vikunja_vikunja_db → vikunja_vikunja_db.pre-update-20260924T205802Z in 412ms +2026/09/24 20:58:02 update.go:1285: [INFO] [stacks] update vikunja: phase starting +2026/09/24 20:58:03 update.go:1285: [INFO] [stacks] update vikunja: phase verifying +2026/09/24 20:58:08 update.go:942: [INFO] [stacks] update vikunja: healthy after 5s (the app's health check passed) +2026/09/24 20:58:08 update.go:958: [INFO] [stacks] update vikunja: DONE in 9s +2026/09/24 20:58:10 unattended.go:327: [INFO] [update-leg] vikunja: step ended done after 10.1 s +2026/09/24 20:58:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=1 undone=0 held=0 failed=0 skipped=0 in 10s [skipped: ] +2026/09/24 20:58:10 scheduler.go:363: [INFO] [scheduler] Job offbox-backup completed (took 10.192s) +2026/09/24 20:58:10 scheduler.go:67: [DEBUG] [scheduler] daily job offbox-backup: next run at 2026-09-25 22:58:00 CEST (waiting 23h59m50s) diff --git a/documentation/audits/night-2026-09-25/C/night6-on/pages-after.json b/documentation/audits/night-2026-09-25/C/night6-on/pages-after.json new file mode 100644 index 00000000..aee1cf44 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/pages-after.json @@ -0,0 +1,164 @@ +{ + "wishlist": { + "state": "running", + "pinned": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "installed": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "catalog": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:33:45Z", + "outcome": "done", + "from": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.66.0" + }, + "to": { + "wishlist": "ghcr.io/cmintey/wishlist:v0.67.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "navidrome": { + "state": "running", + "pinned": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "installed": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "catalog": { + "navidrome": "deluan/navidrome:0.64.1" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:42:00Z", + "outcome": "done", + "from": { + "navidrome": "deluan/navidrome:0.64.0" + }, + "to": { + "navidrome": "deluan/navidrome:0.64.1" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "romm": { + "state": "running", + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "installed": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:37:10Z", + "outcome": "done", + "from": { + "romm": "rommapp/romm:5.3.0", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + }, + "to": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.4", + "romm-redis": "redis:7-alpine" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + }, + "vikunja": { + "state": "running", + "pinned": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "installed": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "catalog": { + "vikunja": "vikunja/vikunja:2.6.0" + }, + "steps_left": null, + "failed_step": null, + "last_auto_update": { + "at": "2026-09-24T20:58:00Z", + "outcome": "done", + "from": { + "vikunja": "vikunja/vikunja:2.3.0" + }, + "to": { + "vikunja": "vikunja/vikunja:2.6.0" + } + }, + "badges": { + "hu": [ + { + "title": "Ez az alkalmazás a legfrissebb elérhető változatot futtatja.", + "text": "Naprakész" + } + ], + "en": [ + { + "title": "This app is running the newest version available.", + "text": "Up to date" + } + ] + } + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/C/night6-on/runner.log b/documentation/audits/night-2026-09-25/C/night6-on/runner.log new file mode 100644 index 00000000..d89535a5 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/runner.log @@ -0,0 +1,7 @@ +22:56:33 night night6-on: W=21:13 -> off-site job + update leg at 22:58 +22:56:42 21:13 CEST) +2026/09/24 20:56:36 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 22:11 → 22:13 (next run 2026-09-25 22:13 CEST) +2026/09/24 20:56:36 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:56 → 22:58 (next run 2026-09-24 22:58 CEST) +2026/09/24 20:56:36 backup_handlers.go:64: [INFO] [web] backup window set to 21:13 (legs 21:13/22:13/22:58) + +22:58:15 SUMMARY: 2026/09/24 20:58:10 unattended.go:235: [INFO] [update-leg] update leg (after-offsite): done=1 undone=0 held=0 failed=0 skipped=0 in 10s [skipped: ] diff --git a/documentation/audits/night-2026-09-25/C/night6-on/window-moved.txt b/documentation/audits/night-2026-09-25/C/night6-on/window-moved.txt new file mode 100644 index 00000000..e5566295 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-on/window-moved.txt @@ -0,0 +1,12 @@ +22:56:36 before: +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 21:07 → 21:11 (next run 2026-09-25 21:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 22:07 → 22:11 (next run 2026-09-25 22:11 CEST) +2026/09/24 20:54:27 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:52 → 22:56 (next run 2026-09-24 22:56 CEST) +22:56:36 POST /backups/window window_start=21:13 -> 303 +22:56:42 after: +2026/09/24 20:56:36 auth.go:142: [DEBUG] [web] auth: valid session for POST /backups/window +2026/09/24 20:56:36 server.go:545: [DEBUG] [web] ServeHTTP: POST /backups/window from 172.18.0.6:50636 +2026/09/24 20:56:36 scheduler.go:172: [INFO] [scheduler] Daily job db-dump rescheduled 21:11 → 21:13 (next run 2026-09-25 21:13 CEST) +2026/09/24 20:56:36 scheduler.go:172: [INFO] [scheduler] Daily job tier2-backup rescheduled 22:11 → 22:13 (next run 2026-09-25 22:13 CEST) +2026/09/24 20:56:36 scheduler.go:172: [INFO] [scheduler] Daily job offbox-backup rescheduled 22:56 → 22:58 (next run 2026-09-24 22:58 CEST) +2026/09/24 20:56:36 backup_handlers.go:64: [INFO] [web] backup window set to 21:13 (legs 21:13/22:13/22:58) diff --git a/documentation/audits/night-2026-09-25/C/night6-switch-on.txt b/documentation/audits/night-2026-09-25/C/night6-switch-on.txt new file mode 100644 index 00000000..56c1e4f8 --- /dev/null +++ b/documentation/audits/night-2026-09-25/C/night6-switch-on.txt @@ -0,0 +1,2 @@ +POST /settings/app-update on -> 200; page reads unattended=true +settings.json: diff --git a/documentation/audits/night-2026-09-25/D/D0-demo-felhom-waiting.json b/documentation/audits/night-2026-09-25/D/D0-demo-felhom-waiting.json new file mode 100644 index 00000000..61453865 --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D0-demo-felhom-waiting.json @@ -0,0 +1,12 @@ +{ + "opengist": { + "pinned": { + "opengist": "ghcr.io/thomiceli/opengist:1.13" + }, + "catalog": { + "opengist": "ghcr.io/thomiceli/opengist:1.15" + }, + "ladder_len": 1, + "leg": "STEP WAITING: {\"opengist\": \"ghcr.io/thomiceli/opengist:1.15\"}" + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/D/D0-demo-hp-waiting.json b/documentation/audits/night-2026-09-25/D/D0-demo-hp-waiting.json new file mode 100644 index 00000000..d0f5791b --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D0-demo-hp-waiting.json @@ -0,0 +1,122 @@ +{ + "adventurelog": { + "pinned": { + "adventurelog": "ghcr.io/seanmorley15/adventurelog-backend:v0.12.1", + "adventurelog-frontend": "ghcr.io/seanmorley15/adventurelog-frontend:v0.12.1", + "adventurelog-postgres": "postgis/postgis:16-3.5-alpine" + }, + "catalog": { + "adventurelog": "ghcr.io/seanmorley15/adventurelog-backend:v0.12.1", + "adventurelog-postgres": "postgis/postgis:16-3.5-alpine", + "adventurelog-frontend": "ghcr.io/seanmorley15/adventurelog-frontend:v0.12.1" + }, + "ladder_len": 0, + "leg": "current (pin = catalog head)" + }, + "bentopdf": { + "pinned": { + "bentopdf": "ghcr.io/alam00000/bentopdf:v2.8.6" + }, + "catalog": { + "bentopdf": "ghcr.io/alam00000/bentopdf:v2.8.6" + }, + "ladder_len": 0, + "leg": "current (pin = catalog head)" + }, + "bookstack": { + "pinned": { + "bookstack": "lscr.io/linuxserver/bookstack:26.05.5", + "bookstack-db": "mariadb:12.3" + }, + "catalog": { + "bookstack": "lscr.io/linuxserver/bookstack:26.05.5", + "bookstack-db": "mariadb:12.3" + }, + "ladder_len": 1, + "leg": "current (pin = catalog head)" + }, + "calibre-web": { + "pinned": { + "calibre-web": "crocodilestick/calibre-web-automated:v4.0.6" + }, + "catalog": { + "calibre-web": "crocodilestick/calibre-web-automated:v4.0.6" + }, + "ladder_len": 0, + "leg": "current (pin = catalog head)" + }, + "docmost": { + "pinned": { + "docmost": "docmost/docmost:0.96.0", + "docmost-postgres": "postgres:16-alpine", + "docmost-redis": "redis:7-alpine" + }, + "catalog": { + "docmost": "docmost/docmost:0.96.0", + "docmost-postgres": "postgres:16-alpine", + "docmost-redis": "redis:7-alpine" + }, + "ladder_len": 1, + "leg": "current (pin = catalog head)" + }, + "kimai": { + "pinned": { + "kimai": "kimai/kimai2:apache-2.57.0", + "kimai-db": "mariadb:11.8" + }, + "catalog": { + "kimai": "kimai/kimai2:apache-2.57.0", + "kimai-db": "mariadb:11.8" + }, + "ladder_len": 1, + "leg": "current (pin = catalog head)" + }, + "opengist": { + "pinned": { + "opengist": "ghcr.io/thomiceli/opengist:1.15" + }, + "catalog": { + "opengist": "ghcr.io/thomiceli/opengist:1.15" + }, + "ladder_len": 1, + "leg": "current (pin = catalog head)" + }, + "paperless-ngx": { + "pinned": { + "paperless-postgres": "postgres:16-alpine", + "paperless-redis": "redis:7-alpine", + "paperless-webserver": "ghcr.io/paperless-ngx/paperless-ngx:2.20.15" + }, + "catalog": { + "paperless-webserver": "ghcr.io/paperless-ngx/paperless-ngx:2.20.15", + "paperless-postgres": "postgres:16-alpine", + "paperless-redis": "redis:7-alpine" + }, + "ladder_len": 0, + "leg": "current (pin = catalog head)" + }, + "privatebin": { + "pinned": { + "privatebin": "privatebin/pdo:2.0.6" + }, + "catalog": { + "privatebin": "privatebin/pdo:2.0.6" + }, + "ladder_len": 1, + "leg": "current (pin = catalog head)" + }, + "romm": { + "pinned": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "catalog": { + "romm": "rommapp/romm:5.3.1", + "romm-db": "mariadb:11.8", + "romm-redis": "redis:7-alpine" + }, + "ladder_len": 3, + "leg": "current (pin = catalog head)" + } +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/D/D0-waiting-steps.md b/documentation/audits/night-2026-09-25/D/D0-waiting-steps.md new file mode 100644 index 00000000..81a674dd --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D0-waiting-steps.md @@ -0,0 +1,11 @@ +# D0 — which demo-box apps have a tested step waiting tonight (read 2026-09-24 ~22:32 CEST, read-only) + +Method: `tools/d_waiting.py` — each box's 9201 `app.yaml` pins against the box's OWN catalog clone, with the leg's +rules (proven, needs_person, files_may_change, older than the ladder, current). Raw: `D0-demo-*-waiting.json`. + +| box | deployed apps | tested step waiting | +|---|---|---| +| demo-felhom (N100) | opengist only | **opengist 1.13 → 1.15** (proven, no marks) — the leg WILL press it | +| demo-hp | 10 (adventurelog, bentopdf, bookstack, calibre-web, docmost, kimai, opengist, paperless-ngx, privatebin, romm) | **none** — every app's pin equals the catalog head | + +Brief claim "a demo box has any tested step waiting tonight": **true for demo-felhom (one app), false for demo-hp.** diff --git a/documentation/audits/night-2026-09-25/D/D1-floor.txt b/documentation/audits/night-2026-09-25/D/D1-floor.txt new file mode 100644 index 00000000..7ec5fe72 --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D1-floor.txt @@ -0,0 +1,4 @@ +# D1 global floor 0.271.0 (declared MinAgent 0.131.0) — 2026-09-24T20:59:04+00:00 +impact before: {"below":5,"valid":true,"version":"0.271.0"} +POST /configuration/global-floor -> 303 Location: /configuration?flash=floor_set +read back: Managed updates — global floor The minimum controller version every box auto-updates to (unless a per-customer override is set). Saving takes effect immediately — boxes below the floor update on their next report, no customer action. Blank = no global floor. This setting is independent of the Day-0 artifact manifest below. Effective floor: v0.271.0 — source: DB (hub_settings) ; env fallback would diff --git a/documentation/audits/night-2026-09-25/D/D2-arrival.txt b/documentation/audits/night-2026-09-25/D/D2-arrival.txt new file mode 100644 index 00000000..a1eee933 --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D2-arrival.txt @@ -0,0 +1,3 @@ +# D2 arrival 2026-09-24T20:59:18+00:00 +== felhom-pve: gitea.dooplex.hu/admin/felhom-controller:0.271.0 gitea.dooplex.hu/admin/felhom-controller:0.271.0 Up 5 seconds (healthy) +== hp: gitea.dooplex.hu/admin/felhom-controller:0.271.0 gitea.dooplex.hu/admin/felhom-controller:0.271.0 Up 6 seconds (healthy) diff --git a/documentation/audits/night-2026-09-25/D/D3-switch-window.txt b/documentation/audits/night-2026-09-25/D/D3-switch-window.txt new file mode 100644 index 00000000..49dc7a1f --- /dev/null +++ b/documentation/audits/night-2026-09-25/D/D3-switch-window.txt @@ -0,0 +1,19 @@ +# D3 switch / window / schedule per box — 2026-09-24T20:59:29+00:00 +== felhom-pve +app_update key: ABSENT = ON by default +window: absent +2026/09/24 20:59:14 [INFO] Self-update enabled (check every 6h, auto-update: false, auto-update time: 04:30) +2026/09/24 20:59:14 [INFO] [scheduler] Daily job db-dump scheduled for 2026-09-25 02:30 CEST +2026/09/24 20:59:14 [INFO] [scheduler] Daily job tier2-backup scheduled for 2026-09-25 03:30 CEST +2026/09/24 20:59:14 [INFO] [scheduler] Daily job offbox-backup scheduled for 2026-09-25 04:15 CEST +== hp +app_update key: ABSENT = ON by default +window: absent +2026/09/24 20:59:16 main.go:754: [INFO] Self-update enabled (check every 6h, auto-update: false, auto-update time: 04:30) +2026/09/24 20:59:16 scheduler.go:132: [INFO] [scheduler] Daily job db-dump scheduled for 2026-09-25 02:30 CEST +2026/09/24 20:59:16 scheduler.go:132: [INFO] [scheduler] Daily job tier2-backup scheduled for 2026-09-25 03:30 CEST +2026/09/24 20:59:16 scheduler.go:132: [INFO] [scheduler] Daily job offbox-backup scheduled for 2026-09-25 04:15 CEST +hu ('Rendszer konfiguráció', 'Az üzemeltető által beállított értékek. Módosításhoz kérd az üzemeltetőt.', 'true') +?lang=en ('System configuration', 'Values set by the operator. Ask the operator to change them.', 'true') +hu id="app-update"> Alkalmazások automatikus frissítése Ha be van kapcsolva, a doboz minden éjjel, a távoli mentés után magától frissíti az alkalmazásaidat: egyszerre egyet, mindig csak kipróbált lépéssel, és ha valami nem sikerül, visszaállítja az előző változatot. Automatikus frissítés bekapcsolva Mentés Hálózat Helyi cím (LAN) — nem állapítható meg Átjáró — nem állapítható meg < | checked +?lang=en id="app-update"> Automatic app updates When this is on, the box updates your apps by itself each night, after the off-site backup: one app at a time, only with a tested step, and if something fails it puts the previous version back. Automatic updates on Save Network Local address (LAN) — cannot be determined Gateway — cannot be determined

] 10.15MB/10.15MB\n c16728afba34 Pull complete \n eed4985bb3f6 Extracting [========> ] 32.77kB/199.8kB\n eed4985bb3f6 Extracting [==================================================>] 199.8kB/199.8kB\n eed4985bb3f6 Extracting [==================================================>] 199.8kB/199.8kB\n eed4985bb3f6 Pull complete \n 475391c2daa8 Extracting [========> ] 32.77kB/200.6kB\n 475391c2daa8 Extracting [==================================================>] 200.6kB/200.6kB\n 475391c2daa8 Extracting [==================================================>] 200.6kB/200.6kB\n 475391c2daa8 Pull complete \n n8n Pulled \nError response from daemon: network traefik-public not found\n", + "engine_state_after": null, + "memory": null, + "marks": [], + "duration_s": 46.7, + "measured_at": "2026-09-24T21:02:47Z", + "evidence": "evidence/MV-n8n", + "scratch_cleared": [], + "total_s": 46.7 +} diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/compose-final.log b/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/compose-final.log new file mode 100644 index 00000000..e69de29b diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/run.log b/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/run.log new file mode 100644 index 00000000..a1a57b29 --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/run.log @@ -0,0 +1,3 @@ +[21:02:01] scratch drive folders cleared before FROM (R-656): none existed +[21:02:01] MV-n8n: deploying n8n at FROM {'n8n': 'n8nio/n8n:2.41.1'} +[21:02:47] FROM deploy FAILED — inconclusive, the edge was never reached \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/verdict.json b/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/verdict.json new file mode 100644 index 00000000..3529430b --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n-control/bench/evidence/MV-n8n/verdict.json @@ -0,0 +1,27 @@ +{ + "harness_version": 3, + "edge": "MV-n8n", + "app": "n8n", + "note": "night 2026-09-23 within-a-major move: n8n=alpine:3.20", + "from": { + "n8n": "n8nio/n8n:2.41.1" + }, + "to": { + "n8n": "alpine:3.20" + }, + "verdict": "inconclusive", + "seed_read_before": false, + "seed_read_after": false, + "healthy_after": false, + "migration_observed": null, + "abort": "not-attempted", + "abort_detail": "FROM deploy failed rc=1: ===========================================>] 10.15MB/10.15MB\n c16728afba34 Pull complete \n eed4985bb3f6 Extracting [========> ] 32.77kB/199.8kB\n eed4985bb3f6 Extracting [==================================================>] 199.8kB/199.8kB\n eed4985bb3f6 Extracting [==================================================>] 199.8kB/199.8kB\n eed4985bb3f6 Pull complete \n 475391c2daa8 Extracting [========> ] 32.77kB/200.6kB\n 475391c2daa8 Extracting [==================================================>] 200.6kB/200.6kB\n 475391c2daa8 Extracting [==================================================>] 200.6kB/200.6kB\n 475391c2daa8 Pull complete \n n8n Pulled \nError response from daemon: network traefik-public not found\n", + "engine_state_after": null, + "memory": null, + "marks": [], + "duration_s": 46.7, + "measured_at": "2026-09-24T21:02:47Z", + "evidence": "evidence/MV-n8n", + "scratch_cleared": [], + "total_s": 46.7 +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/bench/MV-n8n.log b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/MV-n8n.log new file mode 100644 index 00000000..cd3ff6aa --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/MV-n8n.log @@ -0,0 +1,30 @@ +[21:02:58] scratch drive folders cleared before FROM (R-656): none existed +[21:02:58] MV-n8n: deploying n8n at FROM {'n8n': 'n8nio/n8n:2.41.1'} +[21:02:58] FROM deploy FAILED — inconclusive, the edge was never reached +{ + "harness_version": 3, + "edge": "MV-n8n", + "app": "n8n", + "note": "night 2026-09-23 within-a-major move: n8n=n8nio/n8n:2.41.2", + "from": { + "n8n": "n8nio/n8n:2.41.1" + }, + "to": { + "n8n": "n8nio/n8n:2.41.2" + }, + "verdict": "inconclusive", + "seed_read_before": false, + "seed_read_after": false, + "healthy_after": false, + "migration_observed": null, + "abort": "not-attempted", + "abort_detail": "FROM deploy failed rc=1: Error response from daemon: network traefik-public not found\n", + "engine_state_after": null, + "memory": null, + "marks": [], + "duration_s": 0.1, + "measured_at": "2026-09-24T21:02:58Z", + "evidence": "evidence/MV-n8n", + "scratch_cleared": [], + "total_s": 0.1 +} diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/compose-final.log b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/compose-final.log new file mode 100644 index 00000000..e69de29b diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/run.log b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/run.log new file mode 100644 index 00000000..9fe20a5b --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/run.log @@ -0,0 +1,3 @@ +[21:02:58] scratch drive folders cleared before FROM (R-656): none existed +[21:02:58] MV-n8n: deploying n8n at FROM {'n8n': 'n8nio/n8n:2.41.1'} +[21:02:58] FROM deploy FAILED — inconclusive, the edge was never reached \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/verdict.json b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/verdict.json new file mode 100644 index 00000000..ea53b9c4 --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n/bench/evidence/MV-n8n/verdict.json @@ -0,0 +1,27 @@ +{ + "harness_version": 3, + "edge": "MV-n8n", + "app": "n8n", + "note": "night 2026-09-23 within-a-major move: n8n=n8nio/n8n:2.41.2", + "from": { + "n8n": "n8nio/n8n:2.41.1" + }, + "to": { + "n8n": "n8nio/n8n:2.41.2" + }, + "verdict": "inconclusive", + "seed_read_before": false, + "seed_read_after": false, + "healthy_after": false, + "migration_observed": null, + "abort": "not-attempted", + "abort_detail": "FROM deploy failed rc=1: Error response from daemon: network traefik-public not found\n", + "engine_state_after": null, + "memory": null, + "marks": [], + "duration_s": 0.1, + "measured_at": "2026-09-24T21:02:58Z", + "evidence": "evidence/MV-n8n", + "scratch_cleared": [], + "total_s": 0.1 +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/box/badges.json b/documentation/audits/night-2026-09-25/E/apps/n8n/box/badges.json new file mode 100644 index 00000000..c265cd56 --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n/box/badges.json @@ -0,0 +1,21 @@ +{ + "before": { + "hu": [], + "en": [] + }, + "after": { + "hu": [ + { + "title": "Újabb változat érhető el ehhez az alkalmazáshoz. A frissítés indításához nyomd meg a Frissítés gombot.", + "text": "Frissítés elérhető — ma" + } + ], + "en": [ + { + "title": "A newer version of this app is available. Select the Update button to start it.", + "text": "Update available — today" + } + ] + }, + "drill_commit": "b18e8810399b" +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/apps/n8n/box/observables-before.json b/documentation/audits/night-2026-09-25/E/apps/n8n/box/observables-before.json new file mode 100644 index 00000000..c1558573 --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/apps/n8n/box/observables-before.json @@ -0,0 +1,15 @@ +{ + "pinned_images": { + "n8n": "n8nio/n8n:2.41.1" + }, + "installed_images": { + "n8n": "n8nio/n8n:2.41.1" + }, + "catalog_images": null, + "live_compose_image_lines": [ + "image: n8nio/n8n:2.41.1@sha256:52f147bba97908fe58e616738f2f0485696ac7b5670a1baf169ccf1ff3173de7" + ], + "docker_inspect": [ + "n8n n8nio/n8n:2.41.1@sha256:52f147bba97908fe58e616738f2f0485696ac7b5670a1baf169ccf1ff3173de7 running=true restarts=0" + ] +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-25/E/benchq-Q1.txt b/documentation/audits/night-2026-09-25/E/benchq-Q1.txt new file mode 100644 index 00000000..3e55a99a --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/benchq-Q1.txt @@ -0,0 +1,5 @@ +23:01:55 START n8n ['n8n=alpine:3.20'] +23:02:52 DONE n8n rc=0 verdict=inconclusive read_before=False read_after=False abort=not-attempted marks=[] peaks={} detail='FROM deploy failed rc=1: ===========================================>] 10.15MB/10.15MB\n c16728afba34 Pull complete \n eed4985bb3f6 Extracting [========> ' (57s) +23:02:52 START n8n ['n8n=n8nio/n8n:2.41.2'] +23:03:02 DONE n8n rc=0 verdict=inconclusive read_before=False read_after=False abort=not-attempted marks=[] peaks={} detail='FROM deploy failed rc=1: Error response from daemon: network traefik-public not found\n' (10s) +23:03:02 START mealie ['mealie=ghcr.io/mealie-recipes/mealie:v3.28.0'] diff --git a/documentation/audits/night-2026-09-25/E/drift.py b/documentation/audits/night-2026-09-25/E/drift.py new file mode 100644 index 00000000..5b1c1812 --- /dev/null +++ b/documentation/audits/night-2026-09-25/E/drift.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Measure upstream drift for app-catalog-felhom.eu image pins. Read-only, DooPlex-only. + +Reuses the auth-handling approach of scripts/check-image-resolvable.py (per-registry anonymous +bearer-token flow against the Docker Registry v2 API), but instead of `docker manifest inspect` +per-ref, lists the full upstream tag catalogue so we can compare the pinned tag against the newest +one available, and classify the step as same-major (minor/patch) or major. +""" +import json +import re +import sys +import time +from pathlib import Path + +import requests + +SCRATCH = Path(".") + +SESSION = requests.Session() +SESSION.headers.update({"User-Agent": "felhom-catalog-drift-audit/1.0 (read-only measurement)"}) + + +def get_bearer_token(www_auth: str) -> str: + assert www_auth.startswith("Bearer ") + parts = www_auth[len("Bearer "):].split(",") + d = {} + for p in parts: + k, v = p.split("=", 1) + d[k] = v.strip('"') + r = SESSION.get(d["realm"], params={"service": d.get("service"), "scope": d.get("scope")}, timeout=20) + r.raise_for_status() + return r.json()["token"] + + +def tags_list_all(host: str, repo: str, max_pages=1000): + """Full tag list via Docker Registry v2 API, following RFC5988 Link pagination.""" + url = f"https://{host}/v2/{repo}/tags/list" + params = {"n": 1000} + r = SESSION.get(url, params=params, timeout=30) + headers = {} + if r.status_code == 401: + tok = get_bearer_token(r.headers["WWW-Authenticate"]) + headers = {"Authorization": f"Bearer {tok}"} + r = SESSION.get(url, params=params, headers=headers, timeout=30) + r.raise_for_status() + j = r.json() + tags = list(j.get("tags") or []) + link = r.headers.get("Link") + pages = 1 + while link and pages < max_pages: + m = re.search(r'<([^>]+)>;\s*rel="next"', link) + if not m: + break + nexturl = m.group(1) + if nexturl.startswith("/"): + nexturl = f"https://{host}{nexturl}" + r = SESSION.get(nexturl, headers=headers, timeout=30) + r.raise_for_status() + j = r.json() + tags.extend(j.get("tags") or []) + link = r.headers.get("Link") + pages += 1 + return tags + + +def manifest_digest(host: str, repo: str, ref: str): + """Resolve one ref's manifest digest (for floating-tag drift: 'has the tag moved'). Read-only.""" + url = f"https://{host}/v2/{repo}/manifests/{ref}" + accept = ("application/vnd.docker.distribution.manifest.v2+json," + "application/vnd.docker.distribution.manifest.list.v2+json," + "application/vnd.oci.image.manifest.v1+json," + "application/vnd.oci.image.index.v1+json") + r = SESSION.head(url, headers={"Accept": accept}, timeout=30) + if r.status_code == 401: + tok = get_bearer_token(r.headers["WWW-Authenticate"]) + r = SESSION.head(url, headers={"Accept": accept, "Authorization": f"Bearer {tok}"}, timeout=30) + if r.status_code != 200: + return None, f"HTTP {r.status_code}" + return r.headers.get("Docker-Content-Digest"), None + + +# host resolution per repo prefix, mirroring how these refs are actually pulled +def resolve_host_repo(ref_repo: str): + if ref_repo.startswith("ghcr.io/"): + return "ghcr.io", ref_repo[len("ghcr.io/"):] + if ref_repo.startswith("lscr.io/"): + return "lscr.io", ref_repo[len("lscr.io/"):] + if ref_repo.startswith("registry.gitlab.com/"): + return "registry.gitlab.com", ref_repo[len("registry.gitlab.com/"):] + if ref_repo.startswith("gitea.dooplex.hu/"): + return "gitea.dooplex.hu", ref_repo[len("gitea.dooplex.hu/"):] + if ref_repo.startswith("quay.io/"): + return "quay.io", ref_repo[len("quay.io/"):] + # Docker Hub + if "/" not in ref_repo: + return "registry-1.docker.io", f"library/{ref_repo}" + return "registry-1.docker.io", ref_repo + + +VERSION_CORE_RE = re.compile(r'(\d+(?:\.\d+){1,3})') +UNSTABLE_MARKERS = ("rc", "beta", "alpha", "dev", "nightly", "canary", "edge", "preview", "snapshot", "-pr", "test") + + +def split_tag(tag: str): + """(prefix, core_version_tuple, suffix) — first maximal dotted-numeric run of len>=2 is the core.""" + m = VERSION_CORE_RE.search(tag) + if not m: + return None + prefix = tag[:m.start()] + core = tuple(int(x) for x in m.group(1).split(".")) + suffix = tag[m.end():] + return prefix, core, suffix + + +def is_unstable(tag: str) -> bool: + low = tag.lower() + return any(mk in low for mk in UNSTABLE_MARKERS) + + +def newest_matching(all_tags, current_tag): + """Find the newest tag sharing the current tag's prefix/suffix 'shape', by core version tuple.""" + cur = split_tag(current_tag) + if not cur: + return None, "current tag has no parseable version core" + cur_prefix, cur_core, cur_suffix = cur + + # suffix "shape": if suffix contains a long hex-looking token, treat it as a wildcard hash + def suffix_shape(suf): + if re.search(r'[0-9a-f]{7,40}', suf.lower()) and re.search(r'[a-f]', suf.lower()): + return re.sub(r'[0-9a-f]{7,40}', '', suf.lower()) + return suf + + cur_shape = suffix_shape(cur_suffix) + + best = None + best_core = None + for t in all_tags: + if t == current_tag: + continue + if is_unstable(t): + continue + parsed = split_tag(t) + if not parsed: + continue + p, core, suf = parsed + if p != cur_prefix: + continue + if suffix_shape(suf) != cur_shape: + continue + if len(core) != len(cur_core): + continue + # Some registries (linuxserver.io/lscr.io in particular) publish EXTRA tags for a + # same-app-version base-image rebuild, shaped like the release tag plus a YYYYMMDD-ish + # trailing component (e.g. bookstack 26.05.2 coexists with 26.05.20260608). That is not a + # newer application release — it is a rebuild of an existing one — so any component that + # looks like a calendar date (>= 20000, comfortably above any real release counter we saw + # across all 58 pinned-semver images, all of which stayed under 10000) is excluded as a + # candidate "newest" rather than misread as a huge version jump. + if any(c >= 20000 for c in core): + continue + if best_core is None or core > best_core: + best_core = core + best = t + if best is None: + return None, "no matching newer tag found (same prefix/suffix shape)" + if best_core <= cur_core: + return None, "up to date (no tag newer than current within the same shape)" + return (best, best_core, cur_core), None + + + +def newest_same_major(all_tags, current_tag): + cur = split_tag(current_tag) + if not cur: return None + p0, c0, s0 = cur + best=None; bc=None + for t in all_tags: + if t == current_tag or is_unstable(t): continue + pr = split_tag(t) + if not pr: continue + p, c, s = pr + if p != p0 or s != s0 or len(c) != len(c0) or c[0] != c0[0]: continue + if any(x >= 20000 for x in c): continue + if c <= c0: continue + if bc is None or c > bc: bc=c; best=t + return best + +def main(): + import glob, yaml + pins = {} + for f in sorted(glob.glob("/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/templates/*/docker-compose.yml")): + app = f.split("/")[-2] + d = yaml.safe_load(open(f)) + for svc, sd in (d.get("services") or {}).items(): + img = (sd or {}).get("image") + if not img or "${" in img: continue + repo, _, tag = img.rpartition(":") if ":" in img.split("/")[-1] else (img, "", "latest") + pins.setdefault(img, {"repo": repo, "tag": tag, "apps": []})["apps"].append(f"{app}/{svc}") + out = {} + for i,(ref,p) in enumerate(sorted(pins.items())): + e = dict(ref=ref, **p) + host, hrepo = resolve_host_repo(p["repo"]) + if p["repo"].startswith("gitea.dooplex.hu"): + e["status"]="internal"; out[ref]=e; continue + try: + tags = tags_list_all(host, hrepo) + res, err = newest_matching(tags, p["tag"]) + e["newest_any"] = res[0] if res else None + e["newest_same_major"] = newest_same_major(tags, p["tag"]) + e["status"] = "behind" if e["newest_any"] else "current-or-unparsed" + except Exception as ex: + e["status"]="error"; e["error"]=f"{type(ex).__name__}: {ex}" + print(f"[{i+1}/{len(pins)}] {ref}: same-major={e.get('newest_same_major')} any={e.get('newest_any')} {e.get('error','')}", flush=True) + out[ref]=e + time.sleep(0.15) + json.dump(out, open("02-drift.json","w"), indent=2) + +if __name__ == "__main__": + main() diff --git a/documentation/audits/night-2026-09-25/F/F1-agent-refusal-live.txt b/documentation/audits/night-2026-09-25/F/F1-agent-refusal-live.txt new file mode 100644 index 00000000..eed7f4c7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/F1-agent-refusal-live.txt @@ -0,0 +1,32 @@ +# F live: agent 0.134.0-livetest (backupSpaceGrowth=10, build-only patch) selftest=backup of 9201 on demo-hp — must REFUSE, start nothing — 2026-09-24T20:40:20+00:00 +local before: /dev/mapper/pve-root 41424257024 23269031936 16018018304 60% / +archives before: 1 + "crash_consistent": true, + "size_bytes": 0, + "success": false, + "error": "proxmox: task UPID:demo-hp:000E124D:119D831D:6AB58AB5:vzdump:9201:felhom-agent@pve!agent: failed: exitstatus \"job errors\"", + "started_at": "2026-09-24T20:40:21Z", + "duration_seconds": 317.416191027, + "uncovered_volumes": [ + "/etc/felhom-bootstrap", + "/mnt/felhom-drives" + ] + } + [FAIL] backup: backup: vzdump task vmid 9201: proxmox: task UPID:demo-hp:000E124D:119D831D:6AB58AB5:vzdump:9201:felhom-agent@pve!agent: failed: exitstatus "job errors" +selftest_rc=1 +local after: /dev/mapper/pve-root 41424257024 23269306368 16017743872 60% / +archives after: 1 +vzdump tasks in the last 5 min: 0 + +## RESULT: THE PREFLIGHT DID NOT REFUSE — a real vzdump of 9201 started (22:40:21 CEST) and was ABORTED by CC +- The livetest (growth ×10 → need ≈ 77 GiB, 16 GiB free) should have refused. It did not: `spaceFits` reads free + space from `BackupAPI.ListStorage` = `GET /storage`, the cluster storage DEFINITIONS, which carry NO usage + (avail 0) → the check's own fail-open ("an unknown avail never refuses") let the backup run. The unit tests' + fake supplied `Avail` on that list, which production never does — a seam that lied. +- 22:45:3x CEST: CC stopped the task (`pvesh delete /nodes/demo-hp/tasks/`): vzdump log "interrupted by + signal", the `.tmp`/`.dat` removed by vzdump, NO new archive (`pvesm list local`: the 21:59 one only), root + back to 60 % (16.0 GB free), no lock on 9201, 9201 running. Snapshot mode: the guest was suspended 2 s + ("guest is online again after 2 seconds"); its apps were NOT stopped. +- This breached the brief's fence "no whole-box backup on a demo box except Part A4's single proof" for 5 min 16 s. +- Fix (same release, before any tag): read free space from `GET /nodes//storage` (`NodeStorage`); the test + fake now serves avail ONLY there, like production. diff --git a/documentation/audits/night-2026-09-25/F/F2-agent-preflight-live.txt b/documentation/audits/night-2026-09-25/F/F2-agent-preflight-live.txt new file mode 100644 index 00000000..370aab3e --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/F2-agent-preflight-live.txt @@ -0,0 +1,10 @@ +# F2 live, SAFE builds (a hard stop before Vzdump in both) — 2026-09-24T20:47:40+00:00 +=== livetest2 (livetest2 = growth x10 → must REFUSE; livetest3 = the release margin x1.25 → must PASS and stop at the livetest guard) +time=2026-09-24T22:47:40.761+02:00 level=WARN msg="backup SKIPPED by the space preflight (R-685) — nothing was started" vmid=9201 target=local reason="local has 14.9 GiB free; the last archive of guest 9201 was 7.6 GiB, so a new one needs about 77.2 GiB (old archives are removed only after a successful backup)" + "error": "skipped: not enough space: local has 14.9 GiB free; the last archive of guest 9201 was 7.6 GiB, so a new one needs about 77.2 GiB (old archives are removed only after a successful backup)", + [FAIL] backup: backup: skipped: not enough space: local has 14.9 GiB free; the last archive of guest 9201 was 7.6 GiB, so a new one needs about 77.2 GiB (old archives are removed only after a successful backup) +=== livetest3 (livetest2 = growth x10 → must REFUSE; livetest3 = the release margin x1.25 → must PASS and stop at the livetest guard) +time=2026-09-24T22:47:41.341+02:00 level=INFO msg="backup: space preflight passed" vmid=9201 target=local last_archive_bytes=8182759056 need_bytes=11302190644 avail_bytes=16017731584 + [FAIL] backup: LIVETEST: the space preflight PASSED — stopping before vzdump +archives: 1 +0 diff --git a/documentation/audits/night-2026-09-25/F/F3-sign-hp.txt b/documentation/audits/night-2026-09-25/F/F3-sign-hp.txt new file mode 100644 index 00000000..1a128f16 --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/F3-sign-hp.txt @@ -0,0 +1,26 @@ +# F3 demo-hp-bb76ea agent_update 0.134.0 — sha256 verified against the published package: 7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d +## 2026-09-24T20:49:37+00:00 before: felhom-agent 0.133.0 +signed: op=agent_update host=demo-hp-bb76ea guest="" key_id=felhom-op-1 nonce=af8e931008f76e52a819886d6cd06b2b expires=2026-09-24T21:34:37Z +wrote envelope to /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/c9309498-57c3-4b75-aa79-a24434763a8b/scratchpad/au134-hp.json +uploaded signed op to the hub jobs queue +rc=0 +## 2026-09-24T20:58:19+00:00 hp now runs 0.134.0 - waiting 80 s for dwell/commit + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.433+02:00 level=INFO msg="audit: gate decision" class=agent_update host=demo-hp-bb76ea guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=af8e9310… durable_id="" + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.433+02:00 level=INFO msg="gate decision" class=agent_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.433+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=4a61ede14f397a68 op=agent_update key_id=felhom-op-1 nonce=af8e931008f76e52a819886d6cd06b2b + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.434+02:00 level=WARN msg="agent_update: downloading operator-signed binary" version=0.134.0 url=https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.134.0/felhom-agent sha256=7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.694+02:00 level=WARN msg="agent_update: handing staged binary to the guarded wrapper" staged=/var/lib/felhom-agent/selfupdate/felhom-agent-0.134.0 version=0.134.0 + [hp] Sep 24 22:57:49 demo-hp sudo[995535]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded apply /var/lib/felhom-agent/selfupdate/felhom-agent-0.134.0 7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d + [hp] Sep 24 22:57:49 demo-hp felhom-selfupdate-guarded[995564]: applied 0.134.0 (prev 0.133.0, sha 7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d); detached restart scheduled + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.877+02:00 level=WARN msg="agent_update: apply handed off; restart scheduled" version=0.134.0 wrapper="" + [hp] Sep 24 22:57:49 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:49.877+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=4a61ede14f397a68 op=agent_update + [hp] Sep 24 22:57:52 demo-hp felhom-agent[756657]: time=2026-09-24T22:57:52.005+02:00 level=INFO msg="backup: restore-test scheduler shutting down" reason="context canceled" + [hp] Sep 24 22:57:53 demo-hp felhom-agent[995644]: time=2026-09-24T22:57:53.035+02:00 level=INFO msg="backup: restore-test scheduler starting (per-archive due-check)" eval_interval=6h0m0s settle=24h0m0s + [hp] Sep 24 22:57:53 demo-hp felhom-agent[995644]: time=2026-09-24T22:57:53.898+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s + [hp] Sep 24 22:57:53 demo-hp felhom-agent[995644]: time=2026-09-24T22:57:53.898+02:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.134.0 prev=0.133.0 dwell=1m0s + [hp] Sep 24 22:58:53 demo-hp sudo[1001327]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit + [hp] Sep 24 22:58:53 demo-hp felhom-agent[995644]: time=2026-09-24T22:58:53.923+02:00 level=WARN msg="selfupdate: update committed" version=0.134.0 wrapper="" + [hp] active + [hp] felhom-agent 0.134.0 + demo-felhom-8363b5 Demo Ügyfél 0.134.0 floor: declared MinAg + demo-hp-bb76ea Demo HP 0.134.0 floor: declared MinAgent ONLI diff --git a/documentation/audits/night-2026-09-25/F/F3-sign-n100.txt b/documentation/audits/night-2026-09-25/F/F3-sign-n100.txt new file mode 100644 index 00000000..17628d73 --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/F3-sign-n100.txt @@ -0,0 +1,24 @@ +# F3 demo-felhom-8363b5 agent_update 0.134.0 — sha256 verified against the published package: 7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d +## 2026-09-24T20:49:36+00:00 before: felhom-agent 0.133.0 +signed: op=agent_update host=demo-felhom-8363b5 guest="" key_id=felhom-op-1 nonce=d5984dc1ae833ab3fa6292b84f2f83df expires=2026-09-24T21:34:37Z +wrote envelope to /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/c9309498-57c3-4b75-aa79-a24434763a8b/scratchpad/au134-n100.json +uploaded signed op to the hub jobs queue +rc=0 +## 2026-09-24T20:51:47+00:00 felhom-pve now runs 0.134.0 - waiting 80 s for dwell/commit + [felhom-pve] Sep 24 22:51:41 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:41.639+02:00 level=INFO msg="audit: gate decision" class=agent_update host=demo-felhom-8363b5 guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=d5984dc1… durable_id="" + [felhom-pve] Sep 24 22:51:41 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:41.639+02:00 level=INFO msg="gate decision" class=agent_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed + [felhom-pve] Sep 24 22:51:41 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:41.639+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=d3e7f5839bc8bf5c op=agent_update key_id=felhom-op-1 nonce=d5984dc1ae833ab3fa6292b84f2f83df + [felhom-pve] Sep 24 22:51:41 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:41.639+02:00 level=WARN msg="agent_update: downloading operator-signed binary" version=0.134.0 url=https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.134.0/felhom-agent sha256=7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d + [felhom-pve] Sep 24 22:51:41 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:41.864+02:00 level=WARN msg="agent_update: handing staged binary to the guarded wrapper" staged=/var/lib/felhom-agent/selfupdate/felhom-agent-0.134.0 version=0.134.0 + [felhom-pve] Sep 24 22:51:41 demo-felhom sudo[3925617]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded apply /var/lib/felhom-agent/selfupdate/felhom-agent-0.134.0 7593bebe03234c7d22f3ade384e7ed7787dc659aa8c8594b3ee19af2ce81c72d + [felhom-pve] Sep 24 22:51:42 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:42.013+02:00 level=WARN msg="agent_update: apply handed off; restart scheduled" version=0.134.0 wrapper="" + [felhom-pve] Sep 24 22:51:42 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:42.013+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=d3e7f5839bc8bf5c op=agent_update + [felhom-pve] Sep 24 22:51:44 demo-felhom felhom-agent[3860787]: time=2026-09-24T22:51:44.134+02:00 level=INFO msg="backup: restore-test scheduler shutting down" reason="context canceled" + [felhom-pve] Sep 24 22:51:44 demo-felhom felhom-agent[3925782]: time=2026-09-24T22:51:44.757+02:00 level=INFO msg="backup: restore-test scheduler starting (per-archive due-check)" eval_interval=6h0m0s settle=24h0m0s + [felhom-pve] Sep 24 22:51:45 demo-felhom felhom-agent[3925782]: time=2026-09-24T22:51:45.908+02:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.134.0 prev=0.133.0 dwell=1m0s + [felhom-pve] Sep 24 22:51:45 demo-felhom felhom-agent[3925782]: time=2026-09-24T22:51:45.908+02:00 level=INFO msg="janitor: starting (restore-test scratch retry + stale-lock sweep)" interval=10m0s + [felhom-pve] Sep 24 22:52:45 demo-felhom sudo[3927072]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit + [felhom-pve] Sep 24 22:52:45 demo-felhom felhom-selfupdate-guarded[3927082]: committed (pending cleared, .prev retained) + [felhom-pve] Sep 24 22:52:45 demo-felhom felhom-agent[3925782]: time=2026-09-24T22:52:45.931+02:00 level=WARN msg="selfupdate: update committed" version=0.134.0 wrapper="" + [felhom-pve] active + [felhom-pve] felhom-agent 0.134.0 diff --git a/documentation/audits/night-2026-09-25/F/redproof-r685-liststorage.txt b/documentation/audits/night-2026-09-25/F/redproof-r685-liststorage.txt new file mode 100644 index 00000000..aabfbaa9 --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/redproof-r685-liststorage.txt @@ -0,0 +1,7 @@ +# red-proof R-685 seam: spaceFits reading ListStorage (GET /storage, no usage) instead of NodeStorage +=== RUN TestR685_BackupThatCannotFitIsSkipped + r685_space_test.go:36: a vzdump was started on a target that cannot hold it: [{VMID:9201 Storage:local Mode:snapshot Compress: Notes: PruneBackups:keep-last=1}] +--- FAIL: TestR685_BackupThatCannotFitIsSkipped (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/backup 0.007s +FAIL diff --git a/documentation/audits/night-2026-09-25/F/redproof-r685-nocheck.txt b/documentation/audits/night-2026-09-25/F/redproof-r685-nocheck.txt new file mode 100644 index 00000000..3293dff0 --- /dev/null +++ b/documentation/audits/night-2026-09-25/F/redproof-r685-nocheck.txt @@ -0,0 +1,7 @@ +# red-proof R-685: the spaceFits call dropped from backup() +=== RUN TestR685_BackupThatCannotFitIsSkipped + r685_space_test.go:36: a vzdump was started on a target that cannot hold it: [{VMID:9201 Storage:local Mode:snapshot Compress: Notes: PruneBackups:keep-last=1}] +--- FAIL: TestR685_BackupThatCannotFitIsSkipped (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/backup 0.009s +FAIL diff --git a/documentation/audits/night-2026-09-25/PROGRESS.md b/documentation/audits/night-2026-09-25/PROGRESS.md new file mode 100644 index 00000000..184c25b7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/PROGRESS.md @@ -0,0 +1,20 @@ +# PROGRESS — night 2026-09-25 (resume here) + +Baselines verified 21:24 CEST: controller 1495ca43fb86 (v0.270.0), agent 4fe2f81a322b (v0.133.0), felhom.eu 75ff26408a3c, catalog c8025093d23c. Trees clean. +Capacity: phase0-capacity.txt. demo-felhom pool 3.13 %, root 31 %. demo-hp pool 59.01 % data / 2.65 % meta, root (local) 84.33 %. 9202 lives on nvme-scratch (dir, 7.6 %). + +- [x] Phase 0 baselines + capacity +- [x] drill repo = live main c802509, actions off (phase0-drill-reset.txt); Part C commits on top +- [x] A1 demo-felhom 0.133.0 committed 21:35:09 CEST; hub reads 0.133.0 +- [x] A1 demo-hp 0.133.0 committed 21:50:02 CEST; hub reads 0.133.0 +- [x] A2 restore test on, both (agent.json.pre-r672 back; -1 copy kept as agent.json.night-0925-off) +- [x] A3 demo-felhom PASS 85 s pool 3.13→5.68→3.13; demo-hp REFUSED (needs 30.3 GiB, has 22.1), rc 4 +- [x] A4 retention 3→1 (agent.json.pre-a4-retention), pvesm prune keep-last=1, backup by product trigger OK (8.18 GB local + PBS), R-685 filed +- [x] B controller v0.271.0 pushed 9cf13a3, image built+pushed, 11 red-proofs red +- register: R-685 added, R-676 updated; TO CLOSE after C: R-680, R-678, R-643, R-684, R-672/673 (delivered), R-450? (slice 6 automatic part) +- [x] C night1 (done 2, undone 1, skip needs_person), night2 (done 2, failed_before), night3 kill (step put back, leg not resumed R-686), night4 power cut (resumed, done), data read back every time +- [~] C night5 switch OFF; night6 switch ON (vikunja re-tested). 9202 window before = 02:30; controller.yaml.pre-night0925 saved +- [x] F agent v0.134.0 released 0722b2c/309e368 (sha 7593bebe…); INCIDENT: first build read GET /storage → real vzdump 9201 started 22:40:21, aborted 22:45:37, no archive (F1). Signed both boxes 20:49Z; demo-felhom committed 22:52:45 +- register so far: closed R-672 R-673 R-684 R-680 R-678; opened R-685 R-686 R-687; R-676 updated. Before: 341 rows / 685,148 B +- [ ] D floor + real night +- [ ] E / F / G diff --git a/documentation/audits/night-2026-09-25/REPORT-DRAFT-AB.md b/documentation/audits/night-2026-09-25/REPORT-DRAFT-AB.md new file mode 100644 index 00000000..94afeafb --- /dev/null +++ b/documentation/audits/night-2026-09-25/REPORT-DRAFT-AB.md @@ -0,0 +1,31 @@ +## Part A — the agent, the restore test, the HP backup + +**A1 — agent v0.133.0 signed and delivered, one box after the other** (ruling 1, 2026-09-16). Package sha256 +`3aa30345…e69b6` verified against the published package before signing (`A/A1-*.txt`). + +| box | signed (UTC) | box took it | AUTHORIZED → COMPLETED | committed | hub reads | +|---|---|---|---|---|---| +| demo-felhom-8363b5 | 19:25:30 | 21:34:05 CEST | job `c4b592ff7c9ff307` | 21:35:09 | 0.133.0 | +| demo-hp-bb76ea | 19:35:55 | 21:48:58 CEST | job `36aab1ed8ad00979` | 21:50:02 | 0.133.0 | + +Peti's box: never signed for. + +**A2 — the restore test back on, both boxes.** `agent.json.pre-r672` copied back; the diff was only +`restore_test_eval_interval_seconds: -1` (and the trailing newline). The `-1` config is kept as +`agent.json.night-0925-off`. Start-up line on both: `restore-test scheduler starting (per-archive due-check) +eval_interval=6h0m0s settle=24h0m0s`. + +**A3 — one on-demand restore test.** demo-felhom: **PASS** — scratch 990000 restored, booted, verified, +torn down in 1 min 25 s; pool 3.13 % → peak 5.68 % → 3.13 %; preflight "required 12.8 GB, avail 362.8 GB". +demo-hp: **REFUSED, correctly** — "restoring 21.1 GiB (vzdump log: total bytes written) needs 30.3 GiB free, has +22.1 GiB", exit 4, pool 59.01 % before and after, nothing created. + +**A4 — HP whole-box backups, option A.** Before: root disk (`local`) 89 % used, 4.3 GB free, three 9201 archives +(6.2, 6.3, 7.4 GB), retention 3. Changed: `local_backup_retention` 3 → 1 (saved copy +`agent.json.pre-a4-retention`), agent restarted, the two oldest archives removed with PVE's own +`pvesm prune-backups local --vmid 9201 --keep-last 1` (dry-run shown first) → 58 % used, 16.8 GB free. **Why the +prune by hand:** PVE prunes AFTER a successful backup, so lowering the retention alone would not have let the next +backup fit. One whole-box backup by the product's own button (`POST /api/guest-backup/trigger`, the Mentések +page) — 9201's apps were stopped for it (snapshot mode, resumed at "snapshotted"): local archive **8.18 GB** in +7 min, then the PBS tier (the button covers every tier) 23.2 GB in 4.5 min; the new prune kept 1 archive; root +60 %; all 20 app containers healthy. The product row is **R-685**. diff --git a/documentation/audits/night-2026-09-25/phase0-capacity.txt b/documentation/audits/night-2026-09-25/phase0-capacity.txt new file mode 100644 index 00000000..5b62ef91 --- /dev/null +++ b/documentation/audits/night-2026-09-25/phase0-capacity.txt @@ -0,0 +1,41 @@ +=== felhom-pve 2026-09-24T21:24:59+02:00 +demo-felhom +felhom-agent 0.132.0 +VMID Status Lock Name +9201 running demo-felhom +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +felhom-backup dir active 960303848 11072284 900377052 1.15% +felhom-pbs pbs active 0 0 0 0.00% +local dir active 98497780 28717448 64730784 29.16% +local-lvm lvmthin active 365760512 11448304 354312207 3.13% +Filesystem Size Used Avail Use% Mounted on +/dev/mapper/pve-root 94G 28G 62G 31% / + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + data pve twi-aotz-- 348.82g 3.13 0.58 + root pve -wi-ao---- 96.00g + swap pve -wi-ao---- 8.00g + vm-9201-disk-0 pve Vwi-aotz-- 32.00g data 5.37 + vm-9201-disk-1 pve Vwi-aotz-- 250.00g data 3.67 +=== hp 2026-09-24T21:25:02+02:00 +demo-hp +felhom-agent 0.132.0 +VMID Status Lock Name +9201 running demo-hp +9202 running demo-hp-scratch +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +felhom-pbs pbs active 0 0 0 0.00% +local dir active 40453376 34116176 4250084 84.33% +local-lvm lvmthin active 56487936 33333531 23154404 59.01% +nvme-scratch dir active 983379700 74629616 858723472 7.59% +Filesystem Size Used Avail Use% Mounted on +/dev/mapper/pve-root 39G 33G 4.1G 89% / + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + data pve twi-aotz-- 53.87g 59.01 2.65 + root pve -wi-ao---- 39.50g + swap pve -wi-ao---- 8.00g + vm-9201-disk-0 pve Vwi-aotz-- 32.00g data 5.39 + vm-9201-disk-1 pve Vwi-aotz-- 70.00g data 42.95 +mp0: nvme-scratch:9202/vm-9202-disk-1.raw,mp=/var/lib/felhom,backup=1,size=70G +mp8: /mnt/hdd_1/scratch-drives/scratch_hdd,mp=/mnt/felhom-drives/scratch_hdd +mp9: /var/lib/felhom-agent/guests/9202/bootstrap,mp=/etc/felhom-bootstrap,ro=1 +rootfs: nvme-scratch:9202/vm-9202-disk-0.raw,size=32G diff --git a/documentation/audits/night-2026-09-25/phase0-drill-reset.txt b/documentation/audits/night-2026-09-25/phase0-drill-reset.txt new file mode 100644 index 00000000..79e84fab --- /dev/null +++ b/documentation/audits/night-2026-09-25/phase0-drill-reset.txt @@ -0,0 +1,2 @@ +drill=c8025093d23c live=c8025093d23c — drill already equal to live main, no push needed +has_actions: False private: True diff --git a/documentation/audits/night-2026-09-25/tools/a4_backup.py b/documentation/audits/night-2026-09-25/tools/a4_backup.py new file mode 100644 index 00000000..f1e72906 --- /dev/null +++ b/documentation/audits/night-2026-09-25/tools/a4_backup.py @@ -0,0 +1,35 @@ +"""A4: ONE whole-box backup of demo-hp 9201 by the product's own trigger (POST /api/guest-backup/trigger), +polled through /api/guest-backup/status, with the host root disk sampled. Evidence, not product.""" +import json, time, subprocess, os +os.environ.setdefault("GUEST", "9201") +import walk as w +OUT = w.EV + "/A/A4-hp-backup-run.txt" +def host(cmd): + return subprocess.run(["ssh", "hp", "export LC_ALL=C; " + cmd], capture_output=True, text=True, timeout=120).stdout.strip() +lines = [] +def say(s): + s = time.strftime("%H:%M:%S ") + s + print(s, flush=True); lines.append(s); open(OUT, "w").write("\n".join(lines) + "\n") +w.login() +say("root before: " + host("df -B1 / | tail -1")) +c, d = w.ctl("POST", "/api/guest-backup/trigger") +say(f"POST /api/guest-backup/trigger -> {c} {json.dumps(d, ensure_ascii=False)[:300]}") +peak = 0 +t0 = time.time() +while time.time() - t0 < 3600: + time.sleep(20) + c, d = w.ctl("GET", "/api/guest-backup/status") + used = int(host("df -B1 / | tail -1 | awk '{print $3}'") or 0) + peak = max(peak, used) + st = json.dumps(d, ensure_ascii=False)[:400] + say(f"status {c}: {st} | root used {used}") + dd = (d or {}).get("data") or {} + if isinstance(dd, dict) and not dd.get("running", dd.get("active", True)) and time.time() - t0 > 60: + break +say(f"root peak used bytes: {peak}") +say("root after: " + host("df -B1 / | tail -1")) +say("local archives: " + host("pvesm list local | grep 9201")) +say("host journal: " + host("journalctl -u felhom-agent --since '-70min' --no-pager | grep -iE 'vzdump|backup (done|failed|started)|prune|No space' | tail -15")) +w.login() +c, d = w.ctl("GET", "/api/stacks") +say("apps after: " + str([(s['name'], s.get('state')) for s in (d or {}).get('data') or [] if s.get('deployed')])) diff --git a/documentation/audits/night-2026-09-25/tools/applines.py b/documentation/audits/night-2026-09-25/tools/applines.py new file mode 100644 index 00000000..6eb4383b --- /dev/null +++ b/documentation/audits/night-2026-09-25/tools/applines.py @@ -0,0 +1,13 @@ +"""The app page's lines (auto-update / undone / update error / held) for the four drill apps, both languages.""" +import re, sys +import walk as w +w.login() +out = sys.argv[1] +with open(out, "w") as f: + for lang in ("hu", "en"): + for a in ("wishlist", "navidrome", "romm", "vikunja"): + h = w.page(f"/apps/{a}" + ("?lang=en" if lang == "en" else "")) + m = re.findall(r'

]*data-(?:auto-update|update-undone|update-error|held|unhealthy-stop)[^>]*>(.*?)
', h, re.S) + b = re.findall(r']*>([^<]*)<', h)[:2] + f.write(f"[{lang}] {a}: badge={[x.strip() for x in b]} lines={[re.sub(r'<[^>]+>', '', x).strip() for x in m]}\n") +print(open(out).read()) diff --git a/documentation/audits/night-2026-09-25/tools/benchq.py b/documentation/audits/night-2026-09-25/tools/benchq.py new file mode 100644 index 00000000..c28b9ab7 --- /dev/null +++ b/documentation/audits/night-2026-09-25/tools/benchq.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""benchq.py [,] [ ...] + +Runs `upgrade-test.py --move` on bench LXC 9401 for each app IN ORDER, then copies that edge's +evidence OFF the bench into apps//bench/ before starting the next (R-320: the evidence leaves +the machine at the end of the phase that made it). One line per finished edge in benchq-.txt. +""" +import json, os, subprocess, sys, time +HERE = os.path.dirname(os.path.abspath(__file__)) +BENCH = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/c9309498-57c3-4b75-aa79-a24434763a8b/scratchpad/bench.sh" +q = sys.argv[1] +args = sys.argv[2:] +out = open(os.path.join(HERE, "..", "E", f"benchq-{q}.txt"), "a", buffering=1) +for i in range(0, len(args), 2): + app, moves = args[i], args[i + 1].split(",") + label = app + if "@" in app: # romm@engine: a second edge of one app gets its own folder + app, tag = app.split("@", 1) + label = f"{app}-{tag}" + t0 = time.time() + out.write(f"{time.strftime('%H:%M:%S')} START {app} {moves}\n") + pre = f"cd /opt/upg && [ -d evidence/MV-{app} ] && mv evidence/MV-{app} evidence/MV-{app}-$(date +%s); " + subprocess.run([BENCH], input=pre, capture_output=True, text=True, timeout=120) + cmd = (f"cd /opt/upg && timeout 2400 python3 upgrade-test.py --soak 600 --move {app} {' '.join(moves)} " + f"> /opt/upg/MV-{app}.log 2>&1; echo rc=$?") + r = subprocess.run([BENCH], input=cmd, capture_output=True, text=True, timeout=2700) + dest = os.path.join(HERE, "..", "E", "apps", label, "bench") + os.makedirs(dest, exist_ok=True) + pull = (f"cd /opt/upg && tar czf /tmp/ev-{app}.tgz MV-{app}.log evidence/MV-{app} 2>/dev/null; " + f"base64 -w0 /tmp/ev-{app}.tgz") + r2 = subprocess.run([BENCH], input=pull, capture_output=True, text=True, timeout=600) + import base64, io, tarfile + try: + tarfile.open(fileobj=io.BytesIO(base64.b64decode(r2.stdout.strip()))).extractall(dest) + except Exception as e: + out.write(f" evidence pull FAILED for {app}: {e}\n") + v = {} + try: + v = json.load(open(os.path.join(dest, "evidence", f"MV-{app}", "verdict.json"))) + except Exception: + pass + mem = v.get("memory") or {} + peaks = {n: c.get("peak_pct") for n, c in (mem.get("containers") or {}).items()} + out.write(f"{time.strftime('%H:%M:%S')} DONE {app} {r.stdout.strip()} verdict={v.get('verdict')} " + f"read_before={v.get('seed_read_before')} read_after={v.get('seed_read_after')} " + f"abort={v.get('abort')} marks={v.get('marks')} peaks={peaks} " + f"detail={str(v.get('abort_detail'))[:160]!r} ({round(time.time()-t0)}s)\n") +out.write(f"{time.strftime('%H:%M:%S')} QUEUE {q} FINISHED\n") diff --git a/documentation/audits/night-2026-09-25/tools/c1_state.py b/documentation/audits/night-2026-09-25/tools/c1_state.py new file mode 100644 index 00000000..fc514695 --- /dev/null +++ b/documentation/audits/night-2026-09-25/tools/c1_state.py @@ -0,0 +1,15 @@ +"""Read the four drill apps' state through the product (GET /api/stacks/, the app page badges hu+en).""" +import json, sys, os +import walk as w +w.login(); w.sync_rescan() +out = {} +for a in ["wishlist", "navidrome", "romm", "vikunja"]: + st = w.stack(a) + out[a] = {"state": st.get("state"), "pinned": (st.get("app_config") or {}).get("pinned_images"), + "installed": {k: v.get("ref") for k, v in ((st.get("app_config") or {}).get("installed_images") or {}).items()}, + "catalog": st.get("catalog_images"), "steps_left": st.get("ladder_steps_left"), + "failed_step": (st.get("app_config") or {}).get("failed_update_step"), + "last_auto_update": (st.get("app_config") or {}).get("last_auto_update"), + "badges": w.badges(a)} + print(a, out[a]["pinned"], "steps_left", out[a]["steps_left"], out[a]["badges"]["en"][:1]) +json.dump(out, open(sys.argv[1], "w"), indent=2, ensure_ascii=False) diff --git a/documentation/audits/night-2026-09-25/tools/c_night.py b/documentation/audits/night-2026-09-25/tools/c_night.py new file mode 100644 index 00000000..fa5c5d9c --- /dev/null +++ b/documentation/audits/night-2026-09-25/tools/c_night.py @@ -0,0 +1,33 @@ +"""One simulated night on 9202: move W so the off-site job (and the chained update leg) fires in ~2 min, wait for the +leg's summary line, then capture the controller log of the night, the pages (both languages) and the data readback. +Evidence, not product. Usage: GUEST=9202 python3 c_night.py