From daacf84e30c37e3e5b45a36638547da162a84fb6 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 1 Oct 2026 08:40:04 +0200 Subject: [PATCH] Evidence (rulings 2026-10-01): Parts A-D so far; register: R-743, R-744, R-745, R-746, R-749, R-751 closed Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../A/A0-9202-containers-before.txt | 6 + .../rulings-2026-10-01/A/A1-bench-create.txt | 38 ++ .../rulings-2026-10-01/A/A2-drill-reset.txt | 6 + .../A/A3-9202-repoint-drill.txt | 10 + .../rulings-2026-10-01/B/B0-before-probe.txt | 117 ++++ .../rulings-2026-10-01/B/B1-red-proofs.txt | 33 ++ .../B/B2-9202-deploy-and-pass.txt | 27 + .../B/B2-9202-selfupdate.txt | 0 .../B/B3-demo-boxes-before-floor.txt | 13 + .../audits/rulings-2026-10-01/B/B4-floor.txt | 8 + .../B/B5-demo-boxes-after.txt | 45 ++ .../audits/rulings-2026-10-01/C/C0-drill.txt | 3 + .../C/C1-mealie-lockout-1h.txt | 30 + .../rulings-2026-10-01/D/D1-r746-digest.txt | 16 + .../D/D2-outline-fixture-9202.txt | 15 + .../rulings-2026-10-01/T/T1-bench-destroy.txt | 13 + .../audits/rulings-2026-10-01/tools/b_9202.py | 13 + .../rulings-2026-10-01/tools/b_selfupdate.py | 21 + .../rulings-2026-10-01/tools/c_mealie.py | 54 ++ .../rulings-2026-10-01/tools/d_outline.py | 19 + .../rulings-2026-10-01/tools/repoint.py | 60 ++ .../audits/rulings-2026-10-01/tools/walk.py | 560 ++++++++++++++++++ documentation/backlog/OPEN-ITEMS.md | 12 +- 23 files changed, 1113 insertions(+), 6 deletions(-) create mode 100644 documentation/audits/rulings-2026-10-01/A/A0-9202-containers-before.txt create mode 100644 documentation/audits/rulings-2026-10-01/A/A1-bench-create.txt create mode 100644 documentation/audits/rulings-2026-10-01/A/A2-drill-reset.txt create mode 100644 documentation/audits/rulings-2026-10-01/A/A3-9202-repoint-drill.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B0-before-probe.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B1-red-proofs.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B2-9202-deploy-and-pass.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B2-9202-selfupdate.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B3-demo-boxes-before-floor.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B4-floor.txt create mode 100644 documentation/audits/rulings-2026-10-01/B/B5-demo-boxes-after.txt create mode 100644 documentation/audits/rulings-2026-10-01/C/C0-drill.txt create mode 100644 documentation/audits/rulings-2026-10-01/C/C1-mealie-lockout-1h.txt create mode 100644 documentation/audits/rulings-2026-10-01/D/D1-r746-digest.txt create mode 100644 documentation/audits/rulings-2026-10-01/D/D2-outline-fixture-9202.txt create mode 100644 documentation/audits/rulings-2026-10-01/T/T1-bench-destroy.txt create mode 100644 documentation/audits/rulings-2026-10-01/tools/b_9202.py create mode 100644 documentation/audits/rulings-2026-10-01/tools/b_selfupdate.py create mode 100644 documentation/audits/rulings-2026-10-01/tools/c_mealie.py create mode 100644 documentation/audits/rulings-2026-10-01/tools/d_outline.py create mode 100644 documentation/audits/rulings-2026-10-01/tools/repoint.py create mode 100644 documentation/audits/rulings-2026-10-01/tools/walk.py diff --git a/documentation/audits/rulings-2026-10-01/A/A0-9202-containers-before.txt b/documentation/audits/rulings-2026-10-01/A/A0-9202-containers-before.txt new file mode 100644 index 00000000..524e2004 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/A/A0-9202-containers-before.txt @@ -0,0 +1,6 @@ +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.284.2 +filebrowser gtstef/filebrowser:1.3.3-stable +paperless-postgres postgres:18-alpine +paperless-redis redis:7-alpine +paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15 +traefik traefik:v3.6.7 diff --git a/documentation/audits/rulings-2026-10-01/A/A1-bench-create.txt b/documentation/audits/rulings-2026-10-01/A/A1-bench-create.txt new file mode 100644 index 00000000..6fd3df10 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/A/A1-bench-create.txt @@ -0,0 +1,38 @@ ++ date -u +Thu Oct 1 05:13:35 UTC 2026 ++ pct list +VMID Status Lock Name +9201 running demo-hp +9202 running demo-hp-scratch ++ free -g ++ head -2 + total used free shared buff/cache available +Mem: 29 5 4 0 19 23 ++ pvesm status ++ grep -E 'local-lvm|nvme' +local-lvm lvmthin active 56487936 34813514 21674421 61.63% +nvme-scratch dir active 983379700 77139224 856213864 7.84% ++ pct config 9401 ++ head -1 +Configuration file 'nodes/demo-hp/lxc/9401.conf' does not exist ++ ls /var/lib/vz/template/cache/ ++ grep debian-13 ++ pveam download local debian-13-standard_13.6-1_amd64.tar.zst ++ tail -1 +download of 'http://download.proxmox.com/images/system/debian-13-standard_13.6-1_amd64.tar.zst' to '/var/lib/vz/template/cache/debian-13-standard_13.6-1_amd64.tar.zst' finished ++ pct create 9401 local:vztmpl/debian-13-standard_13.6-1_amd64.tar.zst --hostname upgrade-harness --cores 6 --memory 10240 --swap 0 --rootfs nvme-scratch:60 --net0 name=eth0,bridge=vmbr0,ip=dhcp --unprivileged 1 --features nesting=1,keyctl=1 --onboot 0 ++ tail -1 +done: SHA256:wcMZdv+y66Wb/F/7TXnqMtS4ksKrFLFcfT7ie52f01Y root@upgrade-harness ++ pct start 9401 ++ sleep 15 ++ pct exec 9401 -- bash -c 'hostname; ip -4 -o addr show eth0 | awk "{print \$4}"; apt-get update -qq >/dev/null 2>&1; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker.io docker-compose python3 python3-yaml curl postgresql-client sqlite3 >/dev/null 2>&1; docker --version; docker compose version; python3 --version; docker network create traefik-public >/dev/null && echo traefik-public-net; swapon --show; free -m | head -3; df -h / | tail -1' +upgrade-harness +192.168.0.127/24 +Docker version 26.1.5+dfsg1, build a72d7cd +Docker Compose version 2.26.1-4 +Python 3.13.5 +traefik-public-net + total used free shared buff/cache available +Mem: 10240 59 8537 0 1643 10180 +Swap: 0 0 0 +/dev/loop2 59G 1.3G 55G 3% / diff --git a/documentation/audits/rulings-2026-10-01/A/A2-drill-reset.txt b/documentation/audits/rulings-2026-10-01/A/A2-drill-reset.txt new file mode 100644 index 00000000..7f6c4b59 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/A/A2-drill-reset.txt @@ -0,0 +1,6 @@ +# drill reset 2026-10-01T05:14:59Z +drill before: 6a3ead9; live main: efd492d +drill is an ancestor of live — fast-forward, no force +remote: . Processing 1 references +remote: Processed 1 references in total +drill after: efd492d diff --git a/documentation/audits/rulings-2026-10-01/A/A3-9202-repoint-drill.txt b/documentation/audits/rulings-2026-10-01/A/A3-9202-repoint-drill.txt new file mode 100644 index 00000000..f2c89786 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/A/A3-9202-repoint-drill.txt @@ -0,0 +1,10 @@ +git: + branch: main + repo_url: https://gitea.dooplex.hu/admin/app-catalog-drill.git + sync_interval: 15m + token: + username: "admin" +hub: +update: + health_timeout: 90s + diff --git a/documentation/audits/rulings-2026-10-01/B/B0-before-probe.txt b/documentation/audits/rulings-2026-10-01/B/B0-before-probe.txt new file mode 100644 index 00000000..4adb9b53 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B0-before-probe.txt @@ -0,0 +1,117 @@ +## demo-hp 9202 +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = "UTF-8", + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). +file: gitea.dooplex.hu/admin/felhom-controller:0.284.2 +running: gitea.dooplex.hu/admin/felhom-controller:0.284.2 sha256:c187fea3a3b229bc118ea60f1e82cdae6c325b09d3febb11401a262633d182f7 +controller.yaml +controller.yaml.pre-28 +controller.yaml.pre-bakeoff +controller.yaml.pre-cleanup +controller.yaml.pre-immich0930 +controller.yaml.pre-ladder0924 +controller.yaml.pre-more0930 +controller.yaml.pre-night0923 +controller.yaml.pre-night0924 +controller.yaml.pre-night0925 +controller.yaml.pre-night0926 +controller.yaml.pre-pg0928 +controller.yaml.pre-pg0930 +controller.yaml.pre-r649 +controller.yaml.pre-rulings +controller.yaml.pre-rulings0930 +controller.yaml.pre-undofleet +controller.yaml.pre-update-night +controller.yaml.pre-vt0926 +data +--- controller images +gitea.dooplex.hu/admin/felhom-controller:0.283.1 79d28d57f414 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.0 67fbed846822 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.1 4d046ec22473 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 409MB +count: 4 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 13 6 3.168GB 596.4MB (18%) +## demo-hp 9201 +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = "UTF-8", + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). +file: gitea.dooplex.hu/admin/felhom-controller:0.284.2 +running: gitea.dooplex.hu/admin/felhom-controller:0.284.2 sha256:c187fea3a3b229bc118ea60f1e82cdae6c325b09d3febb11401a262633d182f7 +controller.yaml +controller.yaml.pre-gate-day +controller.yaml.pre-undofleet +data +--- controller images +gitea.dooplex.hu/admin/felhom-controller: 89c8fdebc79f 422MB +gitea.dooplex.hu/admin/felhom-controller: 8959861193ec 423MB +gitea.dooplex.hu/admin/felhom-controller: c3f778477cf6 422MB +gitea.dooplex.hu/admin/felhom-controller: c763b06ae13b 422MB +gitea.dooplex.hu/admin/felhom-controller: dfd75983de3d 422MB +count: 83 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 108 20 15.2GB 4.744GB (31%) +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = "UTF-8", + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). +file: gitea.dooplex.hu/admin/felhom-controller:0.284.2 +running: gitea.dooplex.hu/admin/felhom-controller:0.284.2 sha256:c187fea3a3b229bc118ea60f1e82cdae6c325b09d3febb11401a262633d182f7 +controller.yaml +data +--- controller images +gitea.dooplex.hu/admin/felhom-controller:0.282.0 3150750f876e 409MB +gitea.dooplex.hu/admin/felhom-controller:0.283.0 80bf58a8b649 409MB +gitea.dooplex.hu/admin/felhom-controller:0.283.1 79d28d57f414 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 409MB +gitea.dooplex.hu/admin/felhom-controller: 89c8fdebc79f 422MB +count: 75 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 82 5 6.068GB 3.641GB (60%) diff --git a/documentation/audits/rulings-2026-10-01/B/B1-red-proofs.txt b/documentation/audits/rulings-2026-10-01/B/B1-red-proofs.txt new file mode 100644 index 00000000..c331bc53 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B1-red-proofs.txt @@ -0,0 +1,33 @@ +## RED: keep-switch without the previous +--- FAIL: TestControllerRetention_KeepsRunningAndPreviousDeletesOlder (0.00s) + controller_image_retention_test.go:48: the previous controller (0.284.2) was deleted: [gitea.dooplex.hu/admin/felhom-controller:0.283.1 gitea.dooplex.hu/admin/felhom-controller:0.284.1 gitea.dooplex.hu/admin/felhom-controller:0.284.2 sha256:CNONE] +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.011s +## RED: swap record ignored (version order only) +--- FAIL: TestControllerRetention_RecordBeatsVersionOrder (0.00s) + controller_image_retention_test.go:75: the recorded previous (0.283.1) was deleted: [gitea.dooplex.hu/admin/felhom-controller:0.283.1 gitea.dooplex.hu/admin/felhom-controller:0.284.1 sha256:CNONE] +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.012s +## RED: no in-flight swap check +--- FAIL: TestControllerRetention_NothingWhileSwappingAndNewerKept (0.00s) + controller_image_retention_test.go:113: deleted while the controller is swapping itself: [gitea.dooplex.hu/admin/felhom-controller:0.283.1 sha256:CNONE] +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.010s +## RED: no in-use check (first attempt removed the line: build failed — redone below) +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks [build failed] +FAIL +## RED: RecordedPrevious without the success check +--- FAIL: TestRecordedPrevious (0.00s) + state_test.go:23: a failed swap (rolled back: the box runs the 'previous'): got "r:0.284.2", want "" + state_test.go:23: pending: got "r:0.284.2", want "" +## GREEN after restore +ok gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.038s +ok gitea.dooplex.hu/admin/felhom-controller/internal/selfupdate 0.006s +## RED: no in-use check (if false && used[id]) +--- FAIL: TestControllerRetention_InUseAndFailClosed (0.00s) + controller_image_retention_test.go:139: an image a container uses was deleted: [gitea.dooplex.hu/admin/felhom-controller:0.283.1 gitea.dooplex.hu/admin/felhom-controller:0.284.1 sha256:CNONE] +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.023s +## RED: R-751 — RetainImagesAfterUpdate without the nil-stack check (pre-fix code, v0.284.2) +--- FAIL: TestRetainImagesAfterUpdate_AppGoneDoesNotPanic — panic: runtime error: invalid memory address or nil pointer dereference at image_retention.go:291 +## GREEN with the fix: ok diff --git a/documentation/audits/rulings-2026-10-01/B/B2-9202-deploy-and-pass.txt b/documentation/audits/rulings-2026-10-01/B/B2-9202-deploy-and-pass.txt new file mode 100644 index 00000000..ed831928 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B2-9202-deploy-and-pass.txt @@ -0,0 +1,27 @@ +## before +gitea.dooplex.hu/admin/felhom-controller:0.283.1 79d28d57f414 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.0 67fbed846822 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.1 4d046ec22473 409MB +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 409MB +count=4 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 13 6 3.168GB 596.4MB (18%) +/dev/loop1 69G 3.5G 62G 6% /var/lib/docker + +gitea.dooplex.hu/admin/felhom-controller:0.285.0 +gitea.dooplex.hu/admin/felhom-controller:0.285.0 2026-10-01T06:06:10.675047378Z + +## the passes (positive observable: one line per pass) +2026/10/01 06:09:11 controller_image_retention.go:184: [INFO] [stacks] controller image retention: deleted gitea.dooplex.hu/admin/felhom-controller:0.284.1 (4d046ec22473, 409MB) — older than the previous controller and no container uses it (decision 56) +2026/10/01 06:09:11 controller_image_retention.go:184: [INFO] [stacks] controller image retention: deleted gitea.dooplex.hu/admin/felhom-controller:0.284.0 (67fbed846822, 409MB) — older than the previous controller and no container uses it (decision 56) +2026/10/01 06:09:11 controller_image_retention.go:184: [INFO] [stacks] controller image retention: deleted gitea.dooplex.hu/admin/felhom-controller:0.283.1 (79d28d57f414, 409MB) — older than the previous controller and no container uses it (decision 56) +2026/10/01 06:09:11 controller_image_retention.go:115: [INFO] [stacks] controller image retention: pass over 5 controller image(s) — running 0.285.0, previous "0.284.2" (by version order (no swap record names one present)), 3 candidate(s), 3 deleted, the rest kept + +## after +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 409MB +gitea.dooplex.hu/admin/felhom-controller:0.285.0 9ef68bf2a119 409MB +count=2 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 11 6 3.095GB 523.3MB (16%) +/dev/loop1 69G 3.4G 62G 6% /var/lib/docker + diff --git a/documentation/audits/rulings-2026-10-01/B/B2-9202-selfupdate.txt b/documentation/audits/rulings-2026-10-01/B/B2-9202-selfupdate.txt new file mode 100644 index 00000000..e69de29b diff --git a/documentation/audits/rulings-2026-10-01/B/B3-demo-boxes-before-floor.txt b/documentation/audits/rulings-2026-10-01/B/B3-demo-boxes-before-floor.txt new file mode 100644 index 00000000..d6f59184 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B3-demo-boxes-before-floor.txt @@ -0,0 +1,13 @@ +# 2026-10-01T06:10:21Z +## demo-hp 9201 +running: gitea.dooplex.hu/admin/felhom-controller:0.284.2 +controller images: 83 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 108 20 15.2GB 4.744GB (31%) +/dev/mapper/pve-vm--9201--disk--1 69G 18G 48G 28% /var/lib/docker +## N100 9201 +running: gitea.dooplex.hu/admin/felhom-controller:0.284.2 +controller images: 75 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 82 5 6.068GB 3.641GB (60%) +/dev/mapper/pve-vm--9201--disk--1 246G 6.0G 228G 3% /var/lib/docker diff --git a/documentation/audits/rulings-2026-10-01/B/B4-floor.txt b/documentation/audits/rulings-2026-10-01/B/B4-floor.txt new file mode 100644 index 00000000..e5ffccce --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B4-floor.txt @@ -0,0 +1,8 @@ +# floor 2026-10-01T06:10:58Z (the first attempt 06:10:30 had no credential: 302 /login, nothing stored) +HTTP/1.1 303 See Other +Location: /configuration?flash=floor_set +2026/10/01 08:10:59 [INFO] Global controller-version floor set to "0.285.0" (declared MinAgent "0.131.0") +2026/10/01 08:11:01 [INFO] managed floor SERVED for demo-felhom: floor 0.285.0, agent requirement "0.131.0" from declared (golden 0.284.2) +2026/10/01 08:11:02 [INFO] managed floor SERVED for demo-hp: floor 0.285.0, agent requirement "0.131.0" from declared (golden 0.284.2) +demo-hp: gitea.dooplex.hu/admin/felhom-controller:0.285.0 2026-10-01T06:11:08.656824206Z +felhom-pve: gitea.dooplex.hu/admin/felhom-controller:0.285.0 2026-10-01T06:11:06.795144354Z diff --git a/documentation/audits/rulings-2026-10-01/B/B5-demo-boxes-after.txt b/documentation/audits/rulings-2026-10-01/B/B5-demo-boxes-after.txt new file mode 100644 index 00000000..ca00b182 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/B/B5-demo-boxes-after.txt @@ -0,0 +1,45 @@ +# 2026-10-01T06:14:27Z +## demo-hp 9201 +{ + "status": "success", + "previous_version": "0.284.2", + "previous_image": "gitea.dooplex.hu/admin/felhom-controller:0.284.2", + "target_version": "0.285.0", + "target_image": "gitea.dooplex.hu/admin/felhom-controller:0.285.0", + "initiated_at": "2026-10-01T06:11:02Z", + "initiated_by": "auto-floor", + "completed_at": "2026-10-01T06:11:09Z" +} +2026/10/01 06:11:09 updater.go:845: [INFO] [selfupdate] Post-update startup: update successful (0.284.2 → 0.285.0) +2026/10/01 06:14:15 controller_image_retention.go:115: [INFO] [stacks] controller image retention: pass over 84 controller image(s) — running 0.285.0, previous "0.284.2" (the self-update's record), 82 candidate(s), 82 deleted, the rest kept +82 +running: gitea.dooplex.hu/admin/felhom-controller:0.285.0 +controller images: 2 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 27 20 11.65GB 1.892GB (16%) +/dev/mapper/pve-vm--9201--disk--1 69G 15G 51G 22% /var/lib/docker +gitea.dooplex.hu/admin/felhom-controller:0.285.0 9ef68bf2a119 +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 +2 +## felhom-pve 9201 +{ + "status": "success", + "previous_version": "0.284.2", + "previous_image": "gitea.dooplex.hu/admin/felhom-controller:0.284.2", + "target_version": "0.285.0", + "target_image": "gitea.dooplex.hu/admin/felhom-controller:0.285.0", + "initiated_at": "2026-10-01T06:11:01Z", + "initiated_by": "auto-floor", + "completed_at": "2026-10-01T06:11:07Z" +} +2026/10/01 06:11:07 [INFO] [selfupdate] Post-update startup: update successful (0.284.2 → 0.285.0) +2026/10/01 06:14:11 [INFO] [stacks] controller image retention: pass over 76 controller image(s) — running 0.285.0, previous "0.284.2" (the self-update's record), 74 candidate(s), 74 deleted, the rest kept +74 +running: gitea.dooplex.hu/admin/felhom-controller:0.285.0 +controller images: 2 +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 9 5 1.106GB 139.2MB (12%) +/dev/mapper/pve-vm--9201--disk--1 246G 1.2G 233G 1% /var/lib/docker +gitea.dooplex.hu/admin/felhom-controller:0.285.0 9ef68bf2a119 +gitea.dooplex.hu/admin/felhom-controller:0.284.2 c187fea3a3b2 +2 diff --git a/documentation/audits/rulings-2026-10-01/C/C0-drill.txt b/documentation/audits/rulings-2026-10-01/C/C0-drill.txt new file mode 100644 index 00000000..c3cd8e1b --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/C/C0-drill.txt @@ -0,0 +1,3 @@ +# drill reset to live before the mealie test 2026-10-01T06:14:57Z +drill: 804884a live: 804884a +df5a2a2 DRILL mealie: SECURITY_USER_LOCKOUT_TIME=1 (R-747 proof on 9202) diff --git a/documentation/audits/rulings-2026-10-01/C/C1-mealie-lockout-1h.txt b/documentation/audits/rulings-2026-10-01/C/C1-mealie-lockout-1h.txt new file mode 100644 index 00000000..3fa28621 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/C/C1-mealie-lockout-1h.txt @@ -0,0 +1,30 @@ +06:15:29 the box's catalog copy: # R-747: mealie locks the ACCOUNT for SECURITY_USER_LOCKOUT_TIME hours (default 24) after 5 wrong logins, and its + - SECURITY_USER_LOCKOUT_TIME=1 +08:15:29 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['ADMIN_PASSWORD'] +08:15:29 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +08:16:34 [1] deployed, controller state=running, pinned={'mealie': 'ghcr.io/mealie-recipes/mealie:v3.28.0'} +06:16:34 deploy: True +06:16:46 2026/10/01 06:15:29 install_hold.go:106: [INFO] [stacks] mealie: install HOLD before the first start — only the household reaches [recipes.enkisfelhom.hu] until the known first login is replaced +2026/10/01 06:16:34 install_hold.go:135: [INFO] [stacks] mealie: install hold OPENED by after_install — the app is reached as without a hold +06:16:50 setting inside the app: 5 1 +06:16:50 generated password length: 30 +06:16:51 positive control — right password: 200 +06:16:51 stranger wrong try 1: 401 +06:16:51 stranger wrong try 2: 401 +06:16:52 stranger wrong try 3: 401 +06:16:52 stranger wrong try 4: 401 +06:16:52 stranger wrong try 5: 401 +06:16:52 stranger wrong try 6: 423 +06:16:52 household, RIGHT password, right after: 423 +06:16:52 household, RIGHT password by username 'admin': 423 +06:18:52 +2 min right password: 423 +06:20:52 +4 min right password: 423 +06:22:52 +6 min right password: 423 +06:24:52 +8 min right password: 423 +06:26:53 +10 min right password: 423 +06:28:53 +12 min right password: 423 +06:30:53 +14 min right password: 423 +06:32:53 +16 min right password: 423 +06:34:53 +18 min right password: 423 +06:36:53 +20 min right password: 423 +06:38:53 +22 min right password: 423 diff --git a/documentation/audits/rulings-2026-10-01/D/D1-r746-digest.txt b/documentation/audits/rulings-2026-10-01/D/D1-r746-digest.txt new file mode 100644 index 00000000..fb07f151 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/D/D1-r746-digest.txt @@ -0,0 +1,16 @@ +## unit (fixed) + +OK +## unit RED (pre-fix image_digest.py) +FAIL: test_absent_digest_is_refused (__main__.ResolveDigest.test_absent_digest_is_refused) +FAIL: test_malformed_digest_is_refused (__main__.ResolveDigest.test_malformed_digest_is_refused) +FAIL: test_served_digest_is_asked_by_digest (__main__.ResolveDigest.test_served_digest_is_asked_by_digest) +FAILED (failures=3) +## live registry, fixed resolver +tag now: sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 +redis:7-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 +redis:7-alpine@sha256:0000000000000000000000000000000000000000000000000000000000000000 UNRESOLVED: HTTP 404 +rc=2 +## live registry, pre-fix resolver (the false yes) +redis:7-alpine@sha256:0000000000000000000000000000000000000000000000000000000000000000 sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 +rc=0 diff --git a/documentation/audits/rulings-2026-10-01/D/D2-outline-fixture-9202.txt b/documentation/audits/rulings-2026-10-01/D/D2-outline-fixture-9202.txt new file mode 100644 index 00000000..ce241a16 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/D/D2-outline-fixture-9202.txt @@ -0,0 +1,15 @@ +08:16:44 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +08:18:15 [1] deployed, controller state=running, pinned={'outline': 'outlinewiki/outline:1.10.1', 'outline-postgres': 'postgres:18-alpine', 'outline-redis': 'redis:7-alpine'} +08:18:15 deploy: True +08:18:18 running: outlinewiki/outline:1.10.1@sha256:832051f039b446c87aa23929cf92c00980c66aaa2d744d118c48c016ec816ad8 +08:18:18 gate: kb is gated — passed as the household (cookie set) +08:18:18 outline: installation.create http=302 +08:18:18 outline: CSRF cookie __Host-csrfToken +08:18:19 outline: seeded document drilldoc-bd3b1590 +08:18:19 seed: OK +08:18:19 outline: readback of the seeded document http=200 found=True +08:18:19 verify (read back + unknown id + wrong key): True +08:18:20 [X] stop -> 200 {'ok': True, 'message': 'Stack outline stop completed'} +08:18:52 [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'outline', 'volumes_removed': ['outline_outline_data', 'outline_outline_postgres_data', 'outline_outline_redis_data'], 'hdd_pat +08:19:01 [X] after remove: deployed=False leftovers='/opt/docker/stacks/outline' +08:19:01 removed; deployed = False diff --git a/documentation/audits/rulings-2026-10-01/T/T1-bench-destroy.txt b/documentation/audits/rulings-2026-10-01/T/T1-bench-destroy.txt new file mode 100644 index 00000000..89adfce5 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/T/T1-bench-destroy.txt @@ -0,0 +1,13 @@ ++ date -u +Thu Oct 1 06:28:48 UTC 2026 ++ pct stop 9401 ++ pct destroy 9401 --purge +purging CT 9401 from related configurations.. ++ pct list +VMID Status Lock Name +9201 running demo-hp +9202 running demo-hp-scratch ++ rm -f /var/lib/vz/template/cache/debian-13-standard_13.6-1_amd64.tar.zst ++ ls /var/lib/vz/template/cache/ ++ grep -c debian-13 +0 diff --git a/documentation/audits/rulings-2026-10-01/tools/b_9202.py b/documentation/audits/rulings-2026-10-01/tools/b_9202.py new file mode 100644 index 00000000..0d9287ac --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/b_9202.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +"""Part B on 9202 (self-update is off here — no hub): the standard bootstrap deploy of the release, then the +decision-56 pass 3 minutes after the new controller starts. 9202 has no swap record → the version-order fallback.""" +import time, sys +import walk as w +V = sys.argv[1] +IMG = "docker image ls -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Size}}' | grep felhom-controller | sort -V; echo count=$(docker image ls -a --format '{{.Repository}}' | grep -c felhom-controller); docker system df | sed -n 1,2p; df -h /var/lib/docker | tail -1" +print("## before", flush=True); print(w.guest(IMG), flush=True) +print(w.guest(f"docker pull -q gitea.dooplex.hu/admin/felhom-controller:{V} && echo gitea.dooplex.hu/admin/felhom-controller:{V} > /etc/felhom-controller-image && systemctl restart felhom-controller-bootstrap.service; sleep 20; docker inspect felhom-controller --format '{{{{.Config.Image}}}} {{{{.State.StartedAt}}}}'"), flush=True) +time.sleep(210) +print("## the passes (positive observable: one line per pass)", flush=True) +print(w.guest("docker logs --since 5m felhom-controller 2>&1 | grep -E 'image retention' | cut -c1-280"), flush=True) +print("## after", flush=True); print(w.guest(IMG), flush=True) diff --git a/documentation/audits/rulings-2026-10-01/tools/b_selfupdate.py b/documentation/audits/rulings-2026-10-01/tools/b_selfupdate.py new file mode 100644 index 00000000..e6e7b2f2 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/b_selfupdate.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Part B on 9202: the controller's own Update button (POST /api/selfupdate/update) to the newest release, then the +decision-56 pass 3 minutes after the new controller starts. Images + GB before/after; the pass's own log lines.""" +import time, sys +import walk as w +IMG = "docker image ls -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Size}}' | grep felhom-controller | sort -V; echo count=$(docker image ls -a --format '{{.Repository}}' | grep -c felhom-controller); docker system df | sed -n 1,2p; df -h /var/lib/docker | tail -1" +print("## before"); print(w.guest(IMG)) +w.login() +print("check:", w.ctl("POST", "/api/selfupdate/check")) +print("trigger:", w.ctl("POST", "/api/selfupdate/update")) +for _ in range(60): + time.sleep(10) + img = w.guest("docker inspect felhom-controller --format '{{.Config.Image}} {{.State.StartedAt}}'").strip() + if sys.argv[1] in img: + break +print("running:", img) +print("update-state:", w.guest("docker exec felhom-controller cat /opt/docker/felhom-controller/data/update-state.json")) +time.sleep(200) +print("## the passes (positive observable: one line per pass)") +print(w.guest("docker logs --since 10m felhom-controller 2>&1 | grep -E 'image retention|Post-update startup' | cut -c1-260")) +print("## after"); print(w.guest(IMG)) diff --git a/documentation/audits/rulings-2026-10-01/tools/c_mealie.py b/documentation/audits/rulings-2026-10-01/tools/c_mealie.py new file mode 100644 index 00000000..d07b1f4a --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/c_mealie.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""R-747 on 9202: mealie with SECURITY_USER_LOCKOUT_TIME=1 (drill catalog). As a STRANGER (no session, no gate +cookie): five wrong passwords for the public login, then the household's right password — is it refused, and for how +long? Polled every 2 minutes with the RIGHT password (a refused try while locked is not counted by mealie).""" +import subprocess, time, json, sys +from datetime import datetime, timezone +import walk as w + +def now(): + return datetime.now(timezone.utc).strftime("%H:%M:%S") + +def token(user, pw): + r = subprocess.run(["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "15", "-H", f"Host: recipes.{w.DOMAIN}", + "--data-urlencode", f"username={user}", "--data-urlencode", f"password={pw}", f"{w.BASE}/api/auth/token"], + capture_output=True, text=True) + return r.stdout.strip() + +def p(*a): + print(now(), *a, flush=True) + +w.login() +for _ in range(10): + w.sync_rescan() + if "SECURITY_USER_LOCKOUT_TIME=1" in w.guest("grep -h SECURITY_USER_LOCKOUT /var/lib/docker/volumes/felhom-controller-data/_data/catalog-cache/templates/mealie/docker-compose.yml /var/lib/docker/volumes/felhom-controller-data/_data/data/catalog-cache/templates/mealie/docker-compose.yml 2>/dev/null"): + break + time.sleep(20) +p("the box's catalog copy:", w.guest("grep -rh SECURITY_USER_LOCKOUT /var/lib/docker/volumes/felhom-controller-data/_data/ --include=docker-compose.yml 2>/dev/null | sort -u").strip()) +p("deploy:", w.deploy("mealie", "recipes")) +for _ in range(120): + time.sleep(5) + if "hold OPENED" in w.guest("docker logs --since 15m felhom-controller 2>&1 | grep 'mealie: install hold OPENED'"): + break +p(w.guest("docker logs --since 15m felhom-controller 2>&1 | grep -E 'mealie.*(install HOLD|hold OPENED|after_install)' | cut -c1-200").strip()) +p("setting inside the app:", w.guest("docker exec mealie python3 -c 'from mealie.core.config import get_app_settings as g; s=g(); print(s.SECURITY_MAX_LOGIN_ATTEMPTS, s.SECURITY_USER_LOCKOUT_TIME)'").strip()) +gen = (w.GENERATED.get("mealie") or {}).get("ADMIN_PASSWORD", "") +p("generated password length:", len(gen)) +p("positive control — right password:", token("changeme@example.com", gen)) +for i in range(1, 7): + p(f"stranger wrong try {i}:", token("changeme@example.com", f"wrong-{i}-{time.time()}")) +t0 = time.time() +p("household, RIGHT password, right after:", token("changeme@example.com", gen)) +p("household, RIGHT password by username 'admin':", token("admin", gen)) +code = None +while time.time() - t0 < 2.5 * 3600: + time.sleep(120) + code = token("changeme@example.com", gen) + p(f"+{(time.time()-t0)/60:.0f} min right password:", code) + if code == "200": + break +p(f"RESULT: locked for {(time.time()-t0)/60:.0f} min (last answer {code})") +p("right password again:", token("changeme@example.com", gen), "| a wrong one after:", token("changeme@example.com", "wrong-after")) +p(w.guest("docker logs mealie 2>&1 | grep -iE 'lock' | tail -5 | cut -c1-200")) +w.remove("mealie") +p("removed:", w.stack("mealie").get("deployed")) diff --git a/documentation/audits/rulings-2026-10-01/tools/d_outline.py b/documentation/audits/rulings-2026-10-01/tools/d_outline.py new file mode 100644 index 00000000..a3725477 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/d_outline.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""R-744 on 9202: outline 1.10.1 (the live pin) installed through the product, the catalog's box fixture (with the fix) +seeds through outline's own first-run API, then verify() reads it back (and requires an unknown id -> not found and a +wrong key -> refused). Then removed.""" +import os, sys +sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts") +os.environ.setdefault("SC", "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/b4d68b9b-a6cf-4d21-8220-ece956837fc3/scratchpad") +import box_walk as w +import upgrade_fixtures_box as fx +w.login() +f = fx.Outline() +w.say("deploy:", w.deploy("outline", "kb")) +w.say("running:", w.guest("docker inspect outline --format '{{.Config.Image}}'").strip()) +t = f.seed(w, "kb", w.say) +w.say("seed:", "OK" if t else "FAILED - " + getattr(f, "tried", "?")) +if t: + w.say("verify (read back + unknown id + wrong key):", f.verify(w, "kb", t, w.say)) +w.remove("outline") +w.say("removed; deployed =", w.stack("outline").get("deployed")) diff --git a/documentation/audits/rulings-2026-10-01/tools/repoint.py b/documentation/audits/rulings-2026-10-01/tools/repoint.py new file mode 100644 index 00000000..9bc7c935 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/repoint.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Point guest 9202 at the drill catalog (and a 90 s health timeout), or restore the saved config. + +`09` §6.5: `git.repo_url` alone is INERT (R-615) — the cache dir must go too. The saved copy is +`controller.yaml.pre-rulings1001` (NOT the older `.pre-28`, which a restore must never pick up). +""" +import re, sys, io +sys.path.insert(0, '.') +import walk as w + +VOL = "/var/lib/docker/volumes/felhom-controller-data/_data" +DRILL_REPO = "https://gitea.dooplex.hu/admin/app-catalog-drill.git" + + +def creds(): + for l in io.open("/home/kisfenyo/.git-credentials").read().strip().split("\n"): + m = re.match(r'https://(admin):([^@]+)@gitea\.dooplex\.hu', l) + if m: + return m.group(1), m.group(2) + raise SystemExit("no admin credential") + + +def to_drill(): + u, t = creds() + print(w.guest(f""" +set -e +test -f {VOL}/controller.yaml.pre-rulings1001 || cp -p {VOL}/controller.yaml {VOL}/controller.yaml.pre-rulings1001 +python3 - <<'PY' +import re +p = "{VOL}/controller.yaml" +s = open(p).read() +s = re.sub(r'(^\\s+repo_url: ).*$', r'\\g<1>{DRILL_REPO}', s, count=1, flags=re.M) +s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+token: ).*$', r'\\g<1>"{t}"', s, count=1, flags=re.M) +s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+username: ).*$', r'\\g<1>"{u}"', s, count=1, flags=re.M) +if not re.search(r'^update:', s, re.M): + s += "update:\\n health_timeout: 90s\\n" +open(p, "w").write(s) +PY +rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache +docker restart felhom-controller >/dev/null +sleep 15 +grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: /' +grep -A2 '^update:' {VOL}/controller.yaml +""")) + + +def restore(): + print(w.guest(f""" +set -e +cp -p {VOL}/controller.yaml.pre-rulings1001 {VOL}/controller.yaml +rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache +docker restart felhom-controller >/dev/null +sleep 15 +grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: /' +grep -c '^update:' {VOL}/controller.yaml || true +""")) + + +if __name__ == "__main__": + to_drill() if sys.argv[1] == "drill" else restore() diff --git a/documentation/audits/rulings-2026-10-01/tools/walk.py b/documentation/audits/rulings-2026-10-01/tools/walk.py new file mode 100644 index 00000000..bd72de58 --- /dev/null +++ b/documentation/audits/rulings-2026-10-01/tools/walk.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""walk.py — ONE app's full update walk on guest 9202, through the product's own endpoints. + +EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses: + POST /api/stacks//deploy · POST /api/sync · POST /api/stacks/rescan + POST /api/stacks//update · POST /api/stacks//remove +and reads GET /api/stacks/. No controller code exists for it. + +The walk, per `09` §6.4 and the update-night brief §4: + 1 deploy from the DRILL catalog at the LIVE pin + 2 seed through the app's OWN front door (R-156: never a volume, never SQL) + 3 read the seed back <- control C1; a fixture that cannot prove itself proves nothing + 4 „Mentés most" + 5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages + 6 press the guarded Update, record every phase with timestamps + 7 read the seed back through the front door + 8 the four version observables side by side + 9 write the verdict record in `09`'s JSON shape + +`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`. +""" +import argparse, json, os, re, subprocess, sys, time +from datetime import datetime, timezone + +SC = os.environ.get('SC', '/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/b4d68b9b-a6cf-4d21-8220-ece956837fc3/scratchpad') +EV = os.environ.get('EV', '/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/rulings-2026-10-01') +DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill" +# GUEST=9201 selects demo-hp's hub-enabled guest (the mail proof); default 9202, the scratch guest. +GUEST = os.environ.get("GUEST", "9202") +BASE = os.environ.get("BASE") or {"9202": "https://192.168.0.114", "9201": "https://192.168.0.155"}[GUEST] +DOMAIN = os.environ.get("DOMAIN", "enkisfelhom.hu") +HOSTHDR = f"Host: felhom.{DOMAIN}" +HP = "demo-hp" + +LOG = [] + + +def say(*a): + line = " ".join(str(x) for x in a) + ts = datetime.now().strftime("%H:%M:%S") + print(f"{ts} {line}", flush=True) + LOG.append(f"{ts} {line}") + + +def sh(args, timeout=300, inp=None): + try: + return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp) + except (subprocess.TimeoutExpired, OSError) as e: + return subprocess.CompletedProcess(args, 124, "", f"{e}") + + +def guest(script, timeout=600): + """Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting).""" + # ONE TEMP FILE PER CALL (night 2026-09-23): the shared /tmp/w.sh swapped scripts under + # two concurrent walks (memory: guest-helper-shares-one-tmp-file). + import secrets as _s + t = f"/tmp/w{GUEST}-{os.getpid()}-{_s.token_hex(4)}.sh" + r = sh(["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new", HP, + f"export LC_ALL=C; cat > {t}; pct push {GUEST} {t} {t} >/dev/null 2>&1; " + f"pct exec {GUEST} -- bash {t}; pct exec {GUEST} -- rm -f {t}; rm -f {t}"], + timeout=timeout, inp=script) + return r.stdout or "" + + +def login(): + pw = open(f"{SC}/.ctlpw").read().strip() + sh(["curl", "-sk", "-D", f"{SC}/hdr{os.getpid()}.txt", "-o", "/dev/null", "-H", HOSTHDR, + "-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"]) + h = open(f"{SC}/hdr{os.getpid()}.txt").read() + m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I) + if not m: + sys.exit("login failed: no session cookie") + open(f"{SC}/sess{os.getpid()}.txt", "w").write(m.group(0)) + r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"]) + c = re.search(r' the felhom_gate cookie the household's browser would hold (setup gate, v0.280.0) + + +def _loc(out): + m = re.search(r"(?im)^location:\s*(\S+)", out or "") + return m.group(1) if m else "" + + +def gate_cookie(sub): + """Pass the setup gate (`09` decision 46) the way the HOUSEHOLD does: the app host redirects to the + dashboard's /__gate/start, which (with the dashboard session) redirects back to the app host's + /__felhom_gate/cb, which sets `felhom_gate`. Never printed.""" + import urllib.parse + sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip() + r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {sub}.{DOMAIN}", f"{BASE}/"]) + loc = _loc(r.stdout) + if "/__gate/start" not in loc: + GATE[sub] = "" + return "" # not gated (open, or no gate for this app) + u = urllib.parse.urlsplit(loc) + r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", "-H", f"Cookie: {sess}", + f"{BASE}{u.path}?{u.query}"]) + loc = _loc(r.stdout) + u = urllib.parse.urlsplit(loc) + if "/__felhom_gate/cb" not in u.path: + say(f" gate: the dashboard did not hand back a callback for {sub} ({loc[:80]})") + return "" + r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", "Accept: text/html", "-H", f"Host: {u.hostname}", f"{BASE}{u.path}?{u.query}"]) + m = re.search(r"(?im)^set-cookie:\s*(felhom_gate=[^;\r\n]+)", r.stdout or "") + GATE[sub] = m.group(1) if m else "" + say(f" gate: {sub} is gated — passed as the household (cookie {'set' if GATE[sub] else 'NOT set'})") + return GATE[sub] + + +def app_curl(sub, path, *extra, method=None, data=None, timeout=45, _retry=True): + """A call to the APP's own front door on 9202 — the household's route, not ours. Carries the setup + gate's cookie when the app is gated, merged into a fixture's own Cookie header (never a second one).""" + raw = list(extra) + gc = GATE[sub] if sub in GATE else gate_cookie(sub) + ext = list(raw) + if gc: + merged = False + for i, a in enumerate(ext): + if isinstance(a, str) and a.lower().startswith("cookie:") and i > 0 and ext[i - 1] == "-H": + ext[i] = a + "; " + gc + merged = True + if not merged: + ext = ["-H", f"Cookie: {gc}"] + ext + args = ["curl", "-sSk", "--max-time", str(timeout), "-H", f"Host: {sub}.{DOMAIN}", + "-w", "\n%{http_code} %{redirect_url}"] + if method: + args += ["-X", method] + if data is not None: + args += ["--data-binary", "@-"] + args += ext + [f"{BASE}{path}"] + r = sh(args, timeout=timeout + 30, inp=data) + body, _, tail = (r.stdout or "").rpartition("\n") + code, _, redir = tail.strip().partition(" ") + if _retry and "/__gate/start" in redir: + GATE.pop(sub, None) # the gate cookie expired or was never taken — log in as the household again + return app_curl(sub, path, *raw, method=method, data=data, timeout=timeout, _retry=False) + return r.returncode, code.strip(), body + + +def stack(name): + _, d = ctl("GET", f"/api/stacks/{name}") + return (d.get("data") or {}) if isinstance(d, dict) else {} + + +def wait_app(sub, path="/", want=("200", "302", "303", "401", "403"), tries=60, delay=5): + """Settling says the container runs; this says the APP answers. Not the same thing.""" + last = None + for _ in range(tries): + rc, code, _ = app_curl(sub, path, timeout=15) + last = (rc, code) + if rc == 0 and code in want: + return True + time.sleep(delay) + say(f" app never answered on {sub}{path} (last rc={last[0]} code={last[1]})") + return False + + +# ------------------------------------------------------------------ the walk + + +DRIVE = "/mnt/felhom-drives/scratch_hdd/userdata" + +# What THIS run generated for a deploy, per app. Deploy secrets are ENCRYPTED AT REST in +# `app.yaml` (`ENC:…`), which is right and which means a fixture cannot read an app's admin +# password back off the box — the household sees it once. So the value the harness itself +# generated is kept here for the life of the run, and nowhere else. +GENERATED = {} + + +def deploy_values(name, sub): + """Fill EVERY required deploy field the way the wizard would, by asking the box what this app + asks for — `GET /api/stacks//deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN. + + Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because + a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password + (grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only + reason this was safe to discover by running it (live-probes rule). + + A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on + the scratch drive first — the same act the drive browser performs for a household. + """ + code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields") + fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or [] + values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub} + made = [] + for f in fields: + ev, ty = f.get("env_var"), f.get("type") + if ev in values: + continue + # `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses + # when the caller sends none, deliberately ("the user needs to know their password"), + # while `.felhom.yml` declares `required: false` and the API serves that verbatim. A + # caller that trusts the contract gets a 400. Measured tonight on grafana; filed. + if not f.get("required") and ty != "password": + continue # the controller generates the optional secrets itself + if ty == "path": + p = f"{DRIVE}/{name}" + values[ev] = p + made.append(p) + elif ty in ("secret", "password"): + import secrets as _s + values[ev] = "Drill-" + _s.token_hex(12) + GENERATED.setdefault(name, {})[ev] = values[ev] + elif f.get("default"): + values[ev] = f["default"] + else: + values[ev] = f"drill-{name}" + if made: + guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made)) + say(f" [1] made the drive paths this app requires: {made}") + extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")] + if extra: + say(f" [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}") + return values + + +def deploy(name, sub, extra_values=None): + st = stack(name) + if st.get("deployed"): + say(f" [1] {name} already deployed — reusing") + return True + values = deploy_values(name, sub) + if extra_values: + values.update(extra_values) + body = {"values": values} + if os.environ.get("KEPT"): # decision 36: the household's answer when the drive holds old data ("fresh" moves it aside, deletes nothing) + body["kept_data"] = os.environ["KEPT"] + code, d = ctl("POST", f"/api/stacks/{name}/deploy", body) + say(f" [1] deploy -> {code} {str(d)[:120]}") + if code != "202": + return False + # WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the + # container `healthy` while the controller's own state read `unhealthy` — a gate on `running` + # alone therefore times out on an app that is up. The state is RECORDED rather than required; + # the real gate is the fixture's own `wait_app`, which asks whether the APP answers. + seen = None + for _ in range(90): + time.sleep(5) + st = stack(name) + seen = st.get("state") + # `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21: + # tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm + # read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the + # deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark + # (`runComposeDeploy` writes it), so that is what to wait for. + pins = (st.get("app_config") or {}).get("pinned_images") + if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"): + say(f" [1] deployed, controller state={seen}, " + f"pinned={(st.get('app_config') or {}).get('pinned_images')}") + if seen != "running": + say(f" [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, " + f"not treated as a failure; the fixture's front-door wait is the real gate") + return True + say(f" [1] never became deployed (last controller state={seen!r})") + return False + + +def backup_now(name): + """R-648 (2026-09-23): NO whole-box „Mentés most" from a drill, ever. + + `POST /api/backup/run` is the only backup endpoint and it is WHOLE-BOX: on 9201 it stopped and + restarted 9 of 10 standing apps twice, and on 9202 it broke a deploy in flight (R-634). The product + has NO per-app backup endpoint (router.go: /backup/run, /backup/tier2 only); the per-app backup + exists only inside the guarded update, whose `backing-up` phase calls RunAppBackupNow for the one + app. So this presses nothing: the update takes the throwaway app's own backup, and says so in its + phase list. A seed written "after the backup" is therefore written before the update's own backup + — the undo's last-second copy is still the one that must bring it back.""" + say(f" [4] backup press SKIPPED for {name} (R-648: whole-box only; the update's backing-up phase backs up {name} alone)") + return None + +def drill_bump(app, frm, to, service_hint=None): + """Serialised across concurrent walks: one git working tree, one lock.""" + import fcntl + with open(f"{SC}/drill.lock", "w") as lk: + fcntl.flock(lk, fcntl.LOCK_EX) + sh(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) + return _drill_bump(app, frm, to, service_hint) + + +def _drill_bump(app, frm, to, service_hint=None): + """Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates). + + `frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in + two images (adventurelog's backend and frontend) moves both in one edge, while its engine + sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move + every service that carries the app's own version and no others. + """ + comp = f"{DRILL}/templates/{app}/docker-compose.yml" + fy = f"{DRILL}/templates/{app}/.felhom.yml" + s = open(comp).read() + froms = [x.strip() for x in frm.split(",") if x.strip()] + tos = [x.strip() for x in to.split(",") if x.strip()] + if len(froms) != len(tos): + say(f" [5] from/to lists differ in length: {froms} vs {tos}") + return None + for f1, t1 in zip(froms, tos): + if f"image: {f1}" not in s: + say(f" [5] FROM ref not found in compose: {f1}") + return None + s = s.replace(f"image: {f1}", f"image: {t1}") + open(comp, "w").write(s) + f = open(fy).read() + today = datetime.now().strftime("%Y-%m-%d") + f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M) + open(fy, "w").write(f) + sh(["git", "-C", DRILL, "add", "-A"]) + sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"]) + r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120) + h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip() + say(f" [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})") + return h + + +def sync_rescan(expect_app=None, expect_ref=None, tries=12, delay=5): + """Sync, rescan, and — when told what to expect — WAIT FOR THE BADGE TO CATCH UP. + + R-607: `POST /api/sync` answers "nincs valtozas" while the catalog HAS moved, and + `catalog_images` stays stale until a separate rescan. Tonight showed the rescan alone is not + enough either: mealie's badge read "Naprakesz" seconds after its bump was pushed, and the + Update that followed moved nothing and still reported "Frissitve". So when the caller knows + which reference should appear, this polls for it and SAYS HOW LONG IT TOOK — which is the + NUMBER R-607 asks for and has never had. + """ + t0 = time.time() + ctl("POST", "/api/sync") + time.sleep(2) + ctl("POST", "/api/stacks/rescan") + time.sleep(2) + if not expect_app or not expect_ref: + return None + for i in range(tries): + cat = stack(expect_app).get("catalog_images") or {} + if expect_ref in cat.values(): + waited = round(time.time() - t0, 1) + if i: + say(f" [sync] the badge needed {waited}s and {i+1} sync+rescan rounds to catch up " + f"to {expect_ref} — R-607's window, measured") + return waited + time.sleep(delay) + ctl("POST", "/api/sync") + time.sleep(1) + ctl("POST", "/api/stacks/rescan") + say(f" [sync] the badge NEVER caught up to {expect_ref} in {round(time.time()-t0,1)}s — " + f"catalog_images = {stack(expect_app).get('catalog_images')}") + return None + + +def badges(name): + out = {} + for lang, suffix in (("hu", ""), ("en", "?lang=en")): + h = page(f"/apps/{name}{suffix}") + m = re.findall(r']*title="([^"]*)"[^>]*>([^<]*)<', h) + out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3] + return out + + +def press_update(name, poll=1.0, cap_s=1800): + code, d = ctl("POST", f"/api/stacks/{name}/update") + say(f" [6] Update -> {code} {str(d)[:220]}") + if code not in ("202", "200"): + return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0} + phases, seen, t0 = [], None, time.time() + while time.time() - t0 < cap_s: + st = stack(name) + ph = st.get("update_phase") + if ph != seen: + seen = ph + rec = {"t": round(time.time() - t0, 1), "phase": ph, + "label": st.get("update_phase_label"), "updating": st.get("updating"), + "error": st.get("update_error"), "hold": st.get("hold_reason")} + phases.append(rec) + say(f" +{rec['t']:>6.1f}s phase={ph} label={rec['label']} " + f"err={rec['error']} hold={rec['hold']}") + if not st.get("updating") and ph in ("done", "failed", "undone", None) and time.time() - t0 > 3: + break + time.sleep(poll) + st = stack(name) + return {"accepted": True, "http": code, "phases": phases, + "duration_s": round(time.time() - t0, 1), + "final_phase": st.get("update_phase"), "update_error": st.get("update_error"), + "hold_reason": st.get("hold_reason"), "state": st.get("state")} + + +def observables(name): + st = stack(name) + ac = st.get("app_config") or {} + live = guest(f""" +grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//' +echo '---inspect---' +for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do + echo -n "$c "; docker inspect "$c" --format '{{{{.Config.Image}}}} running={{{{.State.Running}}}} restarts={{{{.RestartCount}}}}' +done +""") + a, _, b = live.partition("---inspect---") + return { + "pinned_images": ac.get("pinned_images"), + "installed_images": {k: (v.get("ref") if isinstance(v, dict) else v) + for k, v in (ac.get("installed_images") or {}).items()}, + "catalog_images": st.get("catalog_images"), + "live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()], + "docker_inspect": [x for x in b.strip().splitlines() if x.strip()], + } + + +def app_logs(name, lines=400): + """The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is + one string with escaped newlines — a scan over the envelope sees a single enormous line and + finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96 + rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines.""" + code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}") + if isinstance(d, dict): + data = d.get("data") + if isinstance(data, dict) and isinstance(data.get("logs"), str): + return data["logs"] + if isinstance(d.get("_raw"), str): + return d["_raw"] + return str(d) + + +def write_verdict(rec, appdir): + os.makedirs(appdir, exist_ok=True) + p = os.path.join(appdir, "verdict.json") + json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False) + say(f" [9] verdict {rec['verdict']} -> {p}") + + +def remove(name): + """Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint + refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up.""" + c1, d1 = ctl("POST", f"/api/stacks/{name}/stop") + say(f" [X] stop -> {c1} {str(d1)[:100]}") + for _ in range(24): + time.sleep(5) + if stack(name).get("state") != "running": + break + code, d = ctl("POST", f"/api/stacks/{name}/remove", + {"remove_hdd_data": True, "remove_backups": True}) + say(f" [X] remove (with drive data) -> {code} {str(d)[:160]}") + if code == "409": + # R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive + # path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202 + # `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits + # this. The household's other choice — remove the app, KEEP the data — is accepted, and the + # harness takes it, then tidies its own directory by name at teardown. + say(" [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)" + " — removing the app and KEEPING the drive data instead") + code, d = ctl("POST", f"/api/stacks/{name}/remove", + {"remove_hdd_data": False, "remove_backups": True}) + say(f" [X] remove (keeping drive data) -> {code} {str(d)[:160]}") + time.sleep(5) + st = stack(name) + left = guest(f"ls -d /opt/docker/stacks/{name} 2>/dev/null; " + f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'") + say(f" [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}") + return code + + +def app_env(name, key): + """Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the + app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller + shows them the same value. Data still goes in through the app's own front door.""" + out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1") + if ":" in out: + return out.split(":", 1)[1].strip().strip('"').strip("'") + return "" + + +def snapshots(name): + """The restorable copies the backups page offers for this app.""" + code, d = ctl("GET", f"/api/backup/snapshots?stack={name}") + data = d.get("data") if isinstance(d, dict) else None + if isinstance(data, dict): + for k in ("snapshots", "items", "restore_points"): + if isinstance(data.get(k), list): + return data[k] + return data if isinstance(data, list) else [] + + +def restore(name, snapshot_id=None, wait_s=1200): + """The household's own way out: the „Visszaállítás a mentésből" button on the backups page. + + A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`, + `snapshot_id` — because that is the button the sentence tells them to press. + """ + snaps = snapshots(name) + if snapshot_id is None: + if not snaps: + say(f" [R] no restorable copy offered for {name}") + return {"ok": False, "why": "no snapshot offered", "snapshots": snaps} + first = snaps[0] + snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id") + say(f" [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)") + sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip() + csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip() + r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}", + "-X", "POST", + "--data-urlencode", f"_csrf={csrf}", + "--data-urlencode", f"stack_name={name}", + "--data-urlencode", f"snapshot_id={snapshot_id}", + f"{BASE}/backup/restore"], timeout=180) + head = (r.stdout or "").split("\n")[0].strip() + loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")] + say(f" [R] POST /backup/restore -> {head} {loc[:1]}") + t0 = time.time() + last = None + while time.time() - t0 < wait_s: + code, d = ctl("GET", "/api/backup/restore-status") + dd = d.get("data") or {} + cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message")) + if cur != last: + say(f" +{round(time.time()-t0,1):>6.1f}s restore {cur}") + last = cur + if not dd.get("running", False) and time.time() - t0 > 5: + break + time.sleep(2) + st = stack(name) + say(f" [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} " + f"phase={st.get('update_phase')}") + return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps, + "http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1), + "state_after": st.get("state"), "hold_after": st.get("hold_reason"), + "observables_after": observables(name)} diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index 191b9026..91fd6da5 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -854,15 +854,15 @@ class (an image `VOLUME` at an unmounted path) is still live — `immich-server` | **R-740** | **[P2-MEDIUM] A security fix that upstream ships under the SAME tag (`postgres:18-alpine`, `redis:7-alpine`, `mariadb:12.3` …) reaches NO box — not at night, and not by the household's Update button either, because the catalog never records a re-test of a tag at a new digest.** MEASURED 2026-09-30 (more-night-apps brief, Part C). **(1) The night leg, from source** (`felhom-controller` `d48da6c`): an app at the head tag whose running digest is older than the ladder's tested one reads Behind (`stacks/updateorder.go:86–111` `digestBehind`), but `legCandidate` finds no entry whose `from` is its pin and skips it — `unattended.go:433–435`, `return LegSkipNoTestRecord // at the head, behind only by something no step records`. A unit walk on a scratch copy of the controller confirms it: order Behind → the leg presses nothing, `skipped — no_test_record`; the household's Update button in the same state ends `done` and runs the tested digest (`audits/more-night-apps-2026-09-30/C/C2-*`). **(2) But the catalog never produces that state for a floating tag:** the only writer refuses a step that moves no image (`upgrade-test.py:919`, „--move: nothing moves"), and no ladder in the catalog has an entry with `from == to` or two entries with the same `to` (`C/C4-same-tag-retest.txt`). So the tested digest of `postgres:18-alpine` stays the one of the day the step was tested, a box installed since runs that digest, and the badge reads „Naprakész" while upstream has moved. **(3) How often it matters:** of the 11 distinct floating lines in the catalog today (26 services), the 8 on Docker Hub were pushed 9, 9, 9, 9, 12, 12, 30 and 105 days ago — **7 of 8 within 30 days** (`C/C3-floating-repush.txt`; a push is not proof that the image content changed; ghcr gives no date). Today every tested digest still equals what the registry serves (the tests came after the pushes). Only a box installed BEFORE a step's test can read Behind by digest; the demo boxes have none (`C/C1-digests-demo-hp-9201.txt`, 18 of 18 equal). **(4) Decision 30's cost line says the older image runs „until someone (or the automatic leg) presses Update"** — the automatic leg never does, and the manual press moves the digest only in the legacy case above. **(5) What the box would do if the catalog wrote a same-tag re-test** (unit walk): the leg PRESSES it with no controller change — the newest entry whose `from` is the pin is taken, the update renders the new tested digest, the next night reads Current. So option (a)'s cost is in the catalog. **Needs: a decision (STATUS, 2026-09-30).** **-- BUILT 2026-09-30 late (decision 52, option A):** catalog `6a3ead9` — a re-test entry (`from` == `to`, `digest_from`, `box_evidence`), refused by the gates with no new digest, a digest the registry stopped serving, no box proof, or a `digest_from` that is not the previous digest (8 decoys, seen red); `upgrade-test.py --retest`; the ONE command `scripts/retest-floating.py` (`--dry-run`, `--engines-only`). **Proven end to end on 9202:** docmost at an older `redis:7-alpine` digest, the re-test, „run tonight's chain now”, the leg pressed it (`step ended done after 95.0 s`), the new digest runs, data read back, badge current. **Run for real:** no database/redis line differs today (two exact tags do — R-743). Not a cron job (runbook `runbooks/monthly-floating-retest.md` says why); who presses it monthly is the operator's word (STATUS). `audits/night-rulings-2026-09-30/` | **CLOSED 2026-09-30 — built (catalog `6a3ead9`); the monthly run is a standing step** | | **R-741** | **[P3-LOW] For a few seconds after a fresh install, an `after_install` app answers its PUBLIC default login through the household's front door — the box changes the password only after the app already serves.** MEASURED 2026-09-30 on scratch guest 9202 (calibre-web, controller 0.283.1): polling `GET /opds` with `admin:admin123` (the image's README default) through traefik once a second from the deploy press: 404 until the route existed, **200 at 16:42:06**, 401 from 16:42:07 on (`after_install` done). The first install the same day logged the app started at 15:02:36 and `after_install … done` at 15:02:54 — a fixture probe at 15:02:52 still saw the new password refused, so that window was up to ~18 s. Decision 45 („no app is published with a login a stranger knows") holds after the window, not during it. Who can reach it: anyone who can resolve the app's name in those seconds; calibre-web is not behind the setup gate. Applies to every `after_install` app (calibre-web, mealie, wger, bookstack, claper, …) — not measured for the others. **Needs:** the route published only after `after_install` succeeds (the gate's own route hold, or traefik labels applied after), or the app behind the setup gate until then. `audits/more-night-apps-2026-09-30/A/A3-calibre-default-login-window.txt` **-- FIXED 2026-09-30 late: controller v0.284.x** — an `after_install` app is installed HELD behind the setup gate's door until the login is replaced (or the household says it changed it). **Live on 9202, as a stranger:** calibre-web 0 of 192 default-login tries got in (hold before the first start 20:36:32; opened by after_install 20:37:18, 17 s after the app was up; then 401); mealie 0 of 97 (then mealie's own lock — R-747). The positive control with the generated password was not obtained (calibre-web: a backup stopped the app at that moment; mealie: the lock). Red-proofs RP-IH1..4. `audits/night-rulings-2026-09-30/` | **CLOSED 2026-09-30 — controller v0.284.2** | | **R-742** | **[P3-LOW] zipline 4.8.0 cannot be reached from 4.6.1 in one step: it refuses to start until the database has run the release before it.** MEASURED 2026-09-30 on both venues, the same way: 4.6.1 → 4.8.0 — the new container restarts (exit 1) with `Error: cannot safely migrate from prisma to drizzle: expected migration 20260508022000_add_file_folder_created_at_index was not applied. To resolve this, repair the database with the previous (latest before this) Zipline release before upgrading` (bench `to-full.log`, 15×). **On box 9202 the product's guarded Update saw it unhealthy after 1 m 30 s and UNDID it by itself in 20 s — the previous version back on the data from before the update** (`box/zipline/step.txt`): the undo worked on a real upstream failure, not a drill. Bench verdict `failed`, box `undone`; nothing written. **Also found:** the zipline fixture had two faults, both fixed (`/` answers 301 → wait on `/api/healthcheck`; a wrong-password control first trips its login limit → 429 on the right one; the right password now goes first). **Needs:** the ladder climbs 4.6.1 → 4.7.x → 4.8.0 (two tested steps — the ladder exists for exactly this). `audits/more-night-apps-2026-09-30/bench/apps/zipline/`, `box/zipline/` **-- DONE 2026-09-30 (evening):** the two steps, each proven on both venues — 4.6.1 → 4.7.0 (catalog `a9700e2`; box 103.6 s, bench 0 kills) and 4.7.0 → 4.8.0 (`fb87030`; box 32.8 s, bench 0 kills). | **CLOSED 2026-09-30 — catalog `a9700e2` + `fb87030`** | -| **R-743** | **[P3-LOW] Exact version tags are re-pushed under the same name too — not only floating lines.** MEASURED 2026-09-30 by `retest-floating.py --dry-run` on the live catalog (`6a3ead9`): `nextcloud:34.0.4-apache` (tested `a5ace30c…`, registry `37b10988…`) and `lscr.io/linuxserver/sonarr:4.0.20` (tested `a5c1a5fe…`, registry `f247545d…`) — the registry now serves another build under a tag the ladder tested. No database or redis line differed. Decision 52 starts with the engine lines, so these were NOT re-tested (`--engines-only`). linuxserver rebuilds its images weekly under the same tags, so every linuxserver app will show up here. **Needs:** a word on whether the monthly run covers every app (the command does; `--engines-only` is the switch) or only engines. `audits/night-rulings-2026-09-30/A/` | **WAITING-ON-OPERATOR — rank P3-LOW; owner: VIKTOR decides, CC runs** | -| **R-744** | **[P3-LOW] outline's fixture cannot seed outline 1.10.1: no `csrfToken` cookie after `installation.create`.** MEASURED 2026-09-30 on both venues (bench and 9202, the end-to-end re-test's first attempt): `POST /api/installation.create` answered 302 with the session, and `GET /home` set no `csrfToken` cookie (1.9.1 did; the 1.9.1 → 1.10.1 step read back that afternoon because its read-back uses the API key made at 1.9.1). So the NEXT outline step, and any re-test of outline, stops at C1 with „no csrfToken cookie from GET /home". **Needs:** the fixture reads outline 1.10.1's CSRF the way its page does (measure first). `audits/night-rulings-2026-09-30/A/e2e/*outline-attempt*` | **READY — rank P3-LOW; owner: CC (catalog harness)** | -| **R-745** | **[P3-LOW] Old CONTROLLER images are never deleted: ~50 versions on each demo box.** MEASURED 2026-09-30 after v0.284.2's one-time sweep: every image no container uses on demo-hp and on the N100 is a `felhom-controller` tag (0.201.0 … 0.283.1); the N100 still reads 3.6 GB reclaimable. Decision 53 covers APP images, and the sweep leaves the controller's own images alone on purpose (the self-update may need the previous one). A release is ~150 MB and there are several a day. **Needs:** the same rule for the controller — keep the running and the previous tag — as a decision (the self-update's rollback reads which image?). `audits/night-rulings-2026-09-30/E/` | **READY — rank P3-LOW; owner: CC (controller); the rule needs a word** | -| **R-746** | **[P3-LOW] `image_digest.resolve` ignores a `@digest` in its argument — it answers the TAG's current digest.** MEASURED 2026-09-30: `resolve('redis:7-alpine@sha256:000…0')` returned `sha256:858f…` (the tag's), while a manifest request for that digest answers 404. Every gate today passes it a plain tag, so no gate is wrong; a caller that passes `ref@digest` to ask "is THIS digest still served" gets a false yes. **Needs:** refuse a digest-carrying ref, or ask for the manifest by digest (with a test). `scripts/image_digest.py` | **READY — rank P3-LOW; owner: CC (catalog)** | +| **R-743** | **[P3-LOW] Exact version tags are re-pushed under the same name too — not only floating lines.** MEASURED 2026-09-30 by `retest-floating.py --dry-run` on the live catalog (`6a3ead9`): `nextcloud:34.0.4-apache` (tested `a5ace30c…`, registry `37b10988…`) and `lscr.io/linuxserver/sonarr:4.0.20` (tested `a5c1a5fe…`, registry `f247545d…`) — the registry now serves another build under a tag the ladder tested. No database or redis line differed. Decision 52 starts with the engine lines, so these were NOT re-tested (`--engines-only`). linuxserver rebuilds its images weekly under the same tags, so every linuxserver app will show up here. **Needs:** a word on whether the monthly run covers every app (the command does; `--engines-only` is the switch) or only engines. `audits/night-rulings-2026-09-30/A/` **-- 2026-10-01:** Operator ruling `09` §3 decisions 54 (a CC session the operator starts with the standing brief runs it monthly) and 55 (every app with a proven ladder; `--engines-only` a switch). The first full run: nextcloud and sonarr re-tested on both venues and written (catalog `3b59dfb` nextcloud, `1a37032` sonarr), ~17 min per app (bench ~13 incl. the 10-minute watch, box ~4). `audits/retest-2026-10/`. | **CLOSED 2026-10-01 — decisions 54, 55; both tags re-tested** | +| **R-744** | **[P3-LOW] outline's fixture cannot seed outline 1.10.1: no `csrfToken` cookie after `installation.create`.** MEASURED 2026-09-30 on both venues (bench and 9202, the end-to-end re-test's first attempt): `POST /api/installation.create` answered 302 with the session, and `GET /home` set no `csrfToken` cookie (1.9.1 did; the 1.9.1 → 1.10.1 step read back that afternoon because its read-back uses the API key made at 1.9.1). So the NEXT outline step, and any re-test of outline, stops at C1 with „no csrfToken cookie from GET /home". **Needs:** the fixture reads outline 1.10.1's CSRF the way its page does (measure first). `audits/night-rulings-2026-09-30/A/e2e/*outline-attempt*` **-- 2026-10-01:** Outline 1.10 names the cookie `__Host-csrfToken` on a secure request (`server/utils/csrf.ts`); the fixture accepts both names (catalog `9fc7052`). Proven on 9202 at the live pin 1.10.1: seed, read back, unknown id and wrong key refused. Outline has no newer release than 1.10.1, so no step. `audits/rulings-2026-10-01/D/D2-outline-fixture-9202.txt` | **CLOSED 2026-10-01 — fixture fixed, proven on 9202** | +| **R-745** | **[P3-LOW] Old CONTROLLER images are never deleted: ~50 versions on each demo box.** MEASURED 2026-09-30 after v0.284.2's one-time sweep: every image no container uses on demo-hp and on the N100 is a `felhom-controller` tag (0.201.0 … 0.283.1); the N100 still reads 3.6 GB reclaimable. Decision 53 covers APP images, and the sweep leaves the controller's own images alone on purpose (the self-update may need the previous one). A release is ~150 MB and there are several a day. **Needs:** the same rule for the controller — keep the running and the previous tag — as a decision (the self-update's rollback reads which image?). `audits/night-rulings-2026-09-30/E/` **-- 2026-10-01:** Decision 56 built: controller v0.285.0 keeps the running image + the previous (the swap record; version order as fallback), deletes older/untagged ones. **Measured first:** the agent rolls back to the RUNNING image (what `/etc/felhom-controller-image` named), never to a previous one the controller hands it. After the floor: demo-hp 84 → 2 controller images (Docker images 15.2 → 11.65 GB), the N100 76 → 2 (6.07 → 1.11 GB), 9202 5 → 2. `audits/rulings-2026-10-01/B/` | **CLOSED 2026-10-01 — controller v0.285.0, floor 0.285.0** | +| **R-746** | **[P3-LOW] `image_digest.resolve` ignores a `@digest` in its argument — it answers the TAG's current digest.** MEASURED 2026-09-30: `resolve('redis:7-alpine@sha256:000…0')` returned `sha256:858f…` (the tag's), while a manifest request for that digest answers 404. Every gate today passes it a plain tag, so no gate is wrong; a caller that passes `ref@digest` to ask "is THIS digest still served" gets a false yes. **Needs:** refuse a digest-carrying ref, or ask for the manifest by digest (with a test). `scripts/image_digest.py` **-- 2026-10-01:** `resolve()` asks the registry for the manifest BY DIGEST when the ref carries one; a malformed digest is refused without a request (catalog `804884a`, `scripts/test_image_digest.py`; red: the old resolver fails 3 of 4). Live: `redis:7-alpine@sha256:000…0` → HTTP 404 (was the tag's digest). `audits/rulings-2026-10-01/D/D1-r746-digest.txt` | **CLOSED 2026-10-01 — catalog 804884a** | | **R-747** | **[P3-LOW] A stranger can lock the household out of mealie with five wrong logins.** MEASURED 2026-09-30 on 9202 (the R-741 proof): after the install hold opened, a stranger's default-login tries were refused (401) and after five of them mealie answered 423 (locked) to every login — the generated, correct password included. mealie's own brute-force guard, on an app published on the internet; the setup gate and the install hold do not cover an app after its first setup. Not measured: how long the lock lasts. **Needs:** measure the lock's length; decide whether the page tells the household what to do. `audits/night-rulings-2026-09-30/C/C3-mealie-poll.txt` | **READY — rank P3-LOW; owner: CC** | | **R-748** | **[P3-LOW] The register-shape gate skipped every row whose id has a letter suffix — so R-88a, R-88b and R-209a were never shape-checked, and its count read 3 short.** FOUND 2026-09-30 (late) while counting the register: `register_shape_gate.py` matched `R-\d+` only; the brief's „the reviewer's regex undercounted by 3” is the same three rows. Fixed the same session: `R-\d+[a-z]?`; decoy `suffix-row-eaten-state` (a suffixed row with its state cell eaten) seen passing with the old pattern and convicted with the new. The register is **382** rows by either count now. | **CLOSED 2026-09-30 — `scripts/register_shape_gate.py`** | -| **R-749** | **[P3-LOW] `retest-floating.py` could never start on a fresh bench: it checked for `/opt/upg/upgrade-test.py` on the bench BEFORE the step that copies it there.** FOUND 2026-10-01 at the first full monthly run (decision 55): bench 9401 freshly created by the runbook, the run answered „CANNOT START — missing: the bench LXC 9401 on demo-hp with /opt/upg” in one minute. The runbook says the command syncs the bench itself — it does, but only after the check. On 2026-09-30 the bench had been synced by hand earlier, so nobody saw it. **Needs:** the check asks for what the bench must bring (docker, python3), the sync then provides `/opt/upg`. `audits/rulings-2026-10-01/A/` | **OPEN — rank P3-LOW; owner: CC (catalog harness)** | +| **R-749** | **[P3-LOW] `retest-floating.py` could never start on a fresh bench: it checked for `/opt/upg/upgrade-test.py` on the bench BEFORE the step that copies it there.** FOUND 2026-10-01 at the first full monthly run (decision 55): bench 9401 freshly created by the runbook, the run answered „CANNOT START — missing: the bench LXC 9401 on demo-hp with /opt/upg” in one minute. The runbook says the command syncs the bench itself — it does, but only after the check. On 2026-09-30 the bench had been synced by hand earlier, so nobody saw it. **Needs:** the check asks for what the bench must bring (docker, python3), the sync then provides `/opt/upg`. `audits/rulings-2026-10-01/A/` **-- 2026-10-01:** Fixed the same session (catalog `9e53205`): the check asks for docker + python3; after the sync `/opt/upg/upgrade-test.py` is required. The re-run started at once and finished both apps. | **CLOSED 2026-10-01 — catalog 9e53205** | | **R-750** | **[P3-LOW] The registry no longer holds controller releases older than 0.213.0 (2026-08-12) — something removed them, and nothing records what.** MEASURED 2026-10-01 (anonymous registry API, `audits/rulings-2026-10-01/B/`): `felhom-controller` has 91 tags, the oldest release 0.213.0; `0.201.0` answers 404; Gitea's package list starts 2026-08-12. No runbook, row or memory names a clean-up. Today nothing needs those versions: a box runs a newer one, a whole-guest restore brings the guest's own Docker store back (mp0 `backup=1`), and decision 56 deletes only on the box. **But** a box or a backup that names a removed version cannot pull it again (R-698's shape, for the controller). **Needs:** find what removed them (a Gitea clean-up rule?), and record the rule — or say it was a one-time act. Read-only on DooPlex. | **OPEN — rank P3-LOW; owner: operator (Gitea settings), CC measures** | -| **R-751** | **[P2] The image clean-up after an app update could crash the whole controller: it re-read the app after a rescan and dereferenced a nil stack when the app was gone.** FOUND 2026-10-01 by the full test suite (controller v0.284.2): `RetainImagesAfterUpdate` runs in a goroutine; `TestR705_TheManualLegRunsByDay` removed its temp dir under it → `panic: invalid memory address` at `image_retention.go:291`. In a box the same happens when an app is removed (or its compose vanishes) between an update's end and the clean-up — a panic in a goroutine ends the process (the agent's supervisor restarts it). Fixed in v0.285.0 the same session: it returns when the app is gone; `TestRetainImagesAfterUpdate_AppGoneDoesNotPanic` seen panicking on the old code; both retention seams are no-ops in the stacks tests (`TestMain`), so no test leaves the goroutine running. `audits/rulings-2026-10-01/B/B1-red-proofs.txt` | **OPEN — fixed in controller v0.285.0, closes when the floor delivers it; owner: CC** | +| **R-751** | **[P2] The image clean-up after an app update could crash the whole controller: it re-read the app after a rescan and dereferenced a nil stack when the app was gone.** FOUND 2026-10-01 by the full test suite (controller v0.284.2): `RetainImagesAfterUpdate` runs in a goroutine; `TestR705_TheManualLegRunsByDay` removed its temp dir under it → `panic: invalid memory address` at `image_retention.go:291`. In a box the same happens when an app is removed (or its compose vanishes) between an update's end and the clean-up — a panic in a goroutine ends the process (the agent's supervisor restarts it). Fixed in v0.285.0 the same session: it returns when the app is gone; `TestRetainImagesAfterUpdate_AppGoneDoesNotPanic` seen panicking on the old code; both retention seams are no-ops in the stacks tests (`TestMain`), so no test leaves the goroutine running. `audits/rulings-2026-10-01/B/B1-red-proofs.txt` **-- 2026-10-01:** Delivered: floor 0.285.0 reached both demo boxes in ~6 s (hub `managed floor SERVED … from declared`). | **CLOSED 2026-10-01 — controller v0.285.0, floor 0.285.0** | | **R-752** | **[P3-LOW] Four more catalog apps let a stranger lock the household out with wrong passwords for a known login name — like mealie (R-747).** READ 2026-10-01 in each app's source at its pinned tag (not measured live): **calibre-web-automated v4.0.8** — Flask-Limiter on the login keyed on the lowercased USERNAME, 3/minute and 40/day, checked before the password; the default login is `admin` → up to a day; no env switch (a database setting). **wger 2.7** — django-axes keyed on IP, 10 failures, 30 min, each failure restarts it; behind traefik every client has traefik's IP → everyone is locked out (`AXES_*` env vars exist; `AXES_IPWARE_PROXY_COUNT` 0). **Grafana 13.2.3** — per-account, 5 failures in a sliding 5 minutes; a slow trickle keeps it closed (`GF_SECURITY_*`). **BookStack 26.09.1** — key `email|ip`, 5 tries, 60 s, hard-coded; `APP_PROXIES` empty, so the key is the e-mail alone. gokapi (3 s delay, no lock) and claper (per-IP 10/min, no account lock) cannot. **Needs:** per app, the smallest fix that keeps a guessing guard (calibre-web-automated and wger first — longest and broadest), each proven on 9202 as R-747's was. | **OPEN — rank P3-LOW; owner: CC** |