diff --git a/documentation/tests/golden-0.223.0-2026-08-23/README.md b/documentation/tests/golden-0.223.0-2026-08-23/README.md new file mode 100644 index 00000000..06a15938 --- /dev/null +++ b/documentation/tests/golden-0.223.0-2026-08-23/README.md @@ -0,0 +1,48 @@ +# Golden bake — controller 0.223.0 (2026-08-23) + +**Baked and PUBLISHED by Claude Code. NOT vouched — vouching is the operator's act.** + +| Field | Value | +|---|---| +| `GOLDEN_VERSION` | `0.223.0` | +| `GOLDEN_SHA256` | `9eaf39ac39219b42ec9e6cbf890275febcdcc6f53325fe0c0f591d3431044f17` | +| Controller image | `gitea.dooplex.hu/admin/felhom-controller:0.223.0` | +| MinAgent | `0.129.0` (unchanged) | +| Package URL | `https://gitea.dooplex.hu/api/packages/admin/generic/felhom-golden/0.223.0/golden.tar.zst` | +| Archive size | 657,261,745 B | +| LXC template | `debian-13-standard_13.6-1_amd64.tar.zst` | + +## Acceptance markers, each counted from `bake.log` + +| Marker | Required | Observed | +|---|---|---| +| `docker OK (overlay2` | ≥1 | **1** — `docker OK (overlay2; data-root /var/lib/docker)` | +| `including mount point` (rootfs + mp0) | 2 | **2** | +| `upload OK (HTTP 201)` | 1 | **1** | +| `excluding` | 0 | **0** | +| `FATAL` | 0 | **0** | + +Round trip on the published package: **HTTP 206** on a ranged GET. + +## Why this release needs a golden + +`app_start_failed` is now deliverable and gains a customer-facing toggle — customer-visible behaviour +on a fresh install. A machine installed from the previous golden would receive a controller whose +app-down alarm reaches nobody. + +## The runbook step that is still missing + +§4.1 does not say to run **`pveam update`** first. On the `virgin` snapshot the template *index* is +stale, so `pveam available` offers an old point release and downloading it fails with +`400 Parameter verification failed. template: no such template` — a confusing 400 rather than a +legible "your index is old". Second bake in a row to hit it; recorded in the workspace memory as +`golden-bake-needs-pveam-update`. The runbook itself is still not edited. + +## Vouching — the OPERATOR's step, not done here + +Hub → Configuration → Day-0 artifacts: + +- `golden_version` → `0.223.0` +- `golden_sha256` → `9eaf39ac39219b42ec9e6cbf890275febcdcc6f53325fe0c0f591d3431044f17` +- `min_agent` → `0.129.0` (unchanged) +- then, **last and in its own save**, the global controller floor → `0.223.0`. diff --git a/documentation/tests/golden-0.223.0-2026-08-23/bake.log b/documentation/tests/golden-0.223.0-2026-08-23/bake.log new file mode 100644 index 00000000..2addd96f --- /dev/null +++ b/documentation/tests/golden-0.223.0-2026-08-23/bake.log @@ -0,0 +1,326 @@ +[golden] build-golden.sh v3.0.0 — baking controller gitea.dooplex.hu/admin/felhom-controller:0.223.0 +[golden] creating build LXC 9100 (nesting=1,keyctl=1, unprivileged; rootfs 32G + ONE data volume 24G @ /var/lib/felhom, backup=1) … + Logical volume "vm-9100-disk-0" created. + Logical volume pve/vm-9100-disk-0 changed. +Creating filesystem with 8388608 4k blocks and 2097152 inodes +Filesystem UUID: 49218b02-e17f-43fe-b66d-c973181a88fb +Superblock backups stored on blocks: + 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, + 4096000, 7962624 + Logical volume "vm-9100-disk-1" created. + Logical volume pve/vm-9100-disk-1 changed. +Creating filesystem with 6291456 4k blocks and 1572864 inodes +Filesystem UUID: 8cf237fc-c7d0-4415-9e7d-7b1f2b01d73d +Superblock backups stored on blocks: + 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, +extracting archive '/var/lib/vz/template/cache/debian-13-standard_13.6-1_amd64.tar.zst' +Total bytes read: 553512960 (528MiB, 125MiB/s) +Detected container architecture: amd64 +Creating SSH host key 'ssh_host_ed25519_key' - this may take some time ... +done: SHA256:83obOHb4ekH6cHFXkdyyy6B7+/HAK2l/X+XsfcrUxBU root@felhom-golden +Creating SSH host key 'ssh_host_rsa_key' - this may take some time ... +done: SHA256:S7Bk+RG1RHa6o4fmaVbc1Z8VhjRnkdNk4bUd/US88CE root@felhom-golden +Creating SSH host key 'ssh_host_ecdsa_key' - this may take some time ... +done: SHA256:FHJcuK8+eXyM7xlidgxZCjAHqpsXuR2uARtPU17/zLE root@felhom-golden +[golden] starting + installing Docker (official repo, trixie channel) … +apt-listchanges: Can't set locale; make sure $LC_* and $LANG are correct! +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = (unset), + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to the standard locale ("C"). +locale: Cannot set LC_CTYPE to default locale: No such file or directory +locale: Cannot set LC_MESSAGES to default locale: No such file or directory +locale: Cannot set LC_ALL to default locale: No such file or directory +apt-listchanges: Can't set locale; make sure $LC_* and $LANG are correct! +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = (unset), + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to the standard locale ("C"). +locale: Cannot set LC_CTYPE to default locale: No such file or directory +locale: Cannot set LC_MESSAGES to default locale: No such file or directory +locale: Cannot set LC_ALL to default locale: No such file or directory +[golden] baking daemon.json: classic overlay2 driver (containerd-snapshotter OFF) + log rotation … +[golden] wiring the single data volume (R-165 variant V-c): /var/lib/felhom/{docker,sys_drive} -> binds … +[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) … +Unable to find image 'hello-world:latest' locally +latest: Pulling from library/hello-world +4f55086f7dd0: Pulling fs layer +4f55086f7dd0: Verifying Checksum +4f55086f7dd0: Download complete +4f55086f7dd0: Pull complete +Digest: sha256:5dd0d3e6e255913fc30f90b9f2b1d359cc2cbdb48090cc4b65f1676e203243cc +Status: Downloaded newer image for hello-world:latest + docker OK (overlay2; data-root /var/lib/docker) + /var/lib/docker is a real mount: /dev/mapper/pve-vm--9100--disk--1[/docker] ext4 + /mnt/sys_drive is a real mount: /dev/mapper/pve-vm--9100--disk--1[/sys_drive] ext4 + both paths are ONE filesystem: /dev/mapper/pve-vm--9100--disk--1 23317576 +[golden] baking the in-guest controller image gitea.dooplex.hu/admin/felhom-controller:0.223.0 (no registry cred at deploy) … + +WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'. +Configure a credential helper to remove this warning. See +https://docs.docker.com/go/credential-store/ + +0.223.0: Pulling from admin/felhom-controller +039e6f9f9752: Pulling fs layer +0094c3ac0914: Pulling fs layer +deca1dac7403: Pulling fs layer +11c19a33d1b8: Pulling fs layer +346e1e015691: Pulling fs layer +434fa1745bf7: Pulling fs layer +434fa1745bf7: Waiting +346e1e015691: Waiting +11c19a33d1b8: Waiting +deca1dac7403: Verifying Checksum +deca1dac7403: Download complete +11c19a33d1b8: Verifying Checksum +11c19a33d1b8: Download complete +346e1e015691: Verifying Checksum +346e1e015691: Download complete +434fa1745bf7: Verifying Checksum +434fa1745bf7: Download complete +0094c3ac0914: Verifying Checksum +0094c3ac0914: Download complete +039e6f9f9752: Verifying Checksum +039e6f9f9752: Download complete +039e6f9f9752: Pull complete +0094c3ac0914: Pull complete +deca1dac7403: Pull complete +11c19a33d1b8: Pull complete +346e1e015691: Pull complete +434fa1745bf7: Pull complete +Digest: sha256:de0908f103bdd1d9b59b3bb3922e628b789136d22e7295e1622d6e0c8116aad0 +Status: Downloaded newer image for gitea.dooplex.hu/admin/felhom-controller:0.223.0 +gitea.dooplex.hu/admin/felhom-controller:0.223.0 +[golden] asking the controller which infra images it manages … +[golden] baking infra images (4): traefik:v3.6.7 cloudflare/cloudflared:2026.6.0 gtstef/filebrowser:1.3.3-stable gitea.dooplex.hu/admin/felhom-samba:1.1.0 … +v3.6.7: Pulling from library/traefik +589002ba0eae: Pulling fs layer +ef63511ea6cc: Pulling fs layer +0738e5cb835e: Pulling fs layer +3e6813f70c64: Pulling fs layer +3e6813f70c64: Waiting +589002ba0eae: Download complete +ef63511ea6cc: Verifying Checksum +ef63511ea6cc: Download complete +3e6813f70c64: Verifying Checksum +3e6813f70c64: Download complete +0738e5cb835e: Verifying Checksum +0738e5cb835e: Download complete +589002ba0eae: Pull complete +ef63511ea6cc: Pull complete +0738e5cb835e: Pull complete +3e6813f70c64: Pull complete +Digest: sha256:a9890c898f379c1905ee5b28342f6b408dc863f08db2dab20e46c267d1ff463a +Status: Downloaded newer image for traefik:v3.6.7 +docker.io/library/traefik:v3.6.7 +2026.6.0: Pulling from cloudflare/cloudflared +47de5dd0b812: Pulling fs layer +c172f21841df: Pulling fs layer +99515e7b4d35: Pulling fs layer +99ba982a9142: Pulling fs layer +d6b1b89eccac: Pulling fs layer +2780920e5dbf: Pulling fs layer +7c12895b777b: Pulling fs layer +3214acf345c0: Pulling fs layer +52630fc75a18: Pulling fs layer +dd64bf2dd177: Pulling fs layer +b839dfae01f6: Pulling fs layer +ebddc55facdc: Pulling fs layer +bdfd7f7e5bf6: Pulling fs layer +2d4d7adf6272: Pulling fs layer +40008157d8d2: Pulling fs layer +bd8962e29291: Pulling fs layer +cac2ae0193cb: Pulling fs layer +74d1dac84ecc: Pulling fs layer +dd64bf2dd177: Waiting +b839dfae01f6: Waiting +ebddc55facdc: Waiting +bdfd7f7e5bf6: Waiting +2d4d7adf6272: Waiting +40008157d8d2: Waiting +bd8962e29291: Waiting +cac2ae0193cb: Waiting +74d1dac84ecc: Waiting +2780920e5dbf: Waiting +7c12895b777b: Waiting +3214acf345c0: Waiting +52630fc75a18: Waiting +99ba982a9142: Waiting +d6b1b89eccac: Waiting +47de5dd0b812: Download complete +c172f21841df: Verifying Checksum +c172f21841df: Download complete +47de5dd0b812: Pull complete +99515e7b4d35: Verifying Checksum +99515e7b4d35: Download complete +99ba982a9142: Verifying Checksum +99ba982a9142: Download complete +d6b1b89eccac: Verifying Checksum +d6b1b89eccac: Download complete +2780920e5dbf: Verifying Checksum +2780920e5dbf: Download complete +7c12895b777b: Verifying Checksum +7c12895b777b: Download complete +3214acf345c0: Verifying Checksum +3214acf345c0: Download complete +52630fc75a18: Verifying Checksum +52630fc75a18: Download complete +dd64bf2dd177: Verifying Checksum +dd64bf2dd177: Download complete +c172f21841df: Pull complete +b839dfae01f6: Verifying Checksum +b839dfae01f6: Download complete +ebddc55facdc: Verifying Checksum +ebddc55facdc: Download complete +bdfd7f7e5bf6: Verifying Checksum +bdfd7f7e5bf6: Download complete +2d4d7adf6272: Verifying Checksum +2d4d7adf6272: Download complete +40008157d8d2: Verifying Checksum +40008157d8d2: Download complete +bd8962e29291: Verifying Checksum +bd8962e29291: Download complete +cac2ae0193cb: Download complete +99515e7b4d35: Pull complete +74d1dac84ecc: Verifying Checksum +74d1dac84ecc: Download complete +99ba982a9142: Pull complete +d6b1b89eccac: Pull complete +2780920e5dbf: Pull complete +7c12895b777b: Pull complete +3214acf345c0: Pull complete +52630fc75a18: Pull complete +dd64bf2dd177: Pull complete +b839dfae01f6: Pull complete +ebddc55facdc: Pull complete +bdfd7f7e5bf6: Pull complete +2d4d7adf6272: Pull complete +40008157d8d2: Pull complete +bd8962e29291: Pull complete +cac2ae0193cb: Pull complete +74d1dac84ecc: Pull complete +Digest: sha256:ba461b8aa9c042156dbd39c38657fe7431bafa063220eab8d5330a523863da9f +Status: Downloaded newer image for cloudflare/cloudflared:2026.6.0 +docker.io/cloudflare/cloudflared:2026.6.0 +1.3.3-stable: Pulling from gtstef/filebrowser +6a0ac1617861: Pulling fs layer +ef8806083e82: Pulling fs layer +b74107c861c7: Pulling fs layer +adc935def003: Pulling fs layer +4f4fb700ef54: Pulling fs layer +18695ccc900a: Pulling fs layer +45d119d5c397: Pulling fs layer +dac52db4fc51: Pulling fs layer +6d598f86b2f2: Pulling fs layer +8aa349c8396c: Pulling fs layer +dac52db4fc51: Waiting +6d598f86b2f2: Waiting +8aa349c8396c: Waiting +4f4fb700ef54: Waiting +18695ccc900a: Waiting +45d119d5c397: Waiting +adc935def003: Waiting +6a0ac1617861: Verifying Checksum +6a0ac1617861: Download complete +b74107c861c7: Verifying Checksum +b74107c861c7: Download complete +adc935def003: Verifying Checksum +adc935def003: Download complete +4f4fb700ef54: Verifying Checksum +4f4fb700ef54: Download complete +ef8806083e82: Verifying Checksum +ef8806083e82: Download complete +45d119d5c397: Verifying Checksum +45d119d5c397: Download complete +dac52db4fc51: Verifying Checksum +dac52db4fc51: Download complete +6a0ac1617861: Pull complete +6d598f86b2f2: Verifying Checksum +6d598f86b2f2: Download complete +18695ccc900a: Verifying Checksum +18695ccc900a: Download complete +8aa349c8396c: Verifying Checksum +8aa349c8396c: Download complete +ef8806083e82: Pull complete +b74107c861c7: Pull complete +adc935def003: Pull complete +4f4fb700ef54: Pull complete +18695ccc900a: Pull complete +45d119d5c397: Pull complete +dac52db4fc51: Pull complete +6d598f86b2f2: Pull complete +8aa349c8396c: Pull complete +Digest: sha256:eb3733681db8757412632c61a99ad656f0d94ed6781bb2ea114b4d70babab78c +Status: Downloaded newer image for gtstef/filebrowser:1.3.3-stable +docker.io/gtstef/filebrowser:1.3.3-stable +1.1.0: Pulling from admin/felhom-samba +897d797d2723: Pulling fs layer +3051591aa250: Pulling fs layer +ce57a3f93416: Pulling fs layer +fb94eeec2fe1: Pulling fs layer +fb94eeec2fe1: Waiting +ce57a3f93416: Verifying Checksum +ce57a3f93416: Download complete +fb94eeec2fe1: Verifying Checksum +fb94eeec2fe1: Download complete +897d797d2723: Verifying Checksum +897d797d2723: Download complete +897d797d2723: Pull complete +3051591aa250: Verifying Checksum +3051591aa250: Download complete +3051591aa250: Pull complete +ce57a3f93416: Pull complete +fb94eeec2fe1: Pull complete +Digest: sha256:1c17c09422bec0366d7cf0e0fcfc1486ba6c90334a0a5d5c851073a9342f8f10 +Status: Downloaded newer image for gitea.dooplex.hu/admin/felhom-samba:1.1.0 +gitea.dooplex.hu/admin/felhom-samba:1.1.0 +[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-controller-bootstrap.service' → '/etc/systemd/system/felhom-controller-bootstrap.service'. +[golden] baking the controller-bootstrap PATH unit (starts the service on bootstrap-mount hot-plug — B1) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-controller-bootstrap.path' → '/etc/systemd/system/felhom-controller-bootstrap.path'. +[golden] baking the first-boot SSH host-key regeneration unit (F3) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-regen-hostkeys.service' → '/etc/systemd/system/felhom-regen-hostkeys.service'. +[golden] identity-clean + minimize … +[golden] stop + archive … +INFO: including mount point rootfs ('/') in backup +INFO: including mount point mp0 ('/var/lib/felhom') in backup +INFO: archive file size: 626MB +INFO: Finished Backup of VM 9100 (00:00:41) +[golden] DONE. golden archive volid: local:backup/vzdump-lxc-9100-2026_08_23-11_29_52.tar.zst (rootfs 32G + ONE data volume 24G @ /var/lib/felhom, all in the archive) +[golden] publishing golden (657261745 bytes, sha256 9eaf39ac39219b42…) → https://gitea.dooplex.hu/api/packages/admin/generic/felhom-golden/0.223.0/golden.tar.zst +[golden] pre-delete existing: HTTP 404 (404/204 expected) +[golden] upload OK (HTTP 201) +GOLDEN_VERSION=0.223.0 +GOLDEN_SHA256=9eaf39ac39219b42ec9e6cbf890275febcdcc6f53325fe0c0f591d3431044f17 +[golden] Record in the hub operator UI (Configs → Day-0 artifacts): golden 0.223.0 / 9eaf39ac39219b42ec9e6cbf890275febcdcc6f53325fe0c0f591d3431044f17 +[golden] (the build guest 9100 is stopped; destroy it with: pct destroy 9100 --purge) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index f08bcd69..688de7ca 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,53 @@ +## v0.107.0 — the hub rewrote a severity and said nothing, and the guard for that sat downstream of the rewrite (2026-08-23, R-387) + +**One handler, two fields, opposite discipline.** An unknown **event type** is rejected with a loud +`400` (`allowedEventTypes`). An unknown **severity** was rewritten to `info` without a word — and +`info` is dropped by `severityNotifies` before either delivery leg, so the event was stored, answered +`200`, and mailed to nobody. + +**Two shipped features went out that way.** The controller's `DiskAlertKind.Severity` emitted `"warn"` +until controller v0.215.0; `app_start_failed` emitted it until controller v0.223.0. Both were +undeliverable for the whole life of the feature, on both legs, with every gate green. + +**The mechanism built to catch exactly this was structurally blind to it.** `dispatcher.go` has a line +whose job is to log an unrecognised severity — and for anything arriving over the API it can never +execute, because this handler guarantees the value it looks for has already been overwritten one line +earlier. + +**The coercion STAYS; only the silence is fixed.** A rejected event is a *lost* event, and losing an +alarm is worse than mis-routing one — which is why this was a coercion in the first place. The +producer is our own controller, so the fix belongs at the emitter (controller v0.223.0); this release +is what makes the emitter's mistake **visible the first time it happens instead of never**. The new +line is at `WARN` and names the customer, the event type and the rejected value. + +### The dispatcher's branch is KEPT, and this is the reason + +It was examined as dead code. **It is not dead.** `cmd/hub/main.go` wires `dispatcher.ProcessEvent` +**directly** as the `monitor.EventNotifyFunc` for the staleness, host-staleness and offsite-box +checkers, and those hub-generated events never pass through the ingest handler at all. For every one +of them that line is the **only** severity guard there is. Deleting it as "dead" would have removed +the live half while the dead half supplied the justification. + +Verified while deciding: **all 90 severity literals in `internal/monitor` are already in the +vocabulary**, so the guard is currently silent because the producers are correct — which is what a +working guard looks like. (The `"warn"` strings in `internal/web` are UI badge vocabulary, not +severities.) + +### Tests + +`internal/api/r387_severity_visibility_test.go` — the coercion still happens, the event is still +stored, the event is **not** lost, and the WARN line names all three facts; plus a guard that a valid +severity stays silent, because an alarm on the normal path is one people learn to ignore. +`internal/notify/r329_app_start_failed_test.go` — the routing consequence: with the corrected +severity the **operator is emailed and the customer is not**, unless they opted in, in which case both +legs deliver and the customer's copy carries the Hungarian template rather than raw internal text. + +Test count **702 → 709**. + +**Red-proof (seen failing):** delete the ingest `WARN` line → the coercion test fails with +`the hub rewrote a severity and said nothing`, the log showing only the ordinary `[INFO] Event from +c1: backup_failed (info)`. + ## v0.106.0 — the hub says something when it loses sight of the off-site stores (2026-08-18, R-339) **The gap this closes, measured rather than supposed.** On 2026-08-18 ep0's PBS proxy was wedged for diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index b10266dd..364d90e8 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -2118,10 +2118,32 @@ func (h *Handler) handleEvent(w http.ResponseWriter, r *http.Request) { return } - // Validate/default severity (exact-match lowercase; unknown values coerce to info) + // Validate/default severity (exact-match lowercase; unknown values coerce to info). + // + // R-387 — THE COERCION STAYS. THE SILENCE DOES NOT. + // + // Note the asymmetry two blocks up: an unknown event TYPE is rejected with a loud 400, while an + // unknown SEVERITY was rewritten without a word. The silent one is the one that hid a real defect + // for the whole life of two features — `DiskAlertKind.Severity` emitted "warn" until controller + // v0.215.0, and `app_start_failed` emitted it until v0.223.0. Both were stored, both were + // coerced here to "info", and "info" is dropped by severityNotifies — so both were mailed to + // NOBODY, on either leg, while every POST returned 200. + // + // The dispatcher has a line whose job is exactly this (`unrecognized severity %q`), and it can + // never execute, because this block guarantees the value it looks for cannot reach it. **The + // mechanism built to detect this class was structurally blind to it.** + // + // WHY NOT A 400. A rejected event is a LOST event, and losing an alarm is worse than mis-routing + // one — the same reasoning that made this a coercion in the first place. The producer is our own + // controller, so the fix belongs at the emitter; this line is how the emitter's mistake becomes + // VISIBLE the first time it happens instead of never. switch payload.Severity { case "info", "warning", "error", "critical": default: + h.logger.Printf("[WARN] [api] Event from %s: severity %q is not in {info,warning,error,critical} "+ + "— coercing to \"info\", which severityNotifies DROPS, so this %s alert will reach NOBODY. "+ + "Fix the emitting controller; this event is stored but not routed.", + payload.CustomerID, payload.Severity, payload.EventType) payload.Severity = "info" } diff --git a/hub/internal/api/r387_severity_visibility_test.go b/hub/internal/api/r387_severity_visibility_test.go new file mode 100644 index 00000000..a303e78c --- /dev/null +++ b/hub/internal/api/r387_severity_visibility_test.go @@ -0,0 +1,133 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + "log" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-387 — the ingest handler rewrote an unknown severity WITHOUT SAYING SO, and the guard built to +// catch that class sat downstream of the rewrite, permanently blind to it. +// +// THE ASYMMETRY. Two fields, same handler, opposite discipline: an unknown event TYPE is rejected +// with a loud 400; an unknown SEVERITY was silently coerced to "info" — after which severityNotifies +// drops it and NEITHER leg runs. Two shipped features (`DiskAlertKind.Severity` until controller +// v0.215.0, `app_start_failed` until v0.223.0) emitted "warn" and were mailed to nobody, while every +// POST returned 200 and every dashboard showed the alert. +// +// THE LAYER. The guard sits at INGEST, because that is the last point at which the offending value +// still exists — by design it ceases to exist one line later, which is exactly why nothing downstream +// could ever detect it. +// +// WHY NOT A 400. A rejected event is a LOST event, and losing an alarm is worse than mis-routing one. +// The coercion is deliberate and stays; only the silence is fixed. +// +// RED-PROOF (observed, see REPORT.md): delete the h.logger.Printf from the default branch and +// TestR387_UnknownSeverityIsCoercedAndAnnounced fails with `the hub rewrote a severity and said +// nothing`. + +func severityTestHandler(t *testing.T) (*Handler, *store.Store, *bytes.Buffer) { + t.Helper() + var buf bytes.Buffer + path := filepath.Join(t.TempDir(), "test.db") + st, err := store.New(path, log.New(io.Discard, "", 0)) + if err != nil { + t.Fatalf("store.New: %v", err) + } + t.Cleanup(func() { st.Close() }) + if err := st.SaveCustomerConfig(&store.CustomerConfig{ + CustomerID: "c1", APIKey: "k1", RetrievalPassword: "p", + }); err != nil { + t.Fatal(err) + } + h := New(st, globalKey, "", "", nil, log.New(&buf, "", 0)) + return h, st, &buf +} + +func postEvent(t *testing.T, h *Handler, eventType, severity string) *httptest.ResponseRecorder { + t.Helper() + body, _ := json.Marshal(map[string]any{ + "customer_id": "c1", "event_type": eventType, "severity": severity, + "message": "test message", + }) + return do(h, "POST", "/event", "k1", string(body)) +} + +func TestR387_UnknownSeverityIsCoercedAndAnnounced(t *testing.T) { + h, st, logs := severityTestHandler(t) + + rr := postEvent(t, h, "backup_failed", "warn") + + // The event must NOT be lost — that is the whole reason this is a coercion and not a 400. + if rr.Code >= 400 { + t.Fatalf("HTTP %d — an unknown severity must never LOSE the event; a rejected alarm is worse "+ + "than a mis-routed one", rr.Code) + } + + // The STORED effect: coerced to info, exactly as before. + events, err := st.GetRecentEvents("c1", 10) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 { + t.Fatalf("stored %d events, want 1 — the event was lost", len(events)) + } + if events[0].Severity != "info" { + t.Errorf("stored severity = %q, want \"info\" — the coercion behaviour must not change", events[0].Severity) + } + + // THE FIX: it is no longer silent. + out := logs.String() + if !strings.Contains(out, "WARN") { + t.Fatalf("the hub rewrote a severity and said nothing — this is R-387, and it is how two "+ + "features shipped undeliverable for months. log=%q", out) + } + for _, want := range []string{`"warn"`, "c1", "backup_failed"} { + if !strings.Contains(out, want) { + t.Errorf("the log line does not name %q, so nobody can act on it: %q", want, out) + } + } +} + +// A VALID severity must stay silent — a guard that fires on the normal path is one people learn to +// ignore, which is the same failure one door over. +func TestR387_ValidSeveritiesAreSilent(t *testing.T) { + for _, sev := range []string{"info", "warning", "error", "critical"} { + h, _, logs := severityTestHandler(t) + if rr := postEvent(t, h, "backup_failed", sev); rr.Code >= 400 { + t.Fatalf("severity %q: HTTP %d", sev, rr.Code) + } + if strings.Contains(logs.String(), "not in {info,warning,error,critical}") { + t.Errorf("severity %q produced a coercion warning: %q", sev, logs.String()) + } + } +} + +// The asymmetry this finding is about, pinned so it stays deliberate: an unknown TYPE is still +// rejected loudly, an unknown SEVERITY is still accepted-and-logged. Two different answers, both on +// purpose, and now both visible. +func TestR387_UnknownTypeStillRejectedUnknownSeverityStillAccepted(t *testing.T) { + h, st, _ := severityTestHandler(t) + + if rr := postEvent(t, h, "no_such_event_type", "warning"); rr.Code != 400 { + t.Errorf("unknown event_type: HTTP %d, want 400 — an unlisted type must stay a loud refusal", rr.Code) + } + if rr := postEvent(t, h, "backup_failed", "nonsense"); rr.Code >= 400 { + t.Errorf("unknown severity: HTTP %d, want <400 — it must be accepted and logged, never lost", rr.Code) + } + events, err := st.GetRecentEvents("c1", 10) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 { + t.Errorf("stored %d events, want exactly 1 (the bad-severity one; the bad-type one must not "+ + "be stored)", len(events)) + } +} diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index d2ed6697..8e9fcb79 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -141,6 +141,21 @@ func (d *Dispatcher) ProcessEvent(customerID, eventType, severity, message, deta // warning / error / critical trigger notifications. "info" is an intentional non-notify (status/ // recovery events). Anything else is UNRECOGNIZED — log it (don't silently drop), so a bad severity // surfaces instead of vanishing (the felhom-pve-class lesson: a critical event must never be lost). + // + // R-387 — THIS BRANCH IS **KEPT DELIBERATELY**, and here is why, because the question was asked + // and a branch that cannot execute without a note is the thing to avoid. + // + // For an event arriving over the API it is genuinely unreachable: the ingest handler coerces any + // unknown severity to "info" before this is called, so the one value it looks for cannot arrive. + // **But the API is not the only producer.** `cmd/hub/main.go` wires `dispatcher.ProcessEvent` + // DIRECTLY as the `monitor.EventNotifyFunc` for the staleness, host-staleness and offsite-box + // checkers, and those hub-generated events never pass through the handler at all. For every one + // of them this line is the ONLY severity guard there is. + // + // Removing it as "dead" would therefore have deleted the live half while leaving the dead half + // looking like the reason. Verified 2026-08-23: every severity literal in `internal/monitor` (90 + // of them) is already in the vocabulary — so the guard is currently silent because the producers + // are correct, which is exactly what a guard looks like when it is working. if !severityNotifies(severity) { if severity != "info" { d.logger.Printf("[WARN] Dispatcher: unrecognized severity %q for %s/%s — not routing", severity, customerID, eventType) diff --git a/hub/internal/notify/r329_app_start_failed_test.go b/hub/internal/notify/r329_app_start_failed_test.go new file mode 100644 index 00000000..1a987423 --- /dev/null +++ b/hub/internal/notify/r329_app_start_failed_test.go @@ -0,0 +1,166 @@ +package notify + +import ( + "bytes" + "log" + "strings" + "sync" + "testing" +) + +// R-329 — `app_start_failed` was emitted with severity "warn", which the hub coerces to "info" and +// then drops. These tests pin the ROUTING the fixed severity produces, from the dispatcher's side. +// +// THE LAYER. The emitter's word is pinned in the controller (AST walk). This pins the CONSEQUENCE at +// the hub: with the correct severity the OPERATOR is emailed and the CUSTOMER is not, unless the +// customer opted in. Asserting only the controller's string would be case #9's mistake — mechanism +// pinned, consequence unpinned — and this is the half a customer actually experiences. +// +// RED-PROOF (observed, see REPORT.md): pass "warn" instead of "warning" in Scenario A and it fails +// with `operator was NOT emailed` — the exact live defect, reproduced in a unit test. + +// SCENARIO A — an app goes down and the customer has NOT opted in. +// The operator must be emailed; the customer must not. +func TestR329_ScenarioA_OperatorMailedCustomerNot(t *testing.T) { + st := opOnlyStore(t) + // A customer with an address and the DEFAULT set — app_start_failed deliberately absent. + if err := st.SaveNotificationPrefs("c1", "customer@example.com", + []string{"backup_failed", "disk_warning"}, 6); err != nil { + t.Fatal(err) + } + + rec := &sentTo{} + d := opOnlyDispatcher(t, st, rec) + d.ProcessEvent("c1", "app_start_failed", "warning", + "Telepített alkalmazás nem fut: BookStack", `{"stack_name":"bookstack"}`, "controller") + + var gotOperator, gotCustomer bool + for _, to := range rec.to { + switch to { + case "operator@felhom.eu": + gotOperator = true + case "customer@example.com": + gotCustomer = true + } + } + if !gotOperator { + t.Errorf("operator was NOT emailed for app_start_failed (severity \"warning\") — this is the "+ + "R-329 defect: a severity outside {info,warning,error,critical} is coerced to \"info\" "+ + "and dropped by severityNotifies, reaching nobody. sent=%v", rec.to) + } + if gotCustomer { + t.Errorf("the CUSTOMER was emailed although app_start_failed is not in their enabled events "+ + "— the operator ruled this OFF by default. sent=%v", rec.to) + } +} + +// SCENARIO B — the customer HAS opted in. Both legs deliver, and the customer's copy must carry the +// hub's HUNGARIAN template, not raw English. That is the v0.78.0 defect the registers exist to stop. +// +// This is also the POSITIVE CONTROL for Scenario A's absence claim: it proves the customer leg can +// deliver at all for this event type, so "the customer was not emailed" in A means the gate held, +// not that the path is broken. +func TestR329_ScenarioB_OptedInCustomerGetsTheHungarianMessage(t *testing.T) { + st := opOnlyStore(t) + if err := st.SaveNotificationPrefs("c1", "customer@example.com", + []string{"backup_failed", "app_start_failed"}, 6); err != nil { + t.Fatal(err) + } + + type mail struct{ to, subject, body string } + var sent []mail + d := NewDispatcher(st, "test-key", "from@felhom.eu", "operator@felhom.eu", true, discardLogger()) + d.sendEmailFn = func(to, subject, body string, _ map[string]string) error { + sent = append(sent, mail{to, subject, body}) + return nil + } + d.ProcessEvent("c1", "app_start_failed", "warning", + "Telepített alkalmazás nem fut: BookStack", `{"stack_name":"bookstack"}`, "controller") + + var customer *mail + var gotOperator bool + for i := range sent { + if sent[i].to == "customer@example.com" { + customer = &sent[i] + } + if sent[i].to == "operator@felhom.eu" { + gotOperator = true + } + } + if !gotOperator { + t.Errorf("operator not emailed when the customer opted in — both legs must deliver. sent=%v", sent) + } + if customer == nil { + t.Fatalf("the customer opted in and was NOT emailed — the toggle would be a lie. sent=%v", sent) + } + // The Hungarian template, not the raw English/controller string. + want := customerMessages["app_start_failed"] + if want == "" { + t.Fatal("app_start_failed has no customerMessages entry — a customer-switchable event with " + + "no Hungarian copy sends a household raw internal text (the v0.78.0 defect)") + } + if !strings.Contains(customer.body+customer.subject, want) { + t.Errorf("the customer's mail does not carry the Hungarian template %q.\n subject: %q\n body: %q", + want, customer.subject, customer.body) + } +} + +// The two registers must stay as they are: app_start_failed is customer-reachable BY CONFIGURATION, +// which is precisely what makes the toggle honest. +func TestR329_AppStartFailedIsNotOperatorOnly(t *testing.T) { + if operatorOnlyEvents["app_start_failed"] { + t.Fatal("app_start_failed is in operatorOnlyEvents — the customer toggle added in controller " + + "v0.223.0 would then be visible, switchable and STRUCTURALLY INCAPABLE of delivering. " + + "A toggle that cannot do what it says is worse than no toggle.") + } + if customerMessages["app_start_failed"] == "" { + t.Fatal("app_start_failed has no customerMessages entry — see above") + } +} + +// SCENARIO H's dispatcher half: an unrecognized severity must be LOGGED, not silently dropped. This +// branch is reachable from the hub's own monitor checkers, which call ProcessEvent directly and never +// pass the API handler's coercion — which is why it was KEPT rather than deleted as dead. +func TestR329_UnrecognizedSeverityIsLoggedNotSilent(t *testing.T) { + st := opOnlyStore(t) + if err := st.SaveNotificationPrefs("c1", "customer@example.com", []string{"backup_failed"}, 6); err != nil { + t.Fatal(err) + } + buf := &lineBuf{} + d := NewDispatcher(st, "test-key", "from@felhom.eu", "operator@felhom.eu", true, buf.logger()) + d.sendEmailFn = func(to, _, _ string, _ map[string]string) error { + t.Errorf("an unrecognized severity was ROUTED to %s — it must not be", to) + return nil + } + // The hub-internal path: straight into ProcessEvent, bypassing the handler. + d.ProcessEvent("c1", "backup_failed", "warn", "msg", "{}", "hub") + + out := buf.String() + if !strings.Contains(out, "unrecognized severity") { + t.Errorf("a bad severity from a hub-internal producer vanished without a word: %q", out) + } + if !strings.Contains(out, `"warn"`) { + t.Errorf("the log line does not name the offending value, so nobody can fix it: %q", out) + } +} + +// lineBuf is a tiny concurrency-safe log sink — the dispatcher logs from the calling goroutine here, +// but ProcessEvent is documented as goroutine-safe and the hub calls it with `go`. +type lineBuf struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lineBuf) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lineBuf) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func (b *lineBuf) logger() *log.Logger { return log.New(b, "", 0) } diff --git a/manifests/hub.yaml b/manifests/hub.yaml index 4d1efb86..a6ebac0f 100644 --- a/manifests/hub.yaml +++ b/manifests/hub.yaml @@ -125,7 +125,7 @@ spec: spec: containers: - name: hub - image: gitea.dooplex.hu/admin/felhom-hub:0.106.0 + image: gitea.dooplex.hu/admin/felhom-hub:0.107.0 ports: - containerPort: 8080 name: http