diff --git a/documentation/architecture/01-topology-and-trust.md b/documentation/architecture/01-topology-and-trust.md index 1ff0b725..4dc376cb 100644 --- a/documentation/architecture/01-topology-and-trust.md +++ b/documentation/architecture/01-topology-and-trust.md @@ -169,7 +169,9 @@ traefik's :443 and says so in `X-Forwarded-Proto: http` (measured), which traefi The first link of the entrypoint chain `felhom-forwarded` (`felhom-https-only`) answers it with a 301 to the same https URL, path and query kept, BEFORE any app sees the request — so a login page is never served, and a password never sent, over http. A name with no app route is redirected by a priority-1 catch-all that matches only `X-Forwarded-Proto: http`. -HSTS: one day, no subdomains, no preload (a longer value is the operator's decision). **Where it is enforced:** on the box +HSTS: one day, no subdomains, no preload on every released controller. **[FACT — built, not released, 2026-11-14: +controller `main` sends one year (`31536000`), `09` §3 decision 249, R-1050; still no subdomains, no preload; the +release is built on or after 2026-11-15.]** **Where it is enforced:** on the box (`dynamic/forwarded.yml`, rewritten on every health tick if it drifts), not at Cloudflare: the boxes' Cloudflare keys cannot change zone settings (403, measured on every zone) — Cloudflare's „Always Use HTTPS" is the operator's optional second layer. **How it is checked:** `scripts/https_only_check.py` lists every public name from the box's own traefik diff --git a/documentation/architecture/05-hub-architecture.md b/documentation/architecture/05-hub-architecture.md index 7a6f549f..0cc00818 100644 --- a/documentation/architecture/05-hub-architecture.md +++ b/documentation/architecture/05-hub-architecture.md @@ -659,6 +659,10 @@ Four cases, the four the ruling names. Each leaves an event row and mails the op | A signed job a box took and refused, or whose run failed | the box's agent | the agent's clearing call (`DELETE …/jobs/`) now carries `outcome` (done · rejected · failed · malformed), a `reason` from a closed list (`signedjobs.ReasonCodes`) and the op's name in its query; before, „ran” and „refused” looked the same | `signed_job_refused` (error), once per job. The two clock reasons name the box's clock | an agent older than release 3 says nothing, and nothing is raised. The box's own words never reach a mail (unknown reason → „other”) | | The guest-power watchdog gave up | the box's agent | the host report's `guest_power_gave_up` list (vmid, since, attempts); the watchdog's record is memory | `guest_power_gave_up` (error), once per down period; `guest_power_ok` (info) when a later report shows that guest running | „gone from the list” alone never clears (an agent restart empties the list while the guest is still down) | +**What a box cannot do with these two channels:** a refused job is told only for a job that was in that box's own +queue, so a box cannot make operator mails by clearing job ids it invents; the give-up list is read only for guests +the same report lists (not running), 16 entries at most. + **Not proven on a box or a deployed hub:** all five. The tests use fakes for the mail service and the real gate and verifier for the signed jobs (`audits/night-build-2026-11-14/B/`, 43 mutations seen red). diff --git a/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-told.txt b/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-told.txt index f315e209..289d55ce 100644 --- a/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-told.txt +++ b/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-told.txt @@ -38,3 +38,17 @@ r1383_box_told_test.go:130: after the guest runs again: gave_up=1 ok=0, want one recovery 9 mutations, 0 not a red-proof +== MUTATION: a job that was never queued is told (a box mints operator events) + RED on the mutant, GREEN on the real file + --- FAIL: TestSignedJobOutcome_RefusedIsToldDoneIsNot (0.05s) + r1383_box_told_test.go:52: events=2, want ONE for the refused job +== MUTATION: a vmid the report does not list is read + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestPowerGaveUp_ToldOnceClearedOnlyByARunningGuest (0.07s) + r1383_box_told_test.go:153: 500 invented guests and a list past the cap: gave_up=18 ok=1, want no new alarm +== MUTATION: the give-up list has no cap + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestPowerGaveUp_ToldOnceClearedOnlyByARunningGuest (0.06s) + r1383_box_told_test.go:153: 500 invented guests and a list past the cap: gave_up=3 ok=1, want no new alarm + +3 mutations, 0 not a red-proof diff --git a/documentation/audits/night-build-2026-11-14/C/redproof-agent-boot-key.txt b/documentation/audits/night-build-2026-11-14/C/redproof-agent-boot-key.txt new file mode 100644 index 00000000..e471322f --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/C/redproof-agent-boot-key.txt @@ -0,0 +1,31 @@ +== MUTATION: the key is built from the boot time (the defect) + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootKey_AClockStepDoesNotMoveIt (0.00s) + guest_boot_key_test.go:49: tokens = "1794112518-4242" / "1794112518-4242" +== MUTATION: the key ignores the guest's init start (a guest reboot is not seen) + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootKey_AClockStepDoesNotMoveIt (0.00s) + guest_boot_key_test.go:49: tokens = "1794112518-4242" / "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-0" +== MUTATION: an unreadable boot_id still gives a key + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootKey_AClockStepDoesNotMoveIt (0.00s) + guest_boot_key_test.go:73: boot_id "" gave the key "-991234", want "" (the controller then reads the old token) + guest_boot_key_test.go:73: boot_id "not-a-boot-id" gave the key "not-a-boot-id-991234", want "" (the controller then reads the old token) + guest_boot_key_test.go:73: boot_id "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11\nx" gave the key "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11\nx-991234", want "" (the controller then reads the old token) +== MUTATION: a guest that does not run gives a key + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootKey_AClockStepDoesNotMoveIt (0.00s) + guest_boot_key_test.go:81: a guest that does not run gave the key "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-" + +4 mutations, 0 not a red-proof +(the first two /disks mutants did not compile and proved nothing; done again below) +== MUTATION: /disks does not carry the key under its name + RED on the mutant, GREEN on the real file + --- FAIL: TestDisks_CarriesBothBootTokens (0.00s) + guest_boot_key_test.go:103: /disks boot tokens: id="boot-1" key="", want boot-1 and key-1 ({"ok":true,"data":{"disks":[],"guest_boot_id":"boot-1","guest_boot_k":"key-1","vmid":8200}} +== MUTATION: /disks swaps the two tokens (older controllers lose their signal) + RED on the mutant, GREEN on the real file + --- FAIL: TestDisks_CarriesBothBootTokens (0.00s) + guest_boot_key_test.go:103: /disks boot tokens: id="key-1" key="boot-1", want boot-1 and key-1 ({"ok":true,"data":{"disks":[],"guest_boot_id":"key-1","guest_boot_key":"boot-1","vmid":8200}} + +2 mutations, 0 not a red-proof diff --git a/documentation/audits/night-build-2026-11-14/C/redproof-controller-boot-key.txt b/documentation/audits/night-build-2026-11-14/C/redproof-controller-boot-key.txt new file mode 100644 index 00000000..be61b92f --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/C/redproof-controller-boot-key.txt @@ -0,0 +1,39 @@ +== MUTATION: the key is ignored — the old rule decides (a clock step is a new boot again) + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_AClockStepIsNotANewBoot (0.00s) + guest_boot_verdict_test.go:48: THE CASE: the clock stepped, the key did not move: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794112550-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (true, "1794112550-4 + guest_boot_verdict_test.go:48: same boot, nothing moved: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794112518-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (true, "1794112518-4242"), want (false, "") + guest_boot_verdict_test.go:48: a host reboot: the key changes: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794200000-3907", "c0ffee00-1111-2222-3333-444455556666-3907") = (true, "1794200000-3907"), want (true, +== MUTATION: the old token still decides when a key is there + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_AClockStepIsNotANewBoot (0.00s) + guest_boot_verdict_test.go:48: THE CASE: the clock stepped, the key did not move: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794112550-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (true, "5b1f6c1e-8f0 + guest_boot_verdict_test.go:48: same boot, nothing moved: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794112518-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (true, "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11- +== MUTATION: the move to the key recreates every app once + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_AClockStepIsNotANewBoot (0.00s) + guest_boot_verdict_test.go:48: the first report with a key, same boot by the old token: the record moves, no recreate: guestBootVerdict("1794112518-4242", "1794112518-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (true, "5 +== MUTATION: the move keeps the old record (the key is never taken) + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_AClockStepIsNotANewBoot (0.00s) + guest_boot_verdict_test.go:48: the first report with a key, same boot by the old token: the record moves, no recreate: guestBootVerdict("1794112518-4242", "1794112518-4242", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242") = (false, " +== MUTATION: a real reboot is not seen once the key is in use + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_AClockStepIsNotANewBoot (0.00s) + guest_boot_verdict_test.go:48: a host reboot: the key changes: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794200000-3907", "c0ffee00-1111-2222-3333-444455556666-3907") = (false, "c0ffee00-1111-2222-3333-444455 + guest_boot_verdict_test.go:48: a guest reboot: the key's second half changes: guestBootVerdict("5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242", "1794112518-991234", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-991234") = (false, "5b1f6c1e-8f + guest_boot_verdict_test.go:48: the first report with a key, and the old token differs: a new boot: guestBootVerdict("1794112518-4242", "1794200000-3907", "c0ffee00-1111-2222-3333-444455556666-3907") = (false, "c0ffee00-1111-2222-3 +== MUTATION: the boot pass compares the old token itself (the verdict is not wired) + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_IsWhatTheBootPassReads (0.00s) + guest_boot_verdict_test.go:93: processGuestBootChange: guestBootVerdict(…, resp.GuestBootKey) calls=0 (want 1), raw uses of the old token=2 (want 0), persists of the verdict's token=3 (want 3) +== MUTATION: the boot pass gives the verdict no key + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_IsWhatTheBootPassReads (0.00s) + guest_boot_verdict_test.go:93: processGuestBootChange: guestBootVerdict(…, resp.GuestBootKey) calls=0 (want 1), raw uses of the old token=0 (want 0), persists of the verdict's token=3 (want 3) +== MUTATION: the record moved to the key is not saved + RED on the mutant, GREEN on the real file + --- FAIL: TestGuestBootVerdict_IsWhatTheBootPassReads (0.00s) + guest_boot_verdict_test.go:93: processGuestBootChange: guestBootVerdict(…, resp.GuestBootKey) calls=1 (want 1), raw uses of the old token=0 (want 0), persists of the verdict's token=2 (want 3) + +8 mutations, 0 not a red-proof diff --git a/documentation/audits/night-build-2026-11-14/C/redproof-controller-hsts.txt b/documentation/audits/night-build-2026-11-14/C/redproof-controller-hsts.txt new file mode 100644 index 00000000..f480862a --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/C/redproof-controller-hsts.txt @@ -0,0 +1,18 @@ +== MUTATION: HSTS is back at one day + RED on the mutant, GREEN on the real file + --- FAIL: TestRenderForwardedHeaders_HTTPSOnly (0.00s) + tunnel_test.go:169: HSTS must be one year (31536000) — decision 249 — got 86400 (constant 86400) +== MUTATION: HSTS covers subdomains + RED on the mutant, GREEN on the real file + --- FAIL: TestRenderForwardedHeaders_HTTPSOnly (0.00s) + tunnel_test.go:172: HSTS must not include subdomains nor preload, got true / false +== MUTATION: HSTS asks for preload + RED on the mutant, GREEN on the real file + --- FAIL: TestRenderForwardedHeaders_HTTPSOnly (0.00s) + tunnel_test.go:172: HSTS must not include subdomains nor preload, got false / true +== MUTATION: the rendered file carries another number than the constant + RED on the mutant, GREEN on the real file + --- FAIL: TestRenderForwardedHeaders_HTTPSOnly (0.00s) + tunnel_test.go:169: HSTS must be one year (31536000) — decision 249 — got 86400 (constant 31536000) + +4 mutations, 0 not a red-proof diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index 11eea8e6..f6c204ae 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -268,7 +268,7 @@ stopping line that lies. | **R-779** | Security & access | P4 | **[P3-LOW] Part A's "two outside addresses seen as two" is proven through the simulated tunnel only; on the REAL tunnel the second outside address (ep0, one request allowed) was refused by Cloudflare's edge with 403 and never reached the box.** Measured 2026-10-01 19:51 UTC (`audits/visitors-2026-10-01/A/L2-demo-hp-real-tunnel.txt`): no log line on demo-hp; demo-hp's box has no geo restriction in its settings, so a Cloudflare ZONE rule (country or bot, not read) refused a German datacenter address. DooPlex's own address on the real tunnel was seen as itself. **Needs:** one sign-in from a second Hungarian address (the operator's phone off wifi) while DooPlex is locked out — 2 minutes; and say which Cloudflare rule refused ep0. | **WAITING-ON-OPERATOR — rank P3-LOW; owner: operator (a phone), CC reads the logs** **Re-ranked 2026-10-03: P3→P4: a proof gap on the real tunnel; operator-only follow-up.** | — | — | CC + operator | | **R-904** | Security & access | P4 | **Cloudflare can read every household's app traffic; replacing it with our own relay is a later item.** Facts (reviewer discussion 2026-10-08; `01`): app traffic and the dashboard reach the box through the Cloudflare Tunnel (`01` §5 trust table, rows end-user ↔ apps and customer ↔ controller UI; §7), and the tunnel's public end is Cloudflare's edge, where TLS ends — so Cloudflare can technically read that traffic (the FAQ says so since 2026-10-08, R-900; „TLS ends at the edge" is not written in `01` — add it there). What Cloudflare gives today, free: inbound reach with no router setup, the CGNAT answer (`01` §4, §7); certificates (`01` §7, the free tier covers one level below a zone); the geo-WAF the hub enforces (`01` §5 last row, §7); flood protection (not in `01`). The alternative named: our own EU relay over WireGuard with TLS passthrough by SNI, certificates on the box, the geo-block on the relay. Its costs: one more machine the operator keeps up, and a single point of reach for every box; weaker flood protection; about a week of work after a spike. Operator ruling 2026-10-08 09:07 (`09` §3 decision 184): a later item. | **DEFERRED** — **2026-10-10: the asked sentence is in `01` §7** („TLS ends at Cloudflare's edge …”); the spike stays deferred. **DEFERRED — after the first customers (operator ruling 2026-10-08 09:07)** | — | A spike after the first customers (the relay's reach, cost and flood behaviour, measured) | operator | | **R-1040** | Security & access | P4 | **Until the household makes Sonarr's or Radarr's first admin, any container on the box can read the app's API key.** Measured on the bench 2026-10-14 with the R-1030 seed in place: `AuthenticationRequired=Enabled`, `AuthenticationMethod=None` — `/initialize.json` from another container answers 200 with `apiKey`. Outsiders are blocked by the box's setup gate in that window; a compromised other app is not. | **OPEN** — filed 2026-10-14 | — | Seed the API key at install (`SONARR__AUTH__APIKEY`, measured to work) so the box knows it, and close the window by telling the household in the first steps; or seed a Forms user | CC | -| **R-1050** | Security & access | P4 | **Raise HSTS to one year on or after 2026-11-15, if no https trouble was reported.** `09` §3 decision 249 (D2 = A, operator 2026-10-15): HSTS stays `max-age=86400` now; after a month with no https trouble it becomes one year (`31536000`); never preload, never `includeSubDomains`. The value is `HSTSMaxAgeSeconds` in `felhom-controller/controller/internal/infra/infra.go`, pinned by `TestRenderForwardedHeaders_HTTPSOnly`. „Trouble” = any report of a name that will not open, or a box answering http after the floor. | WATCHING — **2026-11-14: NOT in controller 0.352.1.** That release was built on 2026-11-14, one day before the ruled date, and the operator's words „one day early is OK” were not given. Check date moved to 2026-11-21 (one week: the next day a controller release is likely; a one-day move would only turn the nightly CI red with nothing to release). The search for https trouble since 2026-10-15 is NOT done yet. Earlier: **WATCHING** — filed 2026-10-15 | the date | On or after 2026-11-15: read the register and the operator's mail for https trouble since 2026-10-15; none → a controller release with one year (test updated), bake → vouch → floor | CC | +| **R-1050** | Security & access | P4 | **Raise HSTS to one year on or after 2026-11-15, if no https trouble was reported.** `09` §3 decision 249 (D2 = A, operator 2026-10-15): HSTS stays `max-age=86400` now; after a month with no https trouble it becomes one year (`31536000`); never preload, never `includeSubDomains`. The value is `HSTSMaxAgeSeconds` in `felhom-controller/controller/internal/infra/infra.go`, pinned by `TestRenderForwardedHeaders_HTTPSOnly`. „Trouble” = any report of a name that will not open, or a box answering http after the floor. | NARROWED — one year is in controller `main`, not released. **2026-11-14 (night build):** searched for https trouble since 2026-10-15: the open and closed registers (ASCII fragments; controls: a common word found, a nonsense word not found), the operator's mailbox (words for „does not open”, „not secure”, certificate, HSTS, the browser's error codes: one thread, a tester's install questions, no https trouble), the hub's log since its start today (0 lines). No name that does not open, no box answering http after the floor. Not searched: the hub's log before today's deploy (not kept), Tester 2 (off, R-1104). The release must be built on or after 2026-11-15. WATCHING — **2026-11-14: NOT in controller 0.352.1.** That release was built on 2026-11-14, one day before the ruled date, and the operator's words „one day early is OK” were not given. Check date moved to 2026-11-21 (one week: the next day a controller release is likely; a one-day move would only turn the nightly CI red with nothing to release). The search for https trouble since 2026-10-15 is NOT done yet. Earlier: **WATCHING** — filed 2026-10-15 | the date | On or after 2026-11-15: read the register and the operator's mail for https trouble since 2026-10-15; none → a controller release with one year (test updated), bake → vouch → floor | CC | | **R-1054** | Security & access | P3 | **The hub's operator/global API key is still one bearer that can do everything on `/api/v1`, including reading every box's break-glass root password (`GET /admin/hosts/{id}/recovery-credential`).** Hub v0.152.0 gave the web surface scoped tokens (`05` §17) but tokens are refused on `/api/v1` by design. Who uses the global key today is not listed in one place. Fix: list its consumers; move each operator use to a scoped token on the web side (or a token class on the API), and keep the global key only where a box path needs it — or retire it. | OPEN — filed 2026-10-15 by the token build | — | List the global key's consumers (pod env, scripts, runbooks) | CC | | **R-1194** | Security & access | P3 | **On a fresh box the operator's break-glass SSH door was broken from the first minute: `oob_degraded` „felhom-sshd config invalid (sshd -t fails)".** Rehearsal 2026-10-20, Tester4-816e21, mail at 12:03 UTC (one minute after the host record was created): details `active:false, config_invalid:true, felhom_sshd_port:0, operator_key_configured:false, operator_key_reported:true, reachable:false`. Possibly the shape of a customer with no operator key set (Tester4 is a test customer) — then the alarm is a false one on every such box; or a real config defect. Not diagnosed (no shell). Checked the same day: the only `oob_degraded` in the operator mailbox in 30 days, and no recovery mail followed; the report says port 0. Two candidate mechanisms in the agent (`internal/felhomsshd/health.go:63-66`): `sshd -t -f ` run by the NON-root agent fails on unreadable host keys, or a config rendered before the desired state arrived (port 0). **CAUSE FOUND AND FIXED 2026-10-21 night** (`audits/night-2026-10-21/P5-checks/`): reproduced on a fresh box from the public ISO — `oob_degraded` mailed 20:34 CEST while felhom-sshd was applied 0.4 s after the check and is healthy (port 8822 listening, `sshd -t` passes as the agent's user through its sudo grant — so the non-root theory is FALSE; without sudo it fails on the host key, which the agent never does). The first host report runs before the felhom-sshd loop's first pass and reads a config never rendered. Fix: agent `0da792e` — no OOB stanza until the first pass has run (the hub does not evaluate a missing one; a failed pass counts). `TestR1194_NoOOBStanzaBeforeTheFirstPass` red-proved. | WATCHING — released in agent v0.162.0 (2026-10-22, operator yes; on the 3 test boxes); the false alarm comes only on a FRESH box's first report | the next fresh install with agent ≥ 0.162.0 | At the next fresh install read the hub's events for `oob_degraded` in the first hour (none expected); then close | CC | | **R-1237** | Security & access | P3 | **The nightly CC-secrets sweep found stored one-time codes in a session record dated 2026-10-23: 14 names in one file (bind tokens and setup codes of test boxes — NIGHT_2026_10_23_*, FIXES_2026_10_21_*, NIGHT_2026_10_21_*, REHEARSAL_2026_10_20_* — and R_DEMO_FELHOM in a file of 2026-10-20).** Mail „[DooPlex CC secrets] 2 secret name(s) found in old session records", 2026-10-23 01:57 UTC, read in the day session (names only). One file holding all of them suggests one command printed many stored values at once (a listing or a dump) in the night session. All but R_DEMO_FELHOM are one-time install codes of test boxes; NIGHT_2026_10_23_* die when Tester4-16fdef is deleted (the operator's do list). | **OPEN** — 2026-10-27 (operator yes): the dead names were REMOVED from the store with `sec rm` (46, one commit each; `audits/r1299-window-button-2026-10-27/sec-rm.txt`, names only): every one-time code this row names is gone; R_DEMO_FELHOM stays (unsure). Consequence: the nightly sweep no longer watches for the removed values — they open nothing (their boxes are destroyed). The session transcripts themselves are unchanged — that question stays here. **Earlier:** OPEN | — | CC: find the file and the command by the sweep's own report (names only), add a hook shape for that command if it is not one; R_DEMO_FELHOM (demo-felhom's recovery code) — the operator decides whether to rotate it | CC | @@ -299,7 +299,7 @@ stopping line that lies. | **R-1013** | Box system & updates | P4 | **After a power cut the agent's start-up stale-lock pass logs an ERROR „pct unlock failed … no lock found" when Proxmox's own start-all removed the lock in the same second.** The outcome is the goal (no lock), so the line is a false alarm in the log. Seen on demo-hp 2026-10-14 12:07:44–45 UTC (`audits/power-cut-fixes-2026-10-14/E/after-boot.txt`; Proxmox: „removed left over backup lock from '9201'!"). Small, but the agent's release for this session is spent (v0.156.1). | **OPEN** — filed 2026-10-14 | — | CC: treat „no lock found" from `pct unlock` as already clear (INFO), with a test; ride the next agent release | CC | | **R-1061** | Box system & updates | P4 | **A per-customer floor reached Tester 1 only at its next 15-minute report, not in seconds.** Set 2026-10-15 12:55:51 UTC for tester-1, demo-hp and demo-felhom together (hub 0.153.0, CC's token): demo-hp and demo-felhom ran 0.314.0 within 13 s (the hub's wake); Tester 1 moved at 13:09:35, its regular report (`audits/gate-restore-2026-10-15/release/03-boxes-move.txt`). It had restarted at 12:54:29 onto rc.1 and logged „hub wait channel active". Why the wake did not reach it is not read (the wake line is DEBUG). No harm: the floor arrived. | **OPEN** — filed 2026-10-15 | — | Read Tester 1's DEBUG ring at the next floor change; compare with the hub's bump log | CC | | **R-1285** | Box system & updates | P4 | **D43's removal of our SSH drop-in from a BYO host happens one bundle LATE: the bundle run that delivers agent 0.165.0 is planned by the box's OLD (0.164.0) wrapper, which writes the drop-in as before („same"), and only then replaces itself.** Measured 2026-10-25 on the BYO test host (VM 9371 on demo-hp, `Tester4-f967d4`; attended release run): 0.164.0 day-0 → signed `agent_update` + `agent_config_update` 0.165.0 → the planted drop-in still there; a second `agent_config_update` (same bundle) → `BUNDLE REMOVED … (ours, on a BYO host — D43)`, nothing written, `sshd -T` the owner's own `permitrootlogin yes` (`audits/release-2026-10-26/byo/byo-proof.txt`). No box is affected today (the fleet's only BYO host is this test host); a fresh BYO install on ≥ 0.165.0 runs the new wrapper from the start. | **VERIFY** — 2026-10-27: agent v0.167.0 RELEASED; in the NORMAL signed delivery the pass now runs right after the commit on all three boxes („running the bundle policy pass after the self-update commit (R-1285)” → `BUNDLE POLICY DONE install_mode=appliance removed=0`; `audits/release-2026-10-27/agent-0167-tester1-demofelhom.txt`). Not live: a BYO box coming FROM 0.164.0 (none in the fleet). **Earlier:** **VERIFY** — 2026-10-26: agent `b1bc63d` (unreleased) — the new agent runs the bundle-policy pass once right after its self-update commits (`selfupdate.SetOnCommitted`); test in the real order (commit, then one pass), red-proved. **Not live:** only a box coming FROM 0.164.0 discriminates (0.165.0's wrapper already applies D43), which needs a signed downgrade chain on a BYO box — the live proof rides the next agent release. **Earlier:** **OPEN** — 2026-10-26 (agent 0.166.0 released, attended run): the policy pass is wired, but in the normal delivery it does NOT run: the box gets `agent_update` and `agent_config_update` in ONE pass, and the bundle job runs in the OLD process (demo-hp: pid 1966780, agent 0.165.0, bundle 09:03:07–09:03:09 CET; the restart into 0.166.0 at 09:03:10) — no `bundle policy pass` line on any of the three boxes. So a BYO host coming from 0.164.0 still loses our drop-in one release late. The night proof (`audits/night-2026-10-25/rig/r1285-live.txt`) ran a bundle under an agent that was ALREADY 0.166.0-code, so it did not cover this order. No BYO host is in the fleet today. Fix options: the agent runs the pass once at start-up after `update committed`; or the release procedure sends the bundle job only after the commit. **Earlier state:** **VERIFY** — fixed on main 2026-10-25 night (agent `abcc1e6`), waits for the agent release + the vouch: the agent runs the wrapper's `bundle-policy` pass after every install (red-proved, `redproof-r1285.txt`). **Live on the rig in ONE signed run** (BYO host, staged as upgraded from 0.164.0: the new agent by hand, the old 0.164.0 wrapper on disk, our drop-in planted; one `agent_config_update` of a test bundle): the old wrapper installed the bundle and replaced itself, then „BUNDLE POLICY REMOVED … (ours, on a BYO host)", `sshd -T` the owner's own `permitrootlogin yes` (`rig/r1285-live.txt`); the test package deleted from the registry after. **Earlier state:** OPEN | — | either the wrapper plans again after it replaced itself (one agent change + a test that a 0.164.0→0.165.0 bundle on a BYO record removes our copy in ONE run, red first), or the release procedure sends a BYO box's config bundle twice; until then a BYO box upgraded from ≤ 0.164.0 keeps our drop-in until its next bundle | CC | -| **R-1395** | Box system & updates | P4 | **A step of the host's clock makes the controller read a new boot: it then recreates every drive-backed app, once.** Found 2026-11-14 on the N100 (`audits/release-2a-2026-11-14/D/13b-n100-container-near-the-step.txt`): the time service came back and stepped the clock by +32.7 s; 9 seconds later the controller logged „[gate] boot 1794112550-…: recreating drive-backed app” for radarr, sonarr and qbittorrent (each down for 1 to 5 s; all healthy after). Cause, read on the box: the controller's boot identity carries `/proc/stat` `btime`, and the kernel computes `btime` from the wall clock, so it moved by the same 32 s (the crash guard's `last_boot_at` reads 04:35:18Z, `btime` now 04:35:50Z). The crash guard was NOT fooled: it reads the kernel's `boot_id`. A step this large needs a clock that was wrong for weeks, so a healthy box does not meet it; a fresh box with a wrong hardware clock can, at its first time sync. | OPEN — filed 2026-11-14 | — | make the controller's boot identity the kernel's `boot_id` (as the crash guard's is), with a test that a moved `btime` is not a new boot | CC | +| **R-1395** | Box system & updates | P4 | **A step of the host's clock makes the controller read a new boot: it then recreates every drive-backed app, once.** Found 2026-11-14 on the N100 (`audits/release-2a-2026-11-14/D/13b-n100-container-near-the-step.txt`): the time service came back and stepped the clock by +32.7 s; 9 seconds later the controller logged „[gate] boot 1794112550-…: recreating drive-backed app” for radarr, sonarr and qbittorrent (each down for 1 to 5 s; all healthy after). Cause, read on the box: the controller's boot identity carries `/proc/stat` `btime`, and the kernel computes `btime` from the wall clock, so it moved by the same 32 s (the crash guard's `last_boot_at` reads 04:35:18Z, `btime` now 04:35:50Z). The crash guard was NOT fooled: it reads the kernel's `boot_id`. A step this large needs a clock that was wrong for weeks, so a healthy box does not meet it; a fresh box with a wrong hardware clock can, at its first time sync. | NARROWED — built in the agent and the controller, not released. **2026-11-14 (night build):** the boot token is the AGENT's (`guest_boot_id` in its `/disks` answer), not the controller's; the controller only compares it. The agent now also sends `guest_boot_key` (the kernel's `boot_id` in place of the boot time) and the controller's boot pass prefers it; the move to the key recreates nothing; an older agent or controller behaves as before. Tests with a moved boot time on both sides (`audits/night-build-2026-11-14/C/`). Needs BOTH releases to take effect. Not proven on a box (a clock step cannot be staged without changing a box). OPEN — filed 2026-11-14 | — | make the controller's boot identity the kernel's `boot_id` (as the crash guard's is), with a test that a moved `btime` is not a new boot | CC | ## Monitoring & notifications — 29 rows (P2 3, P3 18, P4 8) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index f3275d68..9e345543 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -15,6 +15,10 @@ `notify.operatorOnlyEvents`. Tests `TestDailyLedger_*`, `TestPrevDailySlot_*`, `TestR1383_*`, `TestLostOperatorMail_*`, `TestSignedJobWait_*`, `TestSignedJobOutcome_*`, `TestGuestPowerGaveUp_*`; 31 mutations seen red (`documentation/audits/night-build-2026-11-14/B/redproof-hub-{daily-ledger,lost-mail,signed-job,box-told}.txt`). + **Two limits added after the commit's own security review:** a refused job is told only when the job WAS in that + box's queue (`store.DeleteSignedJobN`; a box cannot mint operator events by clearing ids it invents, and a repeated + `DELETE` is silent), and the give-up list is read only for guests the same report lists, at most 16 entries. + 3 more mutations seen red (appended to `redproof-hub-box-told.txt`). - **A box's clock against the hub's (R-1384, D79, `09` §3 decision 300).** The host report's new `sent_at` (the box's clock at the moment the agent sends; agents of release 3) is compared with the hub's clock at the moment the request arrives. More than one minute apart, in either direction → one operator-only `box_clock_wrong` (warning), told once; diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index d17e3a4c..17af3fb5 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -1028,7 +1028,11 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) { runningGuests[g.VMID] = true } } - h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests) + knownGuests := map[int]bool{} + for _, g := range rep.Guests { + knownGuests[g.VMID] = true + } + h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests, knownGuests) h.checkCrash(hostID, custID, body) h.checkDeliberateOff(hostID, custID, &rep) @@ -2078,12 +2082,17 @@ func (h *Handler) handleDeleteJob(w http.ResponseWriter, r *http.Request, pathHo http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden) return } - if err := h.store.DeleteSignedJob(pathHostID, jobID); err != nil { + removed, err := h.store.DeleteSignedJobN(pathHostID, jobID) + if err != nil { h.logger.Printf("[ERROR] delete job %s for %s: %v", jobID, pathHostID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } - h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op")) + if removed > 0 { + // only a job that WAS in this box's queue can be told as refused: the id comes from the box, and a box must + // not mint operator events by clearing ids it invents. It also makes a repeated DELETE silent. + h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op")) + } h.logger.Printf("[INFO] host %s cleared signed-op job %s (executed or rejected)", pathHostID, jobID) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -3264,7 +3273,7 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) { // now the hub could not tell the two apart („executed or rejected”), so an agent update that was refused left no // trace outside the box's journal (clock list CA09-4, CA09-7). An agent of release 3 says how the job ended in the // DELETE's query: outcome = done | rejected | failed | malformed, a reason from a CLOSED list, and the op's name. -// Anything but `done` → one operator-only event. No outcome (an older agent) → nothing. The box's words never reach +// Anything but `done` → one operator-only event, and ONLY for a job that was in that box's queue. No outcome (an older agent) → nothing. The box's words never reach // a mail: an unknown reason is told as „other”, an op name is cut to [a-z0-9_]. // (b) guest_power_gave_up — see checkGuestPowerGaveUp. // Pinned by TestSignedJobOutcome_* and TestGuestPowerGaveUp_*. @@ -3293,17 +3302,24 @@ var signedJobReasons = map[string]string{ var signedJobOpRe = regexp.MustCompile(`^[a-z0-9_]{1,40}$`) +// signedJobIDRe is the alphabet of a job id the hub mints (hex) or the operator's tool names; anything else is not +// quoted in a mail. +var signedJobIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`) + +// guestPowerMaxEntries bounds the give-up list read from ONE report (a box has a handful of guests). +const guestPowerMaxEntries = 16 + func (h *Handler) noteSignedJobOutcome(hostID, jobID, outcome, reason, op string) { switch outcome { case "rejected", "failed", "malformed": default: return // "done", or an agent that does not say } - key := "signed_job_refused:" + hostID + ":" + jobID - if !h.store.OSAlarmRaised(key).IsZero() { - return // the agent repeats a DELETE whose answer it did not get + if !signedJobIDRe.MatchString(jobID) { + // the id was a row of the hub's own queue (the caller checked), but it is put there by the operator's upload + // as free text — an id outside the hub's own alphabet is named, not quoted. + jobID = "(id not shown)" } - _ = h.store.SetOSAlarmRaised(key, time.Now()) text := signedJobReasons[reason] if text == "" { reason, text = "other", "the box gave a reason this hub does not know" @@ -3341,7 +3357,14 @@ type hostGuestPowerGaveUp struct { // true here (the HOST reports). One operator-only `guest_power_gave_up` (error) per guest, told once for one down // period; one `guest_power_ok` when a later report shows that guest RUNNING. The agent's record is memory: after an // agent restart the entry is gone while the guest is still down — so „gone from the list” alone never clears. -func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running map[int]bool) { +// +// The list comes from the box, so it is bounded: only a guest that the SAME report lists (and not as running) is +// read, and at most guestPowerMaxEntries entries — a box cannot mint an alarm per invented vmid. +func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running, known map[int]bool) { + if len(gave) > guestPowerMaxEntries { + h.logger.Printf("[WARN] host %s: guest_power_gave_up lists %d guests — only the first %d are read", hostID, len(gave), guestPowerMaxEntries) + gave = gave[:guestPowerMaxEntries] + } emit := func(typ, sev, msg string, details map[string]any) { dj, _ := json.Marshal(details) if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil { @@ -3352,7 +3375,7 @@ func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestP } key := func(vmid int) string { return fmt.Sprintf("guest_power:%s:%d", hostID, vmid) } for _, g := range gave { - if g.VMID <= 0 || running[g.VMID] || !h.store.OSAlarmRaised(key(g.VMID)).IsZero() { + if g.VMID <= 0 || !known[g.VMID] || running[g.VMID] || !h.store.OSAlarmRaised(key(g.VMID)).IsZero() { continue } _ = h.store.SetOSAlarmRaised(key(g.VMID), time.Now()) diff --git a/hub/internal/api/r1383_box_told_test.go b/hub/internal/api/r1383_box_told_test.go index ef8ba9f0..eeee00ab 100644 --- a/hub/internal/api/r1383_box_told_test.go +++ b/hub/internal/api/r1383_box_told_test.go @@ -59,6 +59,15 @@ func TestSignedJobOutcome_RefusedIsToldDoneIsNot(t *testing.T) { if evs[0].Severity != "error" || !strings.Contains(evs[0].DetailsJSON, `"reason":"expired"`) { t.Errorf("event = %+v", evs[0]) } + // a box „clears” job ids that were never in its queue: no event — a box cannot mint operator mails + for _, id := range []string{"never-queued", "never-queued-2", "x%0Ay"} { + if rr := do(h, http.MethodDelete, "/hosts/h1/jobs/"+id+"?outcome=rejected&reason=expired&op=agent_update", "HKEY", ""); rr.Code != 200 { + t.Fatalf("DELETE of an unknown job: %d", rr.Code) + } + } + if evs := r1383Events(t, st, eventSignedJobRefused); len(evs) != 1 { + t.Fatalf("clearing ids that were never queued raised events: %d, want still 1", len(evs)) + } // a failed run, and words the hub does not know: told, and the box's own text never reaches the mail del("j-fail", "?outcome=failed&reason=execution_failed&op=agent_config_update") del("j-odd", "?outcome=rejected&reason=%3Cscript%3E&op=%3Cb%3EX") @@ -133,4 +142,18 @@ func TestGuestPowerGaveUp_ToldOnceClearedOnlyByARunningGuest(t *testing.T) { if c := counts(); c != "gave_up=2 ok=1" { t.Fatalf("a new down period: %s", c) } + // the list is the box's: a vmid the same report does not list is not read, and a long list is cut + many := "[" + for i := 0; i < 500; i++ { + many += fmt.Sprintf(`{"vmid":%d,"since":"2026-11-14T02:00:00Z","attempts":3},`, 5000+i) + } + many += `{"vmid":101,"since":"2026-11-14T02:00:00Z","attempts":3}]` // 101 IS a guest of the report (stopped), but entry 501 + post("stopped", many) + if c := counts(); c != "gave_up=2 ok=1" { + t.Fatalf("500 invented guests and a list past the cap: %s, want no new alarm", c) + } + post("stopped", `[{"vmid":101,"since":"2026-11-14T02:00:00Z","attempts":3},{"vmid":7777,"since":"x","attempts":3}]`) + if c := counts(); c != "gave_up=3 ok=1" { + t.Fatalf("a real second guest (101) within the cap: %s, want one more alarm and none for vmid 7777", c) + } } diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 5057aa5c..95acd58d 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -3797,10 +3797,22 @@ func (s *Store) CountSignedJobs(hostID string) (int, error) { // calls it after executing OR terminally rejecting a job. Idempotent (deleting an absent job is a // no-op, returns nil) — a retried completion must not error. func (s *Store) DeleteSignedJob(hostID, jobID string) error { - _, err := s.db.Exec(`DELETE FROM signed_jobs WHERE host_id = ? AND job_id = ?`, hostID, jobID) + _, err := s.DeleteSignedJobN(hostID, jobID) return err } +// DeleteSignedJobN is DeleteSignedJob that also says how many rows left the queue (0 = the job was not there). The +// hub records how a job ended only for a job that WAS queued (R-1383): a box must not be able to mint events by +// „clearing” job ids it invents. +func (s *Store) DeleteSignedJobN(hostID, jobID string) (int64, error) { + res, err := s.db.Exec(`DELETE FROM signed_jobs WHERE host_id = ? AND job_id = ?`, hostID, jobID) + if err != nil { + return 0, err + } + n, _ := res.RowsAffected() + return n, nil +} + // ---- slice 10D: DR capstone (recovery mode, DR bundle, re-enroll) ---------------------------- // SetRecoveryMode arms recovery mode for a host until `until` (the operator toggle; bounded