From 88e636445a7f1e3636f5d79799edce112bc13a57 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sat, 14 Nov 2026 17:05:40 +0100 Subject: [PATCH] hub (unreleased): the operator is told when a box's clock is more than one minute from the hub's (R-1384, D79) The host report's new sent_at (the box clock at the moment of sending) is compared with the hub's clock at arrival. Both directions; told once; clears within 30 s. A report that waited on the box raises nothing (reported_at is not read). A second box off in the same direction points at the hub's own clock. Operator-only. Not deployed. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- documentation/architecture/08-alarm-ladder.md | 1 + .../B/redproof-agent-sent-at.txt | 18 +++ .../B/redproof-hub-box-clock.txt | 44 ++++++ documentation/backlog/OPEN-ITEMS.md | 2 +- hub/CHANGELOG.md | 9 ++ hub/internal/api/box_clock_test.go | 139 ++++++++++++++++++ hub/internal/api/handler.go | 93 +++++++++++- .../notify/box_clock_operator_only_test.go | 12 ++ hub/internal/notify/dispatcher.go | 2 + 9 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 documentation/audits/night-build-2026-11-14/B/redproof-agent-sent-at.txt create mode 100644 documentation/audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt create mode 100644 hub/internal/api/box_clock_test.go create mode 100644 hub/internal/notify/box_clock_operator_only_test.go diff --git a/documentation/architecture/08-alarm-ladder.md b/documentation/architecture/08-alarm-ladder.md index 4edbd06c..1bc4a492 100644 --- a/documentation/architecture/08-alarm-ladder.md +++ b/documentation/architecture/08-alarm-ladder.md @@ -373,6 +373,7 @@ one info line beside `host_crash_restart`; `operatorOnlyEvents`, pinned by |---|---|---|---|---| | `tunnel_down` | error | the box's two newest host reports say the tunnel is `not_running` (more than one report cycle, 15 min) and the one before did not | the first `running` after it → `tunnel_recovered` (info) | `api/tunnel_test.go` | | `proxmox_trust_problem` | error (warning for „the root CA copy ends within 30 days”) | **[built, not deployed — hub `main` after v0.181.0]** the host report's `proxmox_trust` stanza names a cause (`03` §12a); once per (box, cause), not at every report | the first report with no problem → `proxmox_trust_ok` (info), one for all causes | `api/proxmox_trust_test.go` | +| `box_clock_wrong` | warning | **[built, not deployed — hub `main` after v0.181.0; needs an agent that stamps `sent_at`, built and not released]** the box's clock at the moment of sending is more than **one minute** from the hub's clock at arrival, in either direction (D79, `09` §3 decision 300); told once per direction. The hub cannot tell whose clock is wrong: a second box off in the same direction makes the alarm say „check the hub's own clock first”. An agent without `sent_at` is never judged | the first report within **30 seconds** → `box_clock_ok` (info) | `api/box_clock_test.go`; agent `hub/client_sentat_test.go` | | `os_update_stale` | warning | no successful OS leg for **7 days** while the switch is ON (agents that can run the leg only); the mail names the likely reason (box not reporting / the last leg's failure / no good night backup) | a successful leg | `TestAlarm_StaleLeg`, `TestAlarm_StaleNamesTheReason` | | `os_reboot_needed` | warning | the host has needed a reboot for **14 days** (from the FIRST scanned report that said so) | a scanned pass that finds nothing (agent ≥ 0.141.1 scans the host every pass) | `TestAlarm_RebootNeeded`, `TestRebootNeeded_ClearedByAScannedPass` | | `os_ring0_stalled` | error | ring 0 approved nothing for **7 days** in a layer while it has pending FAST-lane updates (a pending kernel does not count) | a new release | `TestAlarm_Ring0Stalled` | diff --git a/documentation/audits/night-build-2026-11-14/B/redproof-agent-sent-at.txt b/documentation/audits/night-build-2026-11-14/B/redproof-agent-sent-at.txt new file mode 100644 index 00000000..ca663e27 --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/B/redproof-agent-sent-at.txt @@ -0,0 +1,18 @@ +== MUTATION: no stamp at sending (the caller's old value leaves) + RED on the mutant, GREEN on the real file + --- FAIL: TestReport_StampsSentAtAtSending (0.01s) + client_sentat_test.go:43: send 0: sent_at is 40m0.011108314s old, want the moment of sending (reported_at was 40 minutes old) + client_sentat_test.go:43: send 1: sent_at is 40m0.011177234s old, want the moment of sending (reported_at was 40 minutes old) + client_sentat_test.go:51: the second send carries 2026-11-14 15:20:41.341281174 +0000 UTC, not after the first 2026-11-14 15:20:41.341281174 +0000 UTC — the stamp is not taken per send +== MUTATION: the stamp is the collect time + RED on the mutant, GREEN on the real file + --- FAIL: TestReport_StampsSentAtAtSending (0.01s) + client_sentat_test.go:43: send 0: sent_at is 40m0.063148269s old, want the moment of sending (reported_at was 40 minutes old) + client_sentat_test.go:43: send 1: sent_at is 40m0.063229086s old, want the moment of sending (reported_at was 40 minutes old) + client_sentat_test.go:51: the second send carries 2026-11-14 15:20:43 +0000 UTC, not after the first 2026-11-14 15:20:43 +0000 UTC — the stamp is not taken per send +== MUTATION: the stamp is taken once and kept + RED on the mutant, GREEN on the real file + --- FAIL: TestReport_StampsSentAtAtSending (0.01s) + client_sentat_test.go:51: the second send carries 2026-11-14 16:00:44.697310482 +0000 UTC, not after the first 2026-11-14 16:00:44.697310482 +0000 UTC — the stamp is not taken per send + +3 mutations, 0 not a red-proof diff --git a/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt b/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt new file mode 100644 index 00000000..2ac81e26 --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt @@ -0,0 +1,44 @@ +== MUTATION: the hub reads reported_at (the collect time) instead of the send time — a report that waited alarms + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_AQueuedReportRaisesNothing (0.05s) + box_clock_test.go:104: a report that waited on the box raised 2 wrong, 0 ok — an old reported_at is not a wrong clock +== MUTATION: only a box that is AHEAD is judged (the old tunnel-handshake shape) + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_BothDirectionsOnceThenClear (0.10s) + box_clock_test.go:68: wrong=0 ok=0, want ONE alarm for two reports of the same wrong clock +== MUTATION: the alarm is not remembered — every report alarms again + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_BothDirectionsOnceThenClear (0.11s) + box_clock_test.go:68: wrong=2 ok=0, want ONE alarm for two reports of the same wrong clock + box_clock_test.go:68: wrong=2 ok=0, want ONE alarm for two reports of the same wrong clock +== MUTATION: the alarm never clears + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_BothDirectionsOnceThenClear (0.13s) + box_clock_test.go:85: after the clock came back: wrong=1 ok=0 ([]), want one recovery + box_clock_test.go:85: after the clock came back: wrong=1 ok=0 ([]), want one recovery +== MUTATION: the alarm line is 30 s, not one minute + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_BothDirectionsOnceThenClear (0.09s) + box_clock_test.go:62: a clock under one minute off raised 1 wrong, 0 ok + box_clock_test.go:62: a clock under one minute off raised 1 wrong, 0 ok +== MUTATION: the clear line equals the alarm line (a clock at the edge flaps) + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_BothDirectionsOnceThenClear (0.10s) + box_clock_test.go:79: between the two lines: wrong=1 ok=1, want 1 and 0 + box_clock_test.go:79: between the two lines: wrong=1 ok=1, want 1 and 0 +== MUTATION: the hub's-own-clock hint ignores the direction + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_SecondBoxSameDirectionPointsAtTheHub (0.05s) + box_clock_test.go:125: boxes off in OPPOSITE directions must not point at the hub: The clock of h2 is 180 seconds behind of the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time. Re +== MUTATION: no hint at the hub's own clock + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClock_SecondBoxSameDirectionPointsAtTheHub (0.05s) + box_clock_test.go:137: wrong=3; the second box ahead must name the hub's clock and h1: "The clock of h2 is 179 seconds ahead of the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time + +8 mutations, 0 not a red-proof +== MUTATION: the clock events reach the household (not operator-only) + RED on the mutant, GREEN on the real file + --- FAIL: TestBoxClockEventsAreOperatorOnly (0.00s) + box_clock_operator_only_test.go:9: box_clock_wrong must be operator-only + +1 mutations, 0 not a red-proof diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index f1c8c7cc..54017b71 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -294,7 +294,7 @@ stopping line that lies. | **R-1358** | Box system & updates | P3 | **cloudflared 2026.10.4 misses two dependency fixes (VULN-183456 chi, 2026.10.5; VULN-184686 go-jose, 2026.10.6; the newest is 2026.10.7).** No public text on reach. Not pinned tonight: the tunnel is a box's only door, and 9202 has none to prove it on. | OPEN — filed 2026-11-03 | — | raise `CloudflaredImage` on the next controller release with a tunnel proof on Tester 1 | CC | | **R-1380** | Box system & updates | P3 | **Proxmox renews its own certificate and the agent's pin does not follow: from that day the agent cannot control or back up a guest (5 answers).** Found 2026-11-11 (the End-dates list, class T2, `documentation/audits/end-dates-list-2026-11-11/TABLE.md` DB02-3 to DB02-7). MEASURED: the agent trusts one exact Proxmox certificate (a pin), and the certificate on the three test boxes ends about two years after install — the N100's about 2028-07-16, then demo-hp and Tester 1 within three months. Proxmox makes a new certificate by itself about 14 days before the old one ends (so about 2028-07-02 on the N100). No code path gives the agent the new fingerprint, no alarm names the cause, no runbook holds the hand step, and nobody has tried it. Every box installed in the same season fails in the same fortnight. P3 only because the first date is in 2028; it is certain, not a risk. **Ruled 2026-11-11 13:17: yes (`09` §3 decision 297). Not built.** **2026-11-11 (afternoon): tried on demo-hp (`documentation/audits/certpin-spike-2026-11-11/README.md`).** A forced new certificate (`pvecm updatecerts --force`, then a reload of the web service; Proxmox's own renewal does that reload by itself) blinds the agent at once: `TLS pin mismatch`, no reconcile, no host report to the hub, 502 to the controller; guests and apps stay up. One operator mail names a symptom (`agent_channel_unknown`), none names the cause; the hub's Hosts page still read ONLINE eleven minutes after the last report (when it turns offline was not measured). The hand repair works and is written down (`documentation/runbooks/RUNBOOK-agent-proxmox-pin-repair.md`, 2.1 s from restart to a working call). **A CA file in place of the fingerprint works with no agent code:** with `proxmox.tls.ca_file` on a readable copy of the box's root CA and the fingerprint empty, the agent went through a second new certificate with no hand step; a wrong CA file was refused. Two facts for the fix: the agent's user cannot read Proxmox's certificate files, and a set fingerprint wins over the CA file. The design with both ways is `03-host-agent.md` §12a; CC's pick differs from the ruled shape, so it is decision D80. | READY — **2026-11-14 night: BUILT, not released** (felhom-agent `386a0c4` + `a9ab47e`; the installer on felhom.eu `main`, not tagged). The root step keeps the copy; the agent moves a pinned box in the order copy → proof → config, clears the fingerprint last, and a pinned box whose certificate changed heals at the first failed call; the installer gives a new box the copy and no fingerprint; the cause of a trust failure is named. `03` §12a is `[FACT — built, not released]`. **LEFT: the proof on a box** (demo-hp, a forced new certificate, twice; then the release with both signed jobs and the installer tag). What the hub does not tell yet: R-1396. Earlier: READY — **2026-11-14 09:55 — RULED (D80, `09` §3 decision 303): the root CA copy.** Built in release 2 (the host release). OPEN — filed 2026-11-11 (end-dates list) | — | the operator rules D80 (CA file, or follow the fingerprint); then the design's open parts in `03-host-agent.md` §12a (the root step that refreshes the copy; an alarm that names the cause), then build, then prove on a scratch host with a forced renewal | CC | | **R-1382** | Box system & updates | P3 | **A box that was off past an end date: nobody knows if it comes back by itself (5 answers).** Found 2026-11-11 (the End-dates list, class T4, `documentation/audits/end-dates-list-2026-11-11/TABLE.md`; read in source and documents, never tried). After months without power: the web certificate has ended (DB01-6: does Traefik renew at the first start, and what does the household see until then), Debian's package keys may be too old to verify an update (DB16-6), the tunnel client may be below what Cloudflare accepts (DB19-6), a 30-day restore copy is past its date (DB24-6), and the 14-day countdown after a stopped off-site copy ran out while the box was off (DB04-6). Tester 2 has been off since 2026-10-04 and is the natural test. | OPEN — filed 2026-11-11 (end-dates list) | — | When Tester 2 comes back (or with a scratch host and a moved clock on Tier 0, as a task): record what renews by itself and what needs a hand; put the hand steps in a runbook. | CC | -| **R-1384** | Box system & updates | P3 | **A verdict or an „already ran” check trusts a clock that can be wrong (30 answers).** Found 2026-11-11 (the Clock list, class W2, `documentation/audits/clock-list-2026-11-11/TABLE.md`). MEASURED: the N100's host is 32.6 s behind and its time service does not run (R-1301); the other two boxes are in step. Nothing checks a box's clock except one hub alarm for a tunnel handshake stamped in the future. The hub takes the box's own time for backup age, off-site age and proof age (CH02-6, CH07-6, CH07-8, CH08-6); a box with a wrong clock refuses every signed job (CA09-6); a stamp from the future blocks the OS-update leg, the trim and the line test until the clock passes it (CA02-6, CA02-8, CA11-6, CA11-8, CA12-6); the crash guard reads the clock at early boot (CA04-6, CA04-8); a hub clock jump moves its own daily jobs (CH03-6, CH04-6, CH06-6, CH09-6, CH16-6, CH18-6, CH20-6, CH24-6). The others: CA05-6, CA05-8, CA06-6, CA07-6, CA10-6, CA20-6, CC04-6, CC05-6, CC14-6, CC18-6. Known parts stay in their rows: R-1189, R-1206, R-1260. **Ruled 2026-11-11 13:17: yes (`09` §3 decision 300). Not built.** **2026-11-11 (afternoon): why the N100's clock is wrong is known (R-1301, `documentation/audits/n100-clock-2026-11-11/README.md`).** Numbers from source: a host clock more than 120 s behind refuses every signed job (`felhom-agent/internal/authz/verifier.go:21`, `:153-162`); at about 1.25 s a day the N100 gets there in about 70 days (mid-January 2027, an estimate from one measurement). | OPEN — **2026-11-14: the one box whose clock was wrong (the N100, 32.6 s) is right again (R-1301 closed); this row's check is still not built.** Earlier: OPEN — filed 2026-11-11 (clock list) | — | Fix shape (ruled, not built): the hub compares the time in each report with its own, alarms the operator above one minute, and does not take a time from the future as „fresh”. | CC | +| **R-1384** | Box system & updates | P3 | **A verdict or an „already ran” check trusts a clock that can be wrong (30 answers).** Found 2026-11-11 (the Clock list, class W2, `documentation/audits/clock-list-2026-11-11/TABLE.md`). MEASURED: the N100's host is 32.6 s behind and its time service does not run (R-1301); the other two boxes are in step. Nothing checks a box's clock except one hub alarm for a tunnel handshake stamped in the future. The hub takes the box's own time for backup age, off-site age and proof age (CH02-6, CH07-6, CH07-8, CH08-6); a box with a wrong clock refuses every signed job (CA09-6); a stamp from the future blocks the OS-update leg, the trim and the line test until the clock passes it (CA02-6, CA02-8, CA11-6, CA11-8, CA12-6); the crash guard reads the clock at early boot (CA04-6, CA04-8); a hub clock jump moves its own daily jobs (CH03-6, CH04-6, CH06-6, CH09-6, CH16-6, CH18-6, CH20-6, CH24-6). The others: CA05-6, CA05-8, CA06-6, CA07-6, CA10-6, CA20-6, CC04-6, CC05-6, CC14-6, CC18-6. Known parts stay in their rows: R-1189, R-1206, R-1260. **Ruled 2026-11-11 13:17: yes (`09` §3 decision 300). Not built.** **2026-11-11 (afternoon): why the N100's clock is wrong is known (R-1301, `documentation/audits/n100-clock-2026-11-11/README.md`).** Numbers from source: a host clock more than 120 s behind refuses every signed job (`felhom-agent/internal/authz/verifier.go:21`, `:153-162`); at about 1.25 s a day the N100 gets there in about 70 days (mid-January 2027, an estimate from one measurement). | NARROWED — the hub's check is built, not released. **2026-11-14 (night build):** the agent stamps its clock at the moment of sending (`sent_at`) and the hub alarms the operator when it is more than one minute from the hub's clock, in either direction, and says so when it is back (`08-alarm-ladder.md`, `box_clock_wrong`). A report that waited on the box raises nothing (tested). Not released, not seen on a box. LEFT after the release: the 30 answers themselves still trust the clock; the alarm only makes a wrong clock known. OPEN — **2026-11-14: the one box whose clock was wrong (the N100, 32.6 s) is right again (R-1301 closed); this row's check is still not built.** Earlier: OPEN — filed 2026-11-11 (clock list) | — | Fix shape (ruled, not built): the hub compares the time in each report with its own, alarms the operator above one minute, and does not take a time from the future as „fresh”. | CC | | **R-1004** | Box system & updates | P4 | **After an unclean stop every guest ext4 volume waits ~43 s in multi-mount protection before it mounts, so each restart after a power cut takes ~1.5 min longer.** Kernel: „EXT4-fs warning (device dm-6): ext4_multi_mount_protect: MMP interval 42 higher than expected, please wait" on dm-6/dm-7 and loop0/loop1, demo-hp and Tester 1, every round of `audits/night-power-2026-10-13/` (R1 `repairs.txt`). | **READY** — filed 2026-10-14 | — | CC: find where MMP is switched on for the guest volumes (Proxmox `pct` create, or our own mkfs), whether it protects anything on a one-host disk, and measure the restart without it on a scratch guest | CC | | **R-1013** | Box system & updates | P4 | **After a power cut the agent's start-up stale-lock pass logs an ERROR „pct unlock failed … no lock found" when Proxmox's own start-all removed the lock in the same second.** The outcome is the goal (no lock), so the line is a false alarm in the log. Seen on demo-hp 2026-10-14 12:07:44–45 UTC (`audits/power-cut-fixes-2026-10-14/E/after-boot.txt`; Proxmox: „removed left over backup lock from '9201'!"). Small, but the agent's release for this session is spent (v0.156.1). | **OPEN** — filed 2026-10-14 | — | CC: treat „no lock found" from `pct unlock` as already clear (INFO), with a test; ride the next agent release | CC | | **R-1061** | Box system & updates | P4 | **A per-customer floor reached Tester 1 only at its next 15-minute report, not in seconds.** Set 2026-10-15 12:55:51 UTC for tester-1, demo-hp and demo-felhom together (hub 0.153.0, CC's token): demo-hp and demo-felhom ran 0.314.0 within 13 s (the hub's wake); Tester 1 moved at 13:09:35, its regular report (`audits/gate-restore-2026-10-15/release/03-boxes-move.txt`). It had restarted at 12:54:29 onto rc.1 and logged „hub wait channel active". Why the wake did not reach it is not read (the wake line is DEBUG). No harm: the floor arrived. | **OPEN** — filed 2026-10-15 | — | Read Tester 1's DEBUG ring at the next floor change; compare with the hub's bump log | CC | diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index ea93dc2b..75e370b9 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -3,6 +3,15 @@ - `bundlefiles`: the line „bundle X holds the same files as the vouched Y” is logged once per pair (INFO), not at every view of the System page (it was 6 DEBUG lines per view, seen at the 0.181.0 deploy). No behaviour change. - **Release 3 („the hub tells you”), built 2026-11-14 night, NOT deployed. Every new alarm is operator-only.** +- **A box's clock against the hub's (R-1384, D79, `09` §3 decision 300).** The host report's new `sent_at` (the box's + clock at the moment the agent sends; agents of release 3) is compared with the hub's clock at the moment the request + arrives. More than one minute apart, in either direction → one operator-only `box_clock_wrong` (warning), told once; + back within 30 seconds → one `box_clock_ok`. `reported_at` is NOT read for this (it is the collect's start, so a + report that waited would alarm). A slow line cannot raise it (it adds less than the agent's 30 s timeout, in one + direction). **The hub's own clock:** the hub cannot tell whose clock is wrong; when a second box is off in the same + direction the alarm says „check the HUB's own clock (DooPlex) first”. A report with no `sent_at` is never judged. + Tests `TestBoxClock_*`, `TestBoxClockEventsAreOperatorOnly`; 9 mutations seen red + (`documentation/audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt`). - **Why a box's agent does not trust Proxmox (R-1396, R-1380; `03` §12a).** The host report's new `proxmox_trust` stanza (agents of release 2b) is read: one `proxmox_trust_problem` per (box, cause) — told once, not at every report; `error`, and `warning` for „the root CA copy ends within 30 days” — and one `proxmox_trust_ok` when the box diff --git a/hub/internal/api/box_clock_test.go b/hub/internal/api/box_clock_test.go new file mode 100644 index 00000000..f144270d --- /dev/null +++ b/hub/internal/api/box_clock_test.go @@ -0,0 +1,139 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// clockReport is a host report of `host` whose agent stamped `sent_at` = the hub's now + skew, and whose collect +// started `collectAge` before that (reported_at) — a report that WAITED on the box before it was sent. +func clockReport(host string, skew, collectAge time.Duration, withSentAt bool) string { + b := validReportBody(host) + now := time.Now().Add(skew) + extra := fmt.Sprintf(`,"reported_at":%q`, now.Add(-collectAge).UTC().Format(time.RFC3339)) + if withSentAt { + extra += fmt.Sprintf(`,"sent_at":%q`, now.UTC().Format(time.RFC3339Nano)) + } + return b[:strings.LastIndex(b, "}")] + extra + "}" +} + +func clockSetup(t *testing.T) (*Handler, *store.Store, func(host string, skew, age time.Duration, sent bool)) { + t.Helper() + h, st, _ := newTestHandler(t) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"}) + for _, id := range []string{"h1", "h2"} { + st.UpsertHost(&store.Host{HostID: id, CustomerID: "c1", APIKey: "KEY-" + id}) + } + post := func(host string, skew, age time.Duration, sent bool) { + t.Helper() + if rr := do(h, http.MethodPost, "/host-report", "KEY-"+host, clockReport(host, skew, age, sent)); rr.Code != 200 { + t.Fatalf("report: %d %s", rr.Code, rr.Body.String()) + } + } + return h, st, post +} + +func clockEvents(t *testing.T, st *store.Store) (wrong, ok []store.Event) { + t.Helper() + since := time.Now().Add(-time.Hour) + wrong, _ = st.GetEventsByType("c1", eventBoxClockWrong, since) + ok, _ = st.GetEventsByType("c1", eventBoxClockOK, since) + return +} + +// R-1384 (D79): a box more than one minute off, in EITHER direction, is told to the operator once; it clears once +// when the clock is back. COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt). +func TestBoxClock_BothDirectionsOnceThenClear(t *testing.T) { + for _, c := range []struct { + name string + skew time.Duration + dir string + }{{"behind", -95 * time.Second, "behind"}, {"ahead", 95 * time.Second, "ahead"}} { + t.Run(c.name, func(t *testing.T) { + _, st, post := clockSetup(t) + post("h1", 20*time.Second, 0, true) // in step: nothing + post("h1", -45*time.Second, 0, true) // off, but under one minute: nothing + if w, o := clockEvents(t, st); len(w) != 0 || len(o) != 0 { + t.Fatalf("a clock under one minute off raised %d wrong, %d ok", len(w), len(o)) + } + post("h1", c.skew, 0, true) + post("h1", c.skew, 0, true) + w, o := clockEvents(t, st) + if len(w) != 1 || len(o) != 0 { + t.Fatalf("wrong=%d ok=%d, want ONE alarm for two reports of the same wrong clock", len(w), len(o)) + } + if w[0].Severity != "warning" || !strings.Contains(w[0].Message, "h1") || !strings.Contains(w[0].Message, "seconds "+c.dir+" ") || + !strings.Contains(string(w[0].DetailsJSON), `"direction":"`+c.dir+`"`) { + t.Errorf("alarm = %+v, want a warning that names h1 and %q", w[0], c.dir) + } + if strings.Contains(w[0].Message, "HUB's own clock") { + t.Errorf("one box alone must not point at the hub's clock: %s", w[0].Message) + } + post("h1", c.skew/2, 0, true) // 47 s: under the alarm line, above the clear line → no word + if w, o := clockEvents(t, st); len(w) != 1 || len(o) != 0 { + t.Fatalf("between the two lines: wrong=%d ok=%d, want 1 and 0", len(w), len(o)) + } + post("h1", 2*time.Second, 0, true) + post("h1", 2*time.Second, 0, true) + w, o = clockEvents(t, st) + if len(w) != 1 || len(o) != 1 || !strings.Contains(o[0].Message, "was "+c.dir) { + t.Fatalf("after the clock came back: wrong=%d ok=%d (%v), want one recovery", len(w), len(o), o) + } + post("h1", c.skew, 0, true) // and a second time it goes wrong, it is told again + if w, _ := clockEvents(t, st); len(w) != 2 { + t.Fatalf("a second wrong clock after a recovery: wrong=%d, want 2", len(w)) + } + }) + } +} + +// The trap the ruling names: a report that WAITED on the box (a long collect) carries an old reported_at. That is not +// a wrong clock, and it raises nothing — the hub reads the time stamped at SENDING. And an agent that sends no +// sent_at is never judged, however old its reported_at is. +func TestBoxClock_AQueuedReportRaisesNothing(t *testing.T) { + _, st, post := clockSetup(t) + post("h1", 0, 10*time.Minute, true) // collected 10 minutes ago, sent now, clock right + post("h1", 0, 10*time.Minute, false) // an older agent: no sent_at at all + post("h1", 0, -10*time.Minute, false) + if w, o := clockEvents(t, st); len(w) != 0 || len(o) != 0 { + t.Fatalf("a report that waited on the box raised %d wrong, %d ok — an old reported_at is not a wrong clock", len(w), len(o)) + } + // positive control: the same handler, the same box, does alarm on the send time + post("h1", -5*time.Minute, 0, true) + if w, _ := clockEvents(t, st); len(w) != 1 { + t.Fatalf("control: a send time 5 minutes behind raised %d alarm(s), want 1", len(w)) + } +} + +// When the HUB's clock is wrong, every healthy box looks wrong the same way. The second box's alarm says so. A box +// off in the OTHER direction does not. +func TestBoxClock_SecondBoxSameDirectionPointsAtTheHub(t *testing.T) { + _, st, post := clockSetup(t) + post("h1", 3*time.Minute, 0, true) + post("h2", -3*time.Minute, 0, true) + w, _ := clockEvents(t, st) + if len(w) != 2 { + t.Fatalf("wrong=%d, want 2", len(w)) + } + for _, e := range w { + if strings.Contains(e.Message, "HUB's own clock") { + t.Errorf("boxes off in OPPOSITE directions must not point at the hub: %s", e.Message) + } + } + post("h2", 3*time.Minute, 0, true) // h2 flips: now both are ahead + w, _ = clockEvents(t, st) + var last store.Event + for _, e := range w { + if strings.Contains(e.Message, "of h2 ") && strings.Contains(e.Message, "ahead") { + last = e + } + } + if len(w) != 3 || !strings.Contains(last.Message, "HUB's own clock") || !strings.Contains(last.Message, "h1") { + t.Fatalf("wrong=%d; the second box ahead must name the hub's clock and h1: %q", len(w), last.Message) + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 0085776d..fd89c5b9 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -778,7 +778,11 @@ type hostReportPayload struct { // ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it // is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b). ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"` - Guests []struct { + // SentAt (R-1384, D79): the box's clock at the moment the agent SENDS the report (stamped in the agent's + // Client.Report, after the collect). "" = an agent that does not stamp it → the clock is not judged. + // reported_at is NOT used for this: it is stamped when the collect starts, and a collect can take long. + SentAt string `json:"sent_at"` + Guests []struct { VMID int `json:"vmid"` Name string `json:"name"` Status string `json:"status"` @@ -942,6 +946,7 @@ type hostStorageTarget struct { // handleHostReport ingests the agent's host-report (the heartbeat) and returns the // control envelope (agent spec §5). func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) { + received := time.Now() // before the body is read: the clock check (R-1384) compares the box's send time with THIS hostID, custID, isGlobal, ok := h.checkAuthHost(r) if !ok { http.Error(w, "Unauthorized", http.StatusUnauthorized) @@ -1012,6 +1017,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) { h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB) h.checkWGTunnel(hostID, custID, rep.Wireguard) h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust) + h.checkBoxClock(hostID, custID, rep.SentAt, received) h.checkCrash(hostID, custID, body) h.checkDeliberateOff(hostID, custID, &rep) @@ -3240,6 +3246,91 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) { } } +// Box clock alarm (R-1384, D79, `09` §3 decision 300; `08-alarm-ladder.md`). The hub compares the box's clock at +// the moment of SENDING (`sent_at`) with its own clock at the moment the request arrives. More than one minute apart, +// in either direction → one operator-only `box_clock_wrong`; back within 30 seconds → one `box_clock_ok` (the gap +// between 30 and 60 s keeps a clock at the edge from mailing at every report). +// +// What this can NOT tell: whose clock is wrong. A slow line can only make a box look BEHIND, and by less than the +// agent's request timeout (30 s), so it cannot raise the alarm alone. When the HUB's clock is wrong, every healthy box +// looks wrong in the SAME direction — so the message says so when a second box is off in the same direction. +// A report with no `sent_at` (an older agent) is never judged. Pinned by TestBoxClock_*. +const ( + eventBoxClockWrong = "box_clock_wrong" + eventBoxClockOK = "box_clock_ok" + boxClockRaiseAbove = time.Minute + boxClockClearBelow = 30 * time.Second +) + +func (h *Handler) checkBoxClock(hostID, custID, sentAt string, received time.Time) { + if sentAt == "" { + return + } + sent, err := time.Parse(time.RFC3339Nano, sentAt) + if err != nil { + h.logger.Printf("[WARN] host %s: sent_at is not a time (%d bytes); the clock is not judged", hostID, len(sentAt)) + return + } + skew := sent.Sub(received) // > 0: the box is AHEAD of the hub + abs := skew + if abs < 0 { + abs = -abs + } + dir := "ahead" + if skew < 0 { + dir = "behind" + } + key := func(d string) string { return "box_clock:" + hostID + ":" + d } + emit := func(typ, sev, msg string, details map[string]any) { + dj, _ := json.Marshal(details) + if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil { + h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err) + } else if h.dispatcher != nil { + go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub") + } + } + raised := "" + for _, d := range []string{"ahead", "behind"} { + if !h.store.OSAlarmRaised(key(d)).IsZero() { + raised = d + } + } + secs := int64(abs / time.Second) + switch { + case abs > boxClockRaiseAbove: + if raised == dir { + return // said once + } + if raised != "" { + _ = h.store.SetOSAlarmRaised(key(raised), time.Time{}) + } + _ = h.store.SetOSAlarmRaised(key(dir), received) + // the hub's own clock: another box off in the same direction points at the hub, not at the boxes + var same []string + if hosts, lerr := h.store.ListHosts(); lerr == nil { + for _, o := range hosts { + if o.HostID != hostID && !h.store.OSAlarmRaised("box_clock:"+o.HostID+":"+dir).IsZero() { + same = append(same, o.HostID) + } + } + } + sort.Strings(same) + hint := "" + if len(same) > 0 { + hint = fmt.Sprintf(" %d other box(es) are %s too (%s): check the HUB's own clock (DooPlex) first.", len(same), dir, strings.Join(same, ", ")) + } + h.logger.Printf("[WARN] host %s clock: %d s %s of the hub (same direction: %d other)", hostID, secs, dir, len(same)) + emit(eventBoxClockWrong, "warning", + fmt.Sprintf("The clock of %s is %d seconds %s the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time. Read its time service (`chronyc tracking`).%s", hostID, secs, map[string]string{"ahead": "ahead of", "behind": "behind"}[dir], hint), + map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "direction": dir, "same_direction": same}) + case abs <= boxClockClearBelow && raised != "": + _ = h.store.SetOSAlarmRaised(key(raised), time.Time{}) + h.logger.Printf("[INFO] host %s clock: in step again (%d s; was %s)", hostID, secs, raised) + emit(eventBoxClockOK, "info", fmt.Sprintf("The clock of %s is in step with the hub's clock again (was %s).", hostID, raised), + map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "was": raised}) + } +} + // Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts // Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends // within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box, diff --git a/hub/internal/notify/box_clock_operator_only_test.go b/hub/internal/notify/box_clock_operator_only_test.go new file mode 100644 index 00000000..e2f27d14 --- /dev/null +++ b/hub/internal/notify/box_clock_operator_only_test.go @@ -0,0 +1,12 @@ +package notify + +import "testing" + +// R-1384 (D79): a box's clock against the hub's is the operator's to read; the household is never mailed about it. +func TestBoxClockEventsAreOperatorOnly(t *testing.T) { + for _, e := range []string{"box_clock_wrong", "box_clock_ok"} { + if !operatorOnlyEvents[e] { + t.Errorf("%s must be operator-only", e) + } + } +} diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index c2f60bc0..1a534b60 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -844,6 +844,8 @@ var operatorOnlyEvents = map[string]bool{ // R-1396 / R-1380 (hub v0.182.0): why a box's agent does not trust Proxmox's certificate, and its recovery — a box // fact only the operator can act on. Hub-origin only (minted from the host report's `proxmox_trust` stanza). "proxmox_trust_problem": true, + "box_clock_wrong": true, // R-1384: a box's clock against the hub's — the operator's to read + "box_clock_ok": true, "proxmox_trust_ok": true, // R-836 (hub v0.143.0): the kernel lane's step results and the day-before notice's own record — operator facts. The // household's word is the day-before mail itself (SendKernelNotice), never these events.