diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 9f619ab2..682fc87b 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,14 @@ +## Unreleased — R-1344: the agent's restart of a controller that does not answer + +- **`controller_unresponsive_restart`** (warning, operator-only; in `allowedEventTypes` and `operatorOnlyEvents`, pinned + by `TestControllerSupervisorEventsAreAllowlistedAndOperatorOnly`): mailed when the stanza's `unresponsive_restart_at` + MOVES (agent ≥ 0.171.0). That restart also moves `last_restart_at`; it is ONE act, so it is NOT also mailed as + `controller_restarted_by_agent`. +- **The `controller_unresponsive` text follows the agent:** with `unresponsive_restart_after_s` it says the agent + restarts the controller „2 minutes after this alarm" unless a backup, a restore, an update or the brake holds; without + it (an older agent) it says this agent does not restart it. Tests `TestControllerSupervisorChecker_UnresponsiveRestart`, + `…UnresponsiveTextFollowsTheAgent` — red first; 3 mutants killed. + ## v0.177.0 (2026-10-29) — „Reboot guest" from the hub (D65 = A); the slow-request line and the vouch save's timing (R-1346) - **D65 = A (operator ruling 2026-10-29 15:50; `04` §6.1).** The AGENT's closed list (`hostOperatorActionNames`) is diff --git a/hub/internal/api/controller_supervisor_event_test.go b/hub/internal/api/controller_supervisor_event_test.go index da25c51c..20ff066d 100644 --- a/hub/internal/api/controller_supervisor_event_test.go +++ b/hub/internal/api/controller_supervisor_event_test.go @@ -10,7 +10,7 @@ import ( // allowlisted (the project's single register of legitimate types) AND operator-only (a missing // customerMessages entry is not a block — the v0.78.0 defect). func TestControllerSupervisorEventsAreAllowlistedAndOperatorOnly(t *testing.T) { - for _, et := range []string{"controller_restarted_by_agent", "controller_crashloop"} { + for _, et := range []string{"controller_restarted_by_agent", "controller_crashloop", "controller_unresponsive", "controller_unresponsive_restart"} { if !allowedEventTypes[et] { t.Fatalf("%s must be in allowedEventTypes (R-77: an unregistered type is an inert seam)", et) } diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index f63ac010..9866dd83 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -2419,7 +2419,9 @@ var allowedEventTypes = map[string]bool{ // R-539 (hub v0.117.0, agent v0.132.0) — hub-generated from slow_crashloop_since; operator-only. "controller_slow_crashloop": true, // R-1344 (2026-10-29): hub-generated from the stanza's unresponsive_since; operator-only (notify.operatorOnlyEvents). - "controller_unresponsive": true, + // 2026-10-30: controller_unresponsive_restart from unresponsive_restart_at (agent ≥ 0.171.0); operator-only. + "controller_unresponsive": true, + "controller_unresponsive_restart": true, // R-550 (hub v0.117.0, controller v0.246.0) — pushed by the controller at startup when it finds a // restore that was in flight when the box stopped. For the household: they can run it again. "restore_interrupted": true, diff --git a/hub/internal/monitor/controller_supervisor.go b/hub/internal/monitor/controller_supervisor.go index cf63c929..d04eb7be 100644 --- a/hub/internal/monitor/controller_supervisor.go +++ b/hub/internal/monitor/controller_supervisor.go @@ -18,7 +18,10 @@ import ( // - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES; // - `controller_slow_crashloop` (warning, operator-only) when a guest's slow_crashloop_since MOVES // (R-539, operator ruling 3 of 2026-09-16, agent v0.132.0): five restarts inside 24 hours, the -// loop the 15-minute brake cannot see because each restart falls outside its window. +// loop the 15-minute brake cannot see because each restart falls outside its window; +// - `controller_unresponsive` (warning, operator-only) when unresponsive_since MOVES (R-1344), and +// `controller_unresponsive_restart` (warning, operator-only) when unresponsive_restart_at MOVES (agent ≥ 0.171.0) — +// that restart is NOT also mailed as controller_restarted_by_agent. // // TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes // restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not @@ -39,10 +42,11 @@ type ControllerSupervisorChecker struct { } type supSeen struct { - lastRestartAt string - crashloopSince string - slowCrashloopSince string - unresponsiveSince string + lastRestartAt string + crashloopSince string + slowCrashloopSince string + unresponsiveSince string + unresponsiveRestartAt string } // ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape. @@ -62,6 +66,10 @@ type ControllerSupervisorGuest struct { // sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event. Unresponsive bool `json:"unresponsive"` UnresponsiveSince string `json:"unresponsive_since"` + // R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): how long after the alarm the agent restarts a controller that + // still does not answer (0 = an older agent: alarm only), and the last such restart (equals last_restart_at for it). + UnresponsiveRestartAfterS int `json:"unresponsive_restart_after_s"` + UnresponsiveRestartAt string `json:"unresponsive_restart_at"` } // ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed @@ -104,7 +112,7 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests key := fmt.Sprintf("%s/%d", hostID, g.VMID) prev, known := c.seen[key] c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince, - unresponsiveSince: g.UnresponsiveSince} + unresponsiveSince: g.UnresponsiveSince, unresponsiveRestartAt: g.UnresponsiveRestartAt} if !known { if g.Crashloop && g.CrashloopSince != "" { c.emit(customerID, hostID, g, "controller_crashloop") @@ -117,7 +125,12 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests } continue } - if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt { + // A restart for not answering moves last_restart_at too: it is ONE act, mailed once, as the warning. + unrespRestart := g.UnresponsiveRestartAt != "" && g.UnresponsiveRestartAt != prev.unresponsiveRestartAt + if unrespRestart { + c.emit(customerID, hostID, g, "controller_unresponsive_restart") + } + if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt && !(unrespRestart && g.LastRestartAt == g.UnresponsiveRestartAt) { c.emit(customerID, hostID, g, "controller_restarted_by_agent") } if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince { @@ -149,8 +162,17 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll hostID, g.VMID, g.Restarts24h, g.LastReason) case "controller_unresponsive": severity = "warning" - message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent does NOT restart it yet (R-1344: the measurement must first show a busy night never looks like this) — use „Restart controller\" on the host page if it stays so.", - hostID, g.VMID, g.UnresponsiveSince) + if g.UnresponsiveRestartAfterS > 0 { + message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent restarts it if it still does not answer %s after this alarm — unless a backup, a restore or an update is running, or the restart brake holds; then use „Restart controller\" on the host page.", + hostID, g.VMID, g.UnresponsiveSince, humanSeconds(g.UnresponsiveRestartAfterS)) + } else { + message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). This agent does NOT restart it itself (an agent older than 0.171.0) — use „Restart controller\" on the host page if it stays so.", + hostID, g.VMID, g.UnresponsiveSince) + } + case "controller_unresponsive_restart": + severity = "warning" + message = fmt.Sprintf("Host %s guest %d: the controller ran but did not answer its health check (since %s), so the agent restarted it at %s. Look at the box if this repeats. Reason: %s", + hostID, g.VMID, g.UnresponsiveSince, g.UnresponsiveRestartAt, g.LastReason) default: return } @@ -159,7 +181,7 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll "last_restart_at": g.LastRestartAt, "last_reason": g.LastReason, "crashloop_since": g.CrashloopSince, "parked": g.Parked, "restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince, - "unresponsive_since": g.UnresponsiveSince, + "unresponsive_since": g.UnresponsiveSince, "unresponsive_restart_at": g.UnresponsiveRestartAt, }) c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason) if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { @@ -170,3 +192,14 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll c.onEvent(customerID, eventType, severity, message, string(details), "hub") } } + +// humanSeconds says a whole number of minutes as „2 minutes", anything else as „90 seconds". +func humanSeconds(n int) string { + switch { + case n == 60: + return "1 minute" + case n%60 == 0: + return fmt.Sprintf("%d minutes", n/60) + } + return fmt.Sprintf("%d seconds", n) +} diff --git a/hub/internal/monitor/controller_supervisor_test.go b/hub/internal/monitor/controller_supervisor_test.go index fed8d8a3..f793c18a 100644 --- a/hub/internal/monitor/controller_supervisor_test.go +++ b/hub/internal/monitor/controller_supervisor_test.go @@ -3,6 +3,7 @@ package monitor import ( "io" "log" + "strings" "sync" "testing" @@ -175,3 +176,48 @@ func TestControllerSupervisorChecker_Unresponsive(t *testing.T) { t.Fatalf("first sight while unresponsive: %v", evs) } } + +// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as +// controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at +// moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing +// before the branch existed. +func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) { + st := newCapStore(t) + rec := &evRec{} + c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{}) + c.Check() + rec.take() // first sight while unresponsive: the alarm (pinned above) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{}) + c.Check() + if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" { + t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs) + } + c.Check() + if evs := rec.take(); len(evs) != 0 { + t.Fatalf("the same restart emitted again: %v", evs) + } + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{}) + c.Check() + if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" { + t.Fatalf("a not-running restart after it: %v", evs) + } +} + +// The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md +// corollary: when the behaviour changes, the alarm text changes with it). +func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) { + for _, tc := range []struct{ field, want, never string }{ + {`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"}, + {``, "does NOT restart it", "restarts it if"}, + } { + st := newCapStore(t) + var msg string + c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{}) + c.Check() + if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) { + t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never) + } + } +} diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 05063559..b261831d 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -744,8 +744,10 @@ var recordOnlyEvents = map[string]bool{ // is customer-grade and would be DISCARDED by a template). Turning the fallback into a gate would // change behaviour well outside this concern. var operatorOnlyEvents = map[string]bool{ - // R-1344 (2026-10-29): the controller runs but does not answer (agent stanza, alarm only) — the operator's. - "controller_unresponsive": true, + // R-1344 (2026-10-29): the controller runs but does not answer (agent stanza) — the operator's; and (2026-10-30) the + // agent's restart of it. + "controller_unresponsive": true, + "controller_unresponsive_restart": true, // R-97a. A customer can take no action on a failed whole-guest backup, and being told it failed // while it is still retrying behind the R-88 breaker is alarming without being actionable. "whole_guest_backup_failed": true,