hub: controller_unresponsive_restart (R-1344 restart, agent >= 0.171.0) — one mail per restart, not also controller_restarted_by_agent; the unresponsive alarm text says what the agent will do

This commit is contained in:
admin committed 2026-10-30 09:23:51 +01:00
1 parent 28f3e15738
commit be538975fc
6 files changed
+108 -14

No files matched your search

@@ -3,6 +3,7 @@ package monitor
import (
"io"
"log"
"strings"
"sync"
"testing"
@@ -175,3 +176,48 @@ func TestControllerSupervisorChecker_Unresponsive(t *testing.T) {
t.Fatalf("first sight while unresponsive: %v", evs)
}
}
// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as
// controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at
// moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing
// before the branch existed.
func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) {
st := newCapStore(t)
rec := &evRec{}
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
c.Check()
rec.take() // first sight while unresponsive: the alarm (pinned above)
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
c.Check()
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
}
c.Check()
if evs := rec.take(); len(evs) != 0 {
t.Fatalf("the same restart emitted again: %v", evs)
}
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
c.Check()
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
t.Fatalf("a not-running restart after it: %v", evs)
}
}
// The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md
// corollary: when the behaviour changes, the alarm text changes with it).
func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) {
for _, tc := range []struct{ field, want, never string }{
{`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"},
{``, "does NOT restart it", "restarts it if"},
} {
st := newCapStore(t)
var msg string
c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0))
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{})
c.Check()
if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) {
t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never)
}
}
}