hub: controller_unresponsive_restart (R-1344 restart, agent >= 0.171.0) — one mail per restart, not also controller_restarted_by_agent; the unresponsive alarm text says what the agent will do
This commit is contained in:
1 parent
28f3e15738
commit
be538975fc
6 files changed
+108
-14
No files matched your search
@@ -3,6 +3,7 @@ package monitor
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
@@ -175,3 +176,48 @@ func TestControllerSupervisorChecker_Unresponsive(t *testing.T) {
|
||||
t.Fatalf("first sight while unresponsive: %v", evs)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as
|
||||
// controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at
|
||||
// moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing
|
||||
// before the branch existed.
|
||||
func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
rec := &evRec{}
|
||||
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
rec.take() // first sight while unresponsive: the alarm (pinned above)
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
|
||||
t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
|
||||
}
|
||||
c.Check()
|
||||
if evs := rec.take(); len(evs) != 0 {
|
||||
t.Fatalf("the same restart emitted again: %v", evs)
|
||||
}
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
||||
t.Fatalf("a not-running restart after it: %v", evs)
|
||||
}
|
||||
}
|
||||
|
||||
// The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md
|
||||
// corollary: when the behaviour changes, the alarm text changes with it).
|
||||
func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) {
|
||||
for _, tc := range []struct{ field, want, never string }{
|
||||
{`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"},
|
||||
{``, "does NOT restart it", "restarts it if"},
|
||||
} {
|
||||
st := newCapStore(t)
|
||||
var msg string
|
||||
c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) {
|
||||
t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user