224 lines
11 KiB
Go
224 lines
11 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// R-523 — the checker that turns the agent's controller_supervisor stanza into events.
|
|
// The JSON below is the exact wire shape the agent's TestControllerSupervisorStanza_WireShape pins.
|
|
|
|
func supReport(guests string) []byte {
|
|
return []byte(`{"host_id":"h1","controller_supervisor":{"guests":` + guests + `}}`)
|
|
}
|
|
|
|
type evRec struct {
|
|
mu sync.Mutex
|
|
evs []string
|
|
}
|
|
|
|
func (r *evRec) fn(_, et, sev, _, _, _ string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.evs = append(r.evs, et+"/"+sev)
|
|
}
|
|
func (r *evRec) take() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
out := r.evs
|
|
r.evs = nil
|
|
return out
|
|
}
|
|
|
|
// The consequence: an agent restart of a controller reaches the dispatcher as
|
|
// controller_restarted_by_agent, once; a crash-loop reaches it as controller_crashloop (error).
|
|
//
|
|
// RED-PROOF: make observe() skip the `LastRestartAt != prev.lastRestartAt` branch → the second
|
|
// Check emits nothing → "restart did not produce controller_restarted_by_agent".
|
|
func TestControllerSupervisorChecker_EmitsOnMovement(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
|
|
// Seed: an old restart already recorded at first sight → silent.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("first observation must seed silently, got %v", evs)
|
|
}
|
|
// Same record again → silent.
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("unchanged record emitted %v", evs)
|
|
}
|
|
// A new restart.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-09-15T09:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
|
t.Fatalf("restart did not produce controller_restarted_by_agent (got %v)", evs)
|
|
}
|
|
// Agent restarted (counter back to 1) with a NEW timestamp → still an event.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"absent","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
|
t.Fatalf("a restart after an agent restart (counter reset) was lost: %v", evs)
|
|
}
|
|
// Crash-loop.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"exited","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z","parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" {
|
|
t.Fatalf("crash-loop did not produce controller_crashloop/error (got %v)", evs)
|
|
}
|
|
// A pre-v0.131.0 report (no stanza) → nothing.
|
|
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("a report without the stanza emitted %v", evs)
|
|
}
|
|
}
|
|
|
|
// A hub restarted DURING a crash-loop must say so once, not seed it away.
|
|
func TestControllerSupervisorChecker_CrashloopAtFirstSightEmits(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":3,"last_restart_at":"2026-09-15T10:00:00Z","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" {
|
|
t.Fatalf("crash-loop at first sight: want exactly one controller_crashloop, got %v", evs)
|
|
}
|
|
}
|
|
|
|
// R-539 (operator ruling 3, 2026-09-16; agent v0.132.0). A SLOW crash loop — restarts spread wider than
|
|
// the 15-minute brake — is reported by the agent as slow_crashloop_since. The consequence: when that
|
|
// timestamp MOVES, the dispatcher receives controller_slow_crashloop at WARNING; a guest that never
|
|
// sets it never produces one.
|
|
//
|
|
// RED-PROOF: delete the SlowCrashloopSince movement branch in observe() → "slow crash loop produced no
|
|
// controller_slow_crashloop".
|
|
func TestControllerSupervisorChecker_SlowCrashloop(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-17T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":4}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("seed must be silent, got %v", evs)
|
|
}
|
|
// Fifth restart inside 24 h: last_restart_at moves AND slow_crashloop_since appears.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
evs := rec.take()
|
|
found := false
|
|
for _, e := range evs {
|
|
if e == "controller_slow_crashloop/warning" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("slow crash loop produced no controller_slow_crashloop/warning; got %v", evs)
|
|
}
|
|
// Same timestamp on the next sweep → no second event.
|
|
c.Check()
|
|
for _, e := range rec.take() {
|
|
if e == "controller_slow_crashloop/warning" {
|
|
t.Fatalf("an unmoved slow_crashloop_since must not re-emit")
|
|
}
|
|
}
|
|
}
|
|
|
|
// A hub restarted while a guest is in a slow crash loop must not stay silent (the F2 rule the fast
|
|
// crash loop already follows): first sight with slow_crashloop=true emits once.
|
|
func TestControllerSupervisorChecker_SlowCrashloopAtFirstSight(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
evs := rec.take()
|
|
if len(evs) != 1 || evs[0] != "controller_slow_crashloop/warning" {
|
|
t.Fatalf("first sight in a slow crash loop: got %v, want exactly [controller_slow_crashloop/warning]", evs)
|
|
}
|
|
}
|
|
|
|
// R-1344 (operator brief 2026-10-29, Part 2 — ALARM ONLY until the measurement licenses a restart): a controller that
|
|
// runs but does not answer reaches the operator once per new unresponsive_since, as controller_unresponsive (warning),
|
|
// also at first sight; an answering one never. RED-PROOF: seen failing before the branch existed.
|
|
func TestControllerSupervisorChecker_Unresponsive(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("an answering controller emitted %v", evs)
|
|
}
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-29T13:00:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" {
|
|
t.Fatalf("got %v, want exactly [controller_unresponsive/warning]", evs)
|
|
}
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("the same unresponsive_since emitted again: %v", evs)
|
|
}
|
|
// a hub restarted while it is unresponsive says so once
|
|
c2 := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
c2.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" {
|
|
t.Fatalf("first sight while unresponsive: %v", evs)
|
|
}
|
|
}
|
|
|
|
// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as
|
|
// controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at
|
|
// moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing
|
|
// before the branch existed.
|
|
func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
rec.take() // first sight while unresponsive: the alarm (pinned above)
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
|
|
t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
|
|
}
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("the same restart emitted again: %v", evs)
|
|
}
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
|
t.Fatalf("a not-running restart after it: %v", evs)
|
|
}
|
|
}
|
|
|
|
// The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md
|
|
// corollary: when the behaviour changes, the alarm text changes with it).
|
|
func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) {
|
|
for _, tc := range []struct{ field, want, never string }{
|
|
{`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"},
|
|
{``, "does NOT restart it", "restarts it if"},
|
|
} {
|
|
st := newCapStore(t)
|
|
var msg string
|
|
c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) {
|
|
t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never)
|
|
}
|
|
}
|
|
}
|