package monitor import ( "io" "log" "strings" "sync" "testing" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-523 — the checker that turns the agent's controller_supervisor stanza into events. // The JSON below is the exact wire shape the agent's TestControllerSupervisorStanza_WireShape pins. func supReport(guests string) []byte { return []byte(`{"host_id":"h1","controller_supervisor":{"guests":` + guests + `}}`) } type evRec struct { mu sync.Mutex evs []string } func (r *evRec) fn(_, et, sev, _, _, _ string) { r.mu.Lock() defer r.mu.Unlock() r.evs = append(r.evs, et+"/"+sev) } func (r *evRec) take() []string { r.mu.Lock() defer r.mu.Unlock() out := r.evs r.evs = nil return out } // The consequence: an agent restart of a controller reaches the dispatcher as // controller_restarted_by_agent, once; a crash-loop reaches it as controller_crashloop (error). // // RED-PROOF: make observe() skip the `LastRestartAt != prev.lastRestartAt` branch → the second // Check emits nothing → "restart did not produce controller_restarted_by_agent". func TestControllerSupervisorChecker_EmitsOnMovement(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) // Seed: an old restart already recorded at first sight → silent. st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("first observation must seed silently, got %v", evs) } // Same record again → silent. c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("unchanged record emitted %v", evs) } // A new restart. st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-09-15T09:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" { t.Fatalf("restart did not produce controller_restarted_by_agent (got %v)", evs) } // Agent restarted (counter back to 1) with a NEW timestamp → still an event. st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"absent","crashloop":false,"parked":false}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" { t.Fatalf("a restart after an agent restart (counter reset) was lost: %v", evs) } // Crash-loop. st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"exited","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z","parked":false}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" { t.Fatalf("crash-loop did not produce controller_crashloop/error (got %v)", evs) } // A pre-v0.131.0 report (no stanza) → nothing. st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("a report without the stanza emitted %v", evs) } } // A hub restarted DURING a crash-loop must say so once, not seed it away. func TestControllerSupervisorChecker_CrashloopAtFirstSightEmits(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":3,"last_restart_at":"2026-09-15T10:00:00Z","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z"}]`), store.HostReportDenorm{}) c.Check() c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" { t.Fatalf("crash-loop at first sight: want exactly one controller_crashloop, got %v", evs) } } // R-539 (operator ruling 3, 2026-09-16; agent v0.132.0). A SLOW crash loop — restarts spread wider than // the 15-minute brake — is reported by the agent as slow_crashloop_since. The consequence: when that // timestamp MOVES, the dispatcher receives controller_slow_crashloop at WARNING; a guest that never // sets it never produces one. // // RED-PROOF: delete the SlowCrashloopSince movement branch in observe() → "slow crash loop produced no // controller_slow_crashloop". func TestControllerSupervisorChecker_SlowCrashloop(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-17T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":4}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("seed must be silent, got %v", evs) } // Fifth restart inside 24 h: last_restart_at moves AND slow_crashloop_since appears. st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{}) c.Check() evs := rec.take() found := false for _, e := range evs { if e == "controller_slow_crashloop/warning" { found = true } } if !found { t.Fatalf("slow crash loop produced no controller_slow_crashloop/warning; got %v", evs) } // Same timestamp on the next sweep → no second event. c.Check() for _, e := range rec.take() { if e == "controller_slow_crashloop/warning" { t.Fatalf("an unmoved slow_crashloop_since must not re-emit") } } } // A hub restarted while a guest is in a slow crash loop must not stay silent (the F2 rule the fast // crash loop already follows): first sight with slow_crashloop=true emits once. func TestControllerSupervisorChecker_SlowCrashloopAtFirstSight(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{}) c.Check() evs := rec.take() if len(evs) != 1 || evs[0] != "controller_slow_crashloop/warning" { t.Fatalf("first sight in a slow crash loop: got %v, want exactly [controller_slow_crashloop/warning]", evs) } } // R-1344 (operator brief 2026-10-29, Part 2 — ALARM ONLY until the measurement licenses a restart): a controller that // runs but does not answer reaches the operator once per new unresponsive_since, as controller_unresponsive (warning), // also at first sight; an answering one never. RED-PROOF: seen failing before the branch existed. func TestControllerSupervisorChecker_Unresponsive(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":false}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("an answering controller emitted %v", evs) } st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-29T13:00:00Z"}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" { t.Fatalf("got %v, want exactly [controller_unresponsive/warning]", evs) } c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("the same unresponsive_since emitted again: %v", evs) } // a hub restarted while it is unresponsive says so once c2 := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) c2.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" { t.Fatalf("first sight while unresponsive: %v", evs) } } // R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as // controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at // moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing // before the branch existed. func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) { st := newCapStore(t) rec := &evRec{} c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{}) c.Check() rec.take() // first sight while unresponsive: the alarm (pinned above) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" { t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs) } c.Check() if evs := rec.take(); len(evs) != 0 { t.Fatalf("the same restart emitted again: %v", evs) } st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{}) c.Check() if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" { t.Fatalf("a not-running restart after it: %v", evs) } } // The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md // corollary: when the behaviour changes, the alarm text changes with it). func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) { for _, tc := range []struct{ field, want, never string }{ {`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"}, {``, "does NOT restart it", "restarts it if"}, } { st := newCapStore(t) var msg string c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0)) st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{}) c.Check() if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) { t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never) } } }