admin
37ae31fd44
gates / gates (push) Successful in 21s
R-549 (operator ruling A): controllerStatus hardcoded 30m/1h while both staleness checkers and hostStatus read alerting.stale_threshold. Moving the threshold to 45m would have painted a customer amber 15 minutes before the alarm could fire - the second definition rollup.go's header forbids. It now reads the same value, down at 2x. Both 'checker initialized' log lines print the threshold, which no line did before. R-539 (ruling 3 of 2026-09-16): controller_slow_crashloop (warning, operator-only), minted when the agent's slow_crashloop_since moves, with the fast sibling's first-sight rule. R-550: restore_interrupted (warning, for the household) allowlisted with a Hungarian customer message. Red-proofs, each seen failing then passing: the status test with the old hardcoded numbers; the checker test with the movement branch removed; the operator-only test with the registration removed; the household-message test with the Hungarian entry removed (asserted on the SUBJECT - the body legitimately repeats the raw message, which my first version of the test mistook for a fallback). go build/vet/test ./... green, 18 packages. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
149 lines
6.6 KiB
Go
149 lines
6.6 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// R-523 — the checker that turns the agent's controller_supervisor stanza into events.
|
|
// The JSON below is the exact wire shape the agent's TestControllerSupervisorStanza_WireShape pins.
|
|
|
|
func supReport(guests string) []byte {
|
|
return []byte(`{"host_id":"h1","controller_supervisor":{"guests":` + guests + `}}`)
|
|
}
|
|
|
|
type evRec struct {
|
|
mu sync.Mutex
|
|
evs []string
|
|
}
|
|
|
|
func (r *evRec) fn(_, et, sev, _, _, _ string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.evs = append(r.evs, et+"/"+sev)
|
|
}
|
|
func (r *evRec) take() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
out := r.evs
|
|
r.evs = nil
|
|
return out
|
|
}
|
|
|
|
// The consequence: an agent restart of a controller reaches the dispatcher as
|
|
// controller_restarted_by_agent, once; a crash-loop reaches it as controller_crashloop (error).
|
|
//
|
|
// RED-PROOF: make observe() skip the `LastRestartAt != prev.lastRestartAt` branch → the second
|
|
// Check emits nothing → "restart did not produce controller_restarted_by_agent".
|
|
func TestControllerSupervisorChecker_EmitsOnMovement(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
|
|
// Seed: an old restart already recorded at first sight → silent.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("first observation must seed silently, got %v", evs)
|
|
}
|
|
// Same record again → silent.
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("unchanged record emitted %v", evs)
|
|
}
|
|
// A new restart.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-09-15T09:00:00Z","last_reason":"exited","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
|
t.Fatalf("restart did not produce controller_restarted_by_agent (got %v)", evs)
|
|
}
|
|
// Agent restarted (counter back to 1) with a NEW timestamp → still an event.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"absent","crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
|
|
t.Fatalf("a restart after an agent restart (counter reset) was lost: %v", evs)
|
|
}
|
|
// Crash-loop.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-15T10:00:00Z","last_reason":"exited","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z","parked":false}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" {
|
|
t.Fatalf("crash-loop did not produce controller_crashloop/error (got %v)", evs)
|
|
}
|
|
// A pre-v0.131.0 report (no stanza) → nothing.
|
|
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("a report without the stanza emitted %v", evs)
|
|
}
|
|
}
|
|
|
|
// A hub restarted DURING a crash-loop must say so once, not seed it away.
|
|
func TestControllerSupervisorChecker_CrashloopAtFirstSightEmits(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":3,"last_restart_at":"2026-09-15T10:00:00Z","crashloop":true,"crashloop_since":"2026-09-15T10:05:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_crashloop/error" {
|
|
t.Fatalf("crash-loop at first sight: want exactly one controller_crashloop, got %v", evs)
|
|
}
|
|
}
|
|
|
|
// R-539 (operator ruling 3, 2026-09-16; agent v0.132.0). A SLOW crash loop — restarts spread wider than
|
|
// the 15-minute brake — is reported by the agent as slow_crashloop_since. The consequence: when that
|
|
// timestamp MOVES, the dispatcher receives controller_slow_crashloop at WARNING; a guest that never
|
|
// sets it never produces one.
|
|
//
|
|
// RED-PROOF: delete the SlowCrashloopSince movement branch in observe() → "slow crash loop produced no
|
|
// controller_slow_crashloop".
|
|
func TestControllerSupervisorChecker_SlowCrashloop(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":4,"last_restart_at":"2026-09-17T08:00:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":4}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
if evs := rec.take(); len(evs) != 0 {
|
|
t.Fatalf("seed must be silent, got %v", evs)
|
|
}
|
|
// Fifth restart inside 24 h: last_restart_at moves AND slow_crashloop_since appears.
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
evs := rec.take()
|
|
found := false
|
|
for _, e := range evs {
|
|
if e == "controller_slow_crashloop/warning" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("slow crash loop produced no controller_slow_crashloop/warning; got %v", evs)
|
|
}
|
|
// Same timestamp on the next sweep → no second event.
|
|
c.Check()
|
|
for _, e := range rec.take() {
|
|
if e == "controller_slow_crashloop/warning" {
|
|
t.Fatalf("an unmoved slow_crashloop_since must not re-emit")
|
|
}
|
|
}
|
|
}
|
|
|
|
// A hub restarted while a guest is in a slow crash loop must not stay silent (the F2 rule the fast
|
|
// crash loop already follows): first sight with slow_crashloop=true emits once.
|
|
func TestControllerSupervisorChecker_SlowCrashloopAtFirstSight(t *testing.T) {
|
|
st := newCapStore(t)
|
|
rec := &evRec{}
|
|
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":5,"last_restart_at":"2026-09-17T08:20:00Z","last_reason":"exited","crashloop":false,"parked":false,"restarts_24h":5,"slow_crashloop":true,"slow_crashloop_since":"2026-09-17T08:20:00Z"}]`), store.HostReportDenorm{})
|
|
c.Check()
|
|
evs := rec.take()
|
|
if len(evs) != 1 || evs[0] != "controller_slow_crashloop/warning" {
|
|
t.Fatalf("first sight in a slow crash loop: got %v, want exactly [controller_slow_crashloop/warning]", evs)
|
|
}
|
|
}
|