0ff1d3c883
ProcessEvent routed only warning/error; a critical-severity event was silently dropped. Now routes warning/error/critical, logs unrecognized severities, and guards a nil GetNotificationPrefs (which would panic/crash the hub). host_disk_critical emits its natural critical severity. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs
183 lines
6.4 KiB
Go
183 lines
6.4 KiB
Go
package monitor
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func newDiskStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
|
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
|
return st
|
|
}
|
|
|
|
// saveDiskReport records a host-report carrying the host root disk_percent (denorm column) + total/used.
|
|
func saveDiskReport(t *testing.T, st *store.Store, pct float64) {
|
|
t.Helper()
|
|
body := fmt.Sprintf(`{"host_id":"h1","host":{"disk_total_bytes":100000000000,"disk_used_bytes":%d}}`, int64(pct*1e9))
|
|
if err := st.SaveHostReport("h1", "c1", []byte(body), store.HostReportDenorm{DiskPercent: pct}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func quietLog() *log.Logger { return log.New(io.Discard, "", 0) }
|
|
|
|
// TestHostDiskChecker_Bands covers the band contract (§7-A/C, §10): seed ok (no event), ok→warning,
|
|
// warning→critical (escalation), steady (no re-emit), recovery clears + re-arms, and the emitted type is
|
|
// host_disk_* (distinct from the guest disk_*).
|
|
func TestHostDiskChecker_Bands(t *testing.T) {
|
|
st := newDiskStore(t)
|
|
saveDiskReport(t, st, 80) // ok at init
|
|
|
|
var events []string
|
|
onEvent := func(_, eventType, _, _, _, _ string) { events = append(events, eventType) }
|
|
dc := NewHostDiskChecker(st, 90, 95, onEvent, quietLog())
|
|
|
|
if dc.GetState("h1") != "ok" {
|
|
t.Fatalf("seed state = %s, want ok", dc.GetState("h1"))
|
|
}
|
|
if len(events) != 0 {
|
|
t.Fatalf("seed must not emit, got %v", events)
|
|
}
|
|
|
|
// 80 → nothing.
|
|
saveDiskReport(t, st, 80)
|
|
dc.Check()
|
|
if len(events) != 0 {
|
|
t.Fatalf("80%% must not alert, got %v", events)
|
|
}
|
|
|
|
// ok → 91 → warning (one event).
|
|
saveDiskReport(t, st, 91)
|
|
dc.Check()
|
|
if dc.GetState("h1") != bandWarning {
|
|
t.Fatalf("state = %s, want warning", dc.GetState("h1"))
|
|
}
|
|
if len(events) != 1 || events[0] != "host_disk_warning" {
|
|
t.Fatalf("want one host_disk_warning, got %v", events)
|
|
}
|
|
|
|
// steady warning: no re-emit.
|
|
saveDiskReport(t, st, 92)
|
|
dc.Check()
|
|
if len(events) != 1 {
|
|
t.Fatalf("steady warning must not re-emit, got %v", events)
|
|
}
|
|
|
|
// warning → 96 → critical (escalation).
|
|
saveDiskReport(t, st, 96)
|
|
dc.Check()
|
|
if dc.GetState("h1") != bandCritical {
|
|
t.Fatalf("state = %s, want critical", dc.GetState("h1"))
|
|
}
|
|
if len(events) != 2 || events[1] != "host_disk_critical" {
|
|
t.Fatalf("want host_disk_critical escalation, got %v", events)
|
|
}
|
|
|
|
// recovery: 80 clears + re-arms (no recovery event, state back to ok).
|
|
saveDiskReport(t, st, 80)
|
|
dc.Check()
|
|
if dc.GetState("h1") != bandOK {
|
|
t.Fatalf("state = %s, want ok after recovery", dc.GetState("h1"))
|
|
}
|
|
if len(events) != 2 {
|
|
t.Fatalf("recovery must not emit an event, got %v", events)
|
|
}
|
|
|
|
// re-armed: a fresh 91 breach alerts again.
|
|
saveDiskReport(t, st, 91)
|
|
dc.Check()
|
|
if len(events) != 3 || events[2] != "host_disk_warning" {
|
|
t.Fatalf("re-arm: a new breach must alert again, got %v", events)
|
|
}
|
|
}
|
|
|
|
// TestHostDiskChecker_Severity asserts the band→severity mapping the dispatcher requires (it only routes
|
|
// warning/error): warning band → "warning", critical band → "error" (NOT "critical", which would be dropped).
|
|
func TestHostDiskChecker_Severity(t *testing.T) {
|
|
st := newDiskStore(t)
|
|
saveDiskReport(t, st, 50)
|
|
var sev []string
|
|
onEvent := func(_, _, severity, _, _, _ string) { sev = append(sev, severity) }
|
|
dc := NewHostDiskChecker(st, 90, 95, onEvent, quietLog())
|
|
|
|
saveDiskReport(t, st, 92)
|
|
dc.Check()
|
|
saveDiskReport(t, st, 97)
|
|
dc.Check()
|
|
if len(sev) != 2 || sev[0] != "warning" || sev[1] != "critical" {
|
|
t.Fatalf("severities = %v, want [warning critical] (hub v0.24.0 routes the natural critical severity)", sev)
|
|
}
|
|
}
|
|
|
|
// TestHostDiskChecker_BornPersistent is the F2 lesson (§7-B): a disk ALREADY over the critical threshold
|
|
// when the checker (re)starts must alert on the FIRST Check — there is no crossing to observe.
|
|
func TestHostDiskChecker_BornPersistent(t *testing.T) {
|
|
st := newDiskStore(t)
|
|
saveDiskReport(t, st, 97) // already critical at init
|
|
|
|
var events []string
|
|
onEvent := func(_, eventType, _, _, _, _ string) { events = append(events, eventType) }
|
|
dc := NewHostDiskChecker(st, 90, 95, onEvent, quietLog())
|
|
|
|
// Seed must have left the breached host UNSEEDED (so the first Check emits).
|
|
if dc.GetState("h1") != "unknown" {
|
|
t.Fatalf("an already-breached host must be left unseeded at init, state = %s", dc.GetState("h1"))
|
|
}
|
|
if len(events) != 0 {
|
|
t.Fatalf("init must not emit directly, got %v", events)
|
|
}
|
|
|
|
dc.Check()
|
|
if len(events) != 1 || events[0] != "host_disk_critical" {
|
|
t.Fatalf("born-persistent: first Check MUST emit host_disk_critical, got %v", events)
|
|
}
|
|
|
|
// COMPANION RED-PROOF: a transition-only design seeds ALL hosts at init (including breached ones), so
|
|
// the breached host's state == current band at first Check → no transition → SILENT forever. We model
|
|
// that by pre-seeding the state to critical (as a seed-all impl would) on a FRESH checker, then Check.
|
|
st2 := newDiskStore(t)
|
|
saveDiskReport(t, st2, 97)
|
|
var silent []string
|
|
dc2 := NewHostDiskChecker(st2, 90, 95, func(_, et, _, _, _, _ string) { silent = append(silent, et) }, quietLog())
|
|
dc2.mu.Lock()
|
|
dc2.states["h1"] = bandCritical // the WRONG (seed-all) design's seed
|
|
dc2.mu.Unlock()
|
|
dc2.Check()
|
|
if len(silent) != 0 {
|
|
t.Fatalf("control: a seed-all (transition-only) impl should be silent on the born-breach, got %v", silent)
|
|
}
|
|
// → The real checker (which leaves breached hosts unseeded) emitted; the transition-only model stayed
|
|
// silent. That gap is the bug this design fixes.
|
|
}
|
|
|
|
// TestHostDiskChecker_ThresholdDefaults: an unset/invalid threshold config falls back to 90/95, and a
|
|
// misordered (crit ≤ warn) config does too — a typo can never silence or invert the alert.
|
|
func TestHostDiskChecker_ThresholdDefaults(t *testing.T) {
|
|
cases := []struct{ warn, crit, wantWarn, wantCrit float64 }{
|
|
{0, 0, 90, 95},
|
|
{85, 0, 85, 95},
|
|
{0, 98, 90, 98},
|
|
{95, 90, 90, 95}, // misordered → defaults
|
|
{-5, 200, 90, 95}, // out of range → defaults
|
|
}
|
|
for _, c := range cases {
|
|
w, cr := normalizeDiskThresholds(c.warn, c.crit)
|
|
if w != c.wantWarn || cr != c.wantCrit {
|
|
t.Errorf("normalizeDiskThresholds(%v,%v) = (%v,%v), want (%v,%v)", c.warn, c.crit, w, cr, c.wantWarn, c.wantCrit)
|
|
}
|
|
}
|
|
}
|