admin
f8fc09e5cc
store.GetHostOOBStates parses the agent oob heartbeat stanza. monitor/host_oob: transition-based oob_degraded/oob_recovered warning (felhom-sshd down while the operator peer is configured, OR config invalid) — proactive "can the operator get in right now" signal; unconfigured OOB never alerts. Wired into the 60s sweep. Non-hollow tests + transitions. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
104 lines
3.7 KiB
Go
104 lines
3.7 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func oobReport(active, reachable, configInvalid, operatorConfigured bool) []byte {
|
|
b := func(v bool) string {
|
|
if v {
|
|
return "true"
|
|
}
|
|
return "false"
|
|
}
|
|
return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":` + b(active) +
|
|
`,"felhom_sshd_port":8822,"reachable":` + b(reachable) +
|
|
`,"config_invalid":` + b(configInvalid) +
|
|
`,"operator_peer_configured":` + b(operatorConfigured) + `}}`)
|
|
}
|
|
|
|
func newOOBStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
|
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
|
return st
|
|
}
|
|
|
|
func TestHostOOB_DegradedThenRecoveredTransitions(t *testing.T) {
|
|
st := newOOBStore(t)
|
|
// healthy at construction (active+reachable, operator configured)
|
|
st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
|
|
var events []string
|
|
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
if c.IsDegraded("h1") {
|
|
t.Fatal("healthy host seeded degraded")
|
|
}
|
|
|
|
// felhom-sshd goes DOWN with operator configured → one oob_degraded
|
|
st.SaveHostReport("h1", "c1", oobReport(false, false, false, true), store.HostReportDenorm{})
|
|
c.Check()
|
|
if len(events) != 1 || events[0] != "oob_degraded" {
|
|
t.Fatalf("down+operator-configured → one oob_degraded, got %v", events)
|
|
}
|
|
c.Check() // persistent → no re-alert
|
|
if len(events) != 1 {
|
|
t.Fatalf("persistent degraded must not re-emit, got %v", events)
|
|
}
|
|
|
|
// recovers → oob_recovered
|
|
st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
|
|
c.Check()
|
|
if len(events) != 2 || events[1] != "oob_recovered" {
|
|
t.Fatalf("recovery → oob_recovered, got %v", events)
|
|
}
|
|
}
|
|
|
|
func TestHostOOB_ConfigInvalidAlerts(t *testing.T) {
|
|
st := newOOBStore(t)
|
|
st.SaveHostReport("h1", "c1", oobReport(true, true, false, false), store.HostReportDenorm{})
|
|
var events []string
|
|
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
// config invalid (even without operator configured) → degraded
|
|
st.SaveHostReport("h1", "c1", oobReport(true, true, true, false), store.HostReportDenorm{})
|
|
c.Check()
|
|
if len(events) != 1 || events[0] != "oob_degraded" {
|
|
t.Fatalf("config_invalid → oob_degraded, got %v", events)
|
|
}
|
|
}
|
|
|
|
// A box where OOB was NEVER set up (no operator peer) with felhom-sshd down must NOT alert — it's not
|
|
// broken, it's unconfigured.
|
|
func TestHostOOB_DownButNoOperatorNotDegraded(t *testing.T) {
|
|
st := newOOBStore(t)
|
|
st.SaveHostReport("h1", "c1", oobReport(false, false, false, false), store.HostReportDenorm{})
|
|
var events []string
|
|
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
c.Check()
|
|
if len(events) != 0 {
|
|
t.Fatalf("unconfigured OOB (no operator peer) must not alert, got %v", events)
|
|
}
|
|
}
|
|
|
|
// A report with no oob stanza (pre-H1 agent) is never evaluated.
|
|
func TestHostOOB_NoStanzaIgnored(t *testing.T) {
|
|
st := newOOBStore(t)
|
|
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
|
var events []string
|
|
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
c.Check()
|
|
if len(events) != 0 {
|
|
t.Fatalf("no oob stanza must not alert, got %v", events)
|
|
}
|
|
}
|