feat(hub): OOB access health ingest + degraded alert (H1 Part 4)
store.GetHostOOBStates parses the agent oob heartbeat stanza. monitor/host_oob: transition-based oob_degraded/oob_recovered warning (felhom-sshd down while the operator peer is configured, OR config invalid) — proactive "can the operator get in right now" signal; unconfigured OOB never alerts. Wired into the 60s sweep. Non-hollow tests + transitions. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// HostOOBChecker raises an operator WARNING when a host's OOB access path is DEGRADED — felhom-sshd
|
||||
// down (while the operator peer is configured, i.e. OOB is meant to work) OR its config is invalid.
|
||||
// It answers "can the operator get into this box right now, and if not, why" proactively, from the
|
||||
// hub. Transition-based (ok↔degraded, one event per transition — the HostCapabilityChecker shape), so
|
||||
// a persistent problem alerts ONCE, not every 60s sweep, and a recovery is noted.
|
||||
//
|
||||
// A host with no oob stanza (pre-H1 / feature off) is never evaluated. A degraded state requires the
|
||||
// operator peer to be configured — a box where OOB was never set up is not "broken".
|
||||
type HostOOBChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
onEvent EventNotifyFunc
|
||||
|
||||
mu sync.Mutex
|
||||
degraded map[string]bool // hostID → currently-degraded
|
||||
customerOf map[string]string
|
||||
}
|
||||
|
||||
// NewHostOOBChecker seeds per-host degraded state from the latest reports WITHOUT alerting (a problem
|
||||
// present at startup alerts on the first transition-in evaluated after seed = never re-alerts a
|
||||
// steady bad state; matches HostCapabilityChecker). Actually seeds silent, then Check transitions.
|
||||
func NewHostOOBChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostOOBChecker {
|
||||
c := &HostOOBChecker{
|
||||
store: s,
|
||||
logger: logger,
|
||||
onEvent: onEvent,
|
||||
degraded: make(map[string]bool),
|
||||
customerOf: make(map[string]string),
|
||||
}
|
||||
rows, err := s.GetHostOOBStates()
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] Host OOB checker: failed to seed: %v", err)
|
||||
return c
|
||||
}
|
||||
seeded := 0
|
||||
for _, row := range rows {
|
||||
if s.IsCustomerBlocked(row.CustomerID) || !row.Present {
|
||||
continue
|
||||
}
|
||||
c.customerOf[row.HostID] = row.CustomerID
|
||||
if oobDegraded(row) {
|
||||
c.degraded[row.HostID] = true // seed the bad state so we don't re-alert it on cycle 1
|
||||
seeded++
|
||||
}
|
||||
}
|
||||
logger.Printf("[INFO] Host OOB checker initialized: %d host(s) seeded degraded", seeded)
|
||||
return c
|
||||
}
|
||||
|
||||
// oobDegraded is the degraded predicate: config invalid, OR (OOB meant to work — operator peer
|
||||
// configured — AND felhom-sshd is not active/reachable).
|
||||
func oobDegraded(r store.HostOOBRow) bool {
|
||||
if !r.Present {
|
||||
return false
|
||||
}
|
||||
if r.ConfigInvalid {
|
||||
return true
|
||||
}
|
||||
if r.OperatorPeerConfigured && (!r.FelhomSshdActive || !r.Reachable) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Check evaluates all hosts and emits oob_degraded / oob_recovered on transitions.
|
||||
func (c *HostOOBChecker) Check() {
|
||||
rows, err := c.store.GetHostOOBStates()
|
||||
if err != nil {
|
||||
c.logger.Printf("[WARN] Host OOB check failed: %v", err)
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
seen := make(map[string]bool, len(rows))
|
||||
for _, row := range rows {
|
||||
if c.store.IsCustomerBlocked(row.CustomerID) {
|
||||
delete(c.degraded, row.HostID)
|
||||
continue
|
||||
}
|
||||
if !row.Present {
|
||||
continue // no oob stanza → not evaluated
|
||||
}
|
||||
seen[row.HostID] = true
|
||||
c.customerOf[row.HostID] = row.CustomerID
|
||||
bad := oobDegraded(row)
|
||||
was := c.degraded[row.HostID]
|
||||
switch {
|
||||
case bad && !was:
|
||||
c.degraded[row.HostID] = true
|
||||
c.emit(row, "oob_degraded", "warning")
|
||||
case !bad && was:
|
||||
delete(c.degraded, row.HostID)
|
||||
c.emit(row, "oob_recovered", "info")
|
||||
}
|
||||
}
|
||||
for id := range c.degraded {
|
||||
if !seen[id] {
|
||||
delete(c.degraded, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IsDegraded reports the current tracked state for a host (test/UI helper).
|
||||
func (c *HostOOBChecker) IsDegraded(hostID string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.degraded[hostID]
|
||||
}
|
||||
|
||||
func (c *HostOOBChecker) emit(row store.HostOOBRow, eventType, severity string) {
|
||||
var msg string
|
||||
if eventType == "oob_degraded" {
|
||||
reason := "felhom-sshd unreachable"
|
||||
if row.ConfigInvalid {
|
||||
reason = "felhom-sshd config invalid (sshd -t fails)"
|
||||
}
|
||||
msg = "Host " + row.HostID + ": OPERATOR ACCESS DEGRADED — " + reason +
|
||||
". The break-glass net (auto-heal + vaulted root@pam console) is still under the box."
|
||||
} else {
|
||||
msg = "Host " + row.HostID + ": operator access recovered (felhom-sshd reachable again)."
|
||||
}
|
||||
details, _ := json.Marshal(map[string]any{
|
||||
"host_id": row.HostID,
|
||||
"felhom_sshd_port": row.FelhomSshdPort,
|
||||
"active": row.FelhomSshdActive,
|
||||
"reachable": row.Reachable,
|
||||
"config_invalid": row.ConfigInvalid,
|
||||
})
|
||||
c.logger.Printf("[%s] Host OOB: %s (%s)", map[string]string{"warning": "WARN", "info": "INFO"}[severity], row.HostID, eventType)
|
||||
if _, err := c.store.SaveEvent(row.CustomerID, eventType, severity, msg, string(details), "hub"); err != nil {
|
||||
c.logger.Printf("[WARN] save %s for %s: %v", eventType, row.HostID, err)
|
||||
return
|
||||
}
|
||||
if c.onEvent != nil {
|
||||
c.onEvent(row.CustomerID, eventType, severity, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func oobReport(active, reachable, configInvalid, operatorConfigured bool) []byte {
|
||||
b := func(v bool) string {
|
||||
if v {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
}
|
||||
return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":` + b(active) +
|
||||
`,"felhom_sshd_port":8822,"reachable":` + b(reachable) +
|
||||
`,"config_invalid":` + b(configInvalid) +
|
||||
`,"operator_peer_configured":` + b(operatorConfigured) + `}}`)
|
||||
}
|
||||
|
||||
func newOOBStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
||||
return st
|
||||
}
|
||||
|
||||
func TestHostOOB_DegradedThenRecoveredTransitions(t *testing.T) {
|
||||
st := newOOBStore(t)
|
||||
// healthy at construction (active+reachable, operator configured)
|
||||
st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
|
||||
var events []string
|
||||
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
if c.IsDegraded("h1") {
|
||||
t.Fatal("healthy host seeded degraded")
|
||||
}
|
||||
|
||||
// felhom-sshd goes DOWN with operator configured → one oob_degraded
|
||||
st.SaveHostReport("h1", "c1", oobReport(false, false, false, true), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if len(events) != 1 || events[0] != "oob_degraded" {
|
||||
t.Fatalf("down+operator-configured → one oob_degraded, got %v", events)
|
||||
}
|
||||
c.Check() // persistent → no re-alert
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("persistent degraded must not re-emit, got %v", events)
|
||||
}
|
||||
|
||||
// recovers → oob_recovered
|
||||
st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if len(events) != 2 || events[1] != "oob_recovered" {
|
||||
t.Fatalf("recovery → oob_recovered, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostOOB_ConfigInvalidAlerts(t *testing.T) {
|
||||
st := newOOBStore(t)
|
||||
st.SaveHostReport("h1", "c1", oobReport(true, true, false, false), store.HostReportDenorm{})
|
||||
var events []string
|
||||
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
// config invalid (even without operator configured) → degraded
|
||||
st.SaveHostReport("h1", "c1", oobReport(true, true, true, false), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if len(events) != 1 || events[0] != "oob_degraded" {
|
||||
t.Fatalf("config_invalid → oob_degraded, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// A box where OOB was NEVER set up (no operator peer) with felhom-sshd down must NOT alert — it's not
|
||||
// broken, it's unconfigured.
|
||||
func TestHostOOB_DownButNoOperatorNotDegraded(t *testing.T) {
|
||||
st := newOOBStore(t)
|
||||
st.SaveHostReport("h1", "c1", oobReport(false, false, false, false), store.HostReportDenorm{})
|
||||
var events []string
|
||||
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
c.Check()
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("unconfigured OOB (no operator peer) must not alert, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// A report with no oob stanza (pre-H1 agent) is never evaluated.
|
||||
func TestHostOOB_NoStanzaIgnored(t *testing.T) {
|
||||
st := newOOBStore(t)
|
||||
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
||||
var events []string
|
||||
c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
c.Check()
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("no oob stanza must not alert, got %v", events)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user