From b7b165bff5f10ff066482578382fe48abae15ba3 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 29 Jun 2026 18:50:50 +0200 Subject: [PATCH] =?UTF-8?q?hub:=20HostCapabilityChecker=20=E2=80=94=20oper?= =?UTF-8?q?ator=20alert=20on=20agent=20capability-degraded=20(v0.19.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to felhom-agent v0.44.0. New monitor.HostCapabilityChecker (sibling of HostStalenessChecker) reads the capabilities snapshot from the latest host report and emits agent_capability_degraded/recovered (operator-only, 1h cooldown) on ok<->degraded transitions for any Critical capability. store.GetHostCapabilities (MAX(id), no migration). Goldens mirror the new capabilities field. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt --- hub/CHANGELOG.md | 24 +++ hub/cmd/hub/main.go | 4 + .../api/testdata/host-report.golden.json | 1 + hub/internal/monitor/host_capability.go | 178 ++++++++++++++++++ hub/internal/monitor/host_capability_test.go | 107 +++++++++++ hub/internal/store/store.go | 49 +++++ 6 files changed, 363 insertions(+) create mode 100644 hub/internal/monitor/host_capability.go create mode 100644 hub/internal/monitor/host_capability_test.go diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 8cfc5a7..226cdfc 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,29 @@ # Felhom Hub — Changelog +## v0.19.0 — Agent capability-degraded operator alert (HostCapabilityChecker) (2026-06-29) + +Companion to felhom-agent v0.44.0's privileged-capability self-probe: the agent now rides a +`capabilities` snapshot on its host report (each required `sudo -n` grant: ok/degraded), and the hub +alerts the operator when a host transitions into a degraded state — closing the loop that let five +non-root-cutover regressions go undetected until user-visible breakage. + +- **`monitor.HostCapabilityChecker` (NEW):** a deliberate SIBLING of `HostStalenessChecker` — same + per-host state map (`ok`/`degraded`), seed-without-event, emit-only-on-transition shape. A host is + `degraded` iff its latest report has any **Critical** capability with `status:"degraded"`; + non-critical degradations ride the report but never alert. Runs on the existing 60s sweep next to + the staleness checkers. +- **Events:** `agent_capability_degraded` (warning) on ok→degraded, naming the degraded capabilities + + gated features in the message and details JSON; `agent_capability_recovered` (info) on + degraded→ok. Routed through the existing `Dispatcher.ProcessEvent` — **operator-only** (the type + is not a customer notification toggle, same as `host_stale`) with the standard 1 h operator + cooldown (no per-cycle re-alert). +- **`store.GetHostCapabilities` (NEW):** reads the capability snapshot from the latest host-report's + `report_json` per host (keyed on `MAX(id)` — within-second `received_at` ties would otherwise + return multiple rows). **No schema migration** — the array rides the existing report body. A + pre-v0.44.0 agent (no `capabilities`) reads as `ok`, so an old agent can't trip a false alert. +- Cross-repo `host-report.golden.json` mirrors the new `capabilities: []` field (byte-identical with + the agent copy). Version `0.18.0 → 0.19.0`. + ## v0.18.0 — App-email passthrough: POST /api/v1/mail → Resend SMTP (2026-06-29) The hub can now relay a customer box's outbound app email to Resend, re-emitting the raw MIME diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 8b9f0be..9d24d9d 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -316,6 +316,9 @@ func main() { // v0.7.0: host-domain dead-man's-switch (sibling; the controller checker above is // unchanged and keeps running until the slice-10 cutover). Same 60s cadence. hostStalenessChecker := monitor.NewHostStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger) + // v0.44.0-agent: operator alert when an agent reports a degraded privileged capability (a missing + // `sudo -n` grant — the 2026-06-28 cutover class). Same 60s sweep + transition/cooldown plumbing. + hostCapabilityChecker := monitor.NewHostCapabilityChecker(dataStore, dispatcher.ProcessEvent, logger) go func() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() @@ -326,6 +329,7 @@ func main() { case <-ticker.C: stalenessChecker.Check() hostStalenessChecker.Check() + hostCapabilityChecker.Check() } } }() diff --git a/hub/internal/api/testdata/host-report.golden.json b/hub/internal/api/testdata/host-report.golden.json index 16832c1..4392845 100644 --- a/hub/internal/api/testdata/host-report.golden.json +++ b/hub/internal/api/testdata/host-report.golden.json @@ -132,6 +132,7 @@ ], "cloudflared": { "status": "active" }, "audit_tail": [], + "capabilities": [], "dr_recipe": { "recipe_version": 1, "guests": [ diff --git a/hub/internal/monitor/host_capability.go b/hub/internal/monitor/host_capability.go new file mode 100644 index 0000000..b69f45d --- /dev/null +++ b/hub/internal/monitor/host_capability.go @@ -0,0 +1,178 @@ +package monitor + +import ( + "encoding/json" + "log" + "sort" + "strings" + "sync" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// HostCapabilityChecker is the operator-facing alert for the agent's privileged-capability +// self-check (felhom-agent v0.44.0). It is a deliberate SIBLING of HostStalenessChecker: same +// state-transition shape (per-host ok/degraded map, seed-without-event, emit only on change), same +// dispatcher/event plumbing — but it keys on the capability snapshot the agent rides in its host +// report (whether each required `sudo -n` grant is usable) rather than report recency. +// +// A host is "degraded" iff its latest report has any CRITICAL capability with status "degraded" +// (a missing user-facing grant — the multi-drive-flapping class). Non-critical degradations ride +// the report + the agent's own logs but do NOT alert the operator (avoid noise). The event is +// attributed to the host's customer, so the existing operator notification UX picks it up +// unchanged. Customer is NEVER notified — internal capability health is operator-only. +type HostCapabilityChecker struct { + store *store.Store + logger *log.Logger + onEvent EventNotifyFunc + + mu sync.Mutex + states map[string]string // hostID → "ok" | "degraded" + customerOf map[string]string // hostID → customerID +} + +// NewHostCapabilityChecker creates the checker and seeds state from the current snapshots. No +// events are generated during initialization (mirrors the staleness checker). +func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostCapabilityChecker { + cc := &HostCapabilityChecker{ + store: s, + logger: logger, + onEvent: onEvent, + states: make(map[string]string), + customerOf: make(map[string]string), + } + rows, err := s.GetHostCapabilities() + if err != nil { + logger.Printf("[WARN] Host capability checker: failed to seed states: %v", err) + return cc + } + var okCount, degCount int + for _, row := range rows { + if s.IsCustomerBlocked(row.CustomerID) { + continue + } + cc.customerOf[row.HostID] = row.CustomerID + st, _, _ := capabilityState(row.Capabilities) + cc.states[row.HostID] = st + if st == "degraded" { + degCount++ + } else { + okCount++ + } + } + logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded", okCount, degCount) + return cc +} + +// Check evaluates all hosts and emits an operator event on each ok↔degraded transition. Call on the +// same sweep as the staleness checker (every 60s). +func (cc *HostCapabilityChecker) Check() { + rows, err := cc.store.GetHostCapabilities() + if err != nil { + cc.logger.Printf("[WARN] Host capability check failed: %v", err) + return + } + cc.mu.Lock() + defer cc.mu.Unlock() + + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + seen[row.HostID] = true + if cc.store.IsCustomerBlocked(row.CustomerID) { + delete(cc.states, row.HostID) + continue + } + cc.customerOf[row.HostID] = row.CustomerID + + newState, names, features := capabilityState(row.Capabilities) + oldState := cc.states[row.HostID] + if oldState == "" { + cc.states[row.HostID] = newState // first observation — no event + continue + } + if oldState == newState { + continue + } + cc.states[row.HostID] = newState + cc.emitTransition(row.HostID, row.CustomerID, oldState, newState, names, features) + } + + for id := range cc.states { + if !seen[id] { + delete(cc.states, id) + } + } +} + +// GetState returns the current capability state for a host ("unknown" if unseen). +func (cc *HostCapabilityChecker) GetState(hostID string) string { + cc.mu.Lock() + defer cc.mu.Unlock() + s := cc.states[hostID] + if s == "" { + return "unknown" + } + return s +} + +// capabilityState reduces a snapshot to "ok"/"degraded" plus the degraded CRITICAL capability names +// + features (for the event detail). An empty/absent snapshot (pre-v0.44.0 agent) is "ok" — an old +// agent can't trip a false alert. +func capabilityState(caps []store.CapabilityStatus) (state string, names, features []string) { + for _, c := range caps { + if c.Critical && c.Status == "degraded" { + names = append(names, c.Name) + features = append(features, c.Feature) + } + } + sort.Strings(names) + if len(names) > 0 { + return "degraded", names, features + } + return "ok", nil, nil +} + +func (cc *HostCapabilityChecker) emitTransition(hostID, customerID, oldState, newState string, names, features []string) { + var eventType, severity, message string + switch { + case newState == "degraded": + eventType = "agent_capability_degraded" + severity = "warning" + message = "Host " + hostID + ": agent privileged capability degraded — " + + strings.Join(names, ", ") + " (impairs: " + strings.Join(dedup(features), "; ") + ")" + case newState == "ok" && oldState == "degraded": + eventType = "agent_capability_recovered" + severity = "info" + message = "Host " + hostID + ": agent privileged capabilities recovered (all required grants restored)" + default: + return + } + + details, _ := json.Marshal(map[string]any{ + "host_id": hostID, + "degraded_capabilities": names, + "gated_features": dedup(features), + }) + + cc.logger.Printf("[INFO] Host capability: %s %s → %s (%s)", hostID, oldState, newState, eventType) + + if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { + cc.logger.Printf("[WARN] Failed to save host capability event for %s: %v", hostID, err) + return + } + if cc.onEvent != nil { + cc.onEvent(customerID, eventType, severity, message, string(details), "hub") + } +} + +func dedup(in []string) []string { + seen := make(map[string]bool, len(in)) + var out []string + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} diff --git a/hub/internal/monitor/host_capability_test.go b/hub/internal/monitor/host_capability_test.go new file mode 100644 index 0000000..b74d2e5 --- /dev/null +++ b/hub/internal/monitor/host_capability_test.go @@ -0,0 +1,107 @@ +package monitor + +import ( + "io" + "log" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +// reportWith builds a host-report body carrying the given capability snapshot (the only field the +// capability checker reads). One critical-degraded entry makes the host "degraded". +func reportWith(caps string) []byte { + return []byte(`{"host_id":"h1","capabilities":` + caps + `}`) +} + +const capAllOK = `[{"name":"guest-init-pid","feature":"drive-gate","critical":true,"status":"ok"}]` +const capCritDegraded = `[{"name":"guest-init-pid","feature":"drive-gate guest-sees","critical":true,"status":"degraded","reason":"sudo policy denied"}]` +const capNonCritDegraded = `[{"name":"disk-smart","feature":"SMART","critical":false,"status":"degraded","reason":"binary not found"}]` + +func newCapStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}) + return st +} + +// TestHostCapabilityChecker covers the full transition contract (§10): seed (no event), ok→degraded +// (one event), steady degraded (none), degraded→ok (recovered), and that a non-critical degradation +// never trips the host state. +func TestHostCapabilityChecker(t *testing.T) { + st := newCapStore(t) + st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{}) + + var events []string + onEvent := func(_, eventType, _, _, _, _ string) { events = append(events, eventType) } + + // Seed ok → no event on init. + cc := NewHostCapabilityChecker(st, onEvent, log.New(io.Discard, "", 0)) + if cc.GetState("h1") != "ok" { + t.Fatalf("seed state = %s, want ok", cc.GetState("h1")) + } + if len(events) != 0 { + t.Fatalf("seed must not emit, got %v", events) + } + + // ok → degraded: exactly ONE operator event. + st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{}) + cc.Check() + if cc.GetState("h1") != "degraded" { + t.Fatalf("state = %s, want degraded", cc.GetState("h1")) + } + if len(events) != 1 || events[0] != "agent_capability_degraded" { + t.Fatalf("want one agent_capability_degraded, got %v", events) + } + + // Steady degraded: NO duplicate event. + st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{}) + cc.Check() + if len(events) != 1 { + t.Fatalf("steady degraded must not re-emit, got %v", events) + } + + // degraded → ok: recovered event. + st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{}) + cc.Check() + if cc.GetState("h1") != "ok" { + t.Fatalf("state = %s, want ok", cc.GetState("h1")) + } + if len(events) != 2 || events[1] != "agent_capability_recovered" { + t.Fatalf("want recovered event, got %v", events) + } +} + +// A NON-critical degradation must NOT flip the host to degraded (no operator noise). +func TestHostCapabilityChecker_NonCriticalIgnored(t *testing.T) { + st := newCapStore(t) + st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{}) + var events []string + cc := NewHostCapabilityChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + + st.SaveHostReport("h1", "c1", reportWith(capNonCritDegraded), store.HostReportDenorm{}) + cc.Check() + if cc.GetState("h1") != "ok" { + t.Fatalf("non-critical degradation flipped state to %s, want ok", cc.GetState("h1")) + } + if len(events) != 0 { + t.Fatalf("non-critical degradation must not alert, got %v", events) + } +} + +// A pre-v0.44.0 agent (no capabilities array) is "ok" — an old agent can't trip a false alert. +func TestHostCapabilityChecker_OldAgentNoCaps(t *testing.T) { + st := newCapStore(t) + st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{}) + cc := NewHostCapabilityChecker(st, func(_, _, _, _, _, _ string) {}, log.New(io.Discard, "", 0)) + if cc.GetState("h1") != "ok" { + t.Fatalf("old agent (no caps) state = %s, want ok", cc.GetState("h1")) + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 00e956f..c874fce 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -1624,3 +1624,52 @@ func (s *Store) GetHostStaleness() ([]HostStaleRow, error) { } return out, rows.Err() } + +// CapabilityStatus mirrors the agent's capability.Status wire shape (felhom-agent v0.44.0): one +// privileged `sudo -n` grant the non-root agent depends on, and whether it is currently usable. +type CapabilityStatus struct { + Name string `json:"name"` + Feature string `json:"feature"` + Critical bool `json:"critical"` + Status string `json:"status"` // "ok" | "degraded" + Reason string `json:"reason,omitempty"` +} + +// HostCapabilityRow is the per-host capability snapshot the HostCapabilityChecker reads — extracted +// from the latest host-report's report_json (no dedicated column; the array rides the report body). +type HostCapabilityRow struct { + HostID string + CustomerID string + Capabilities []CapabilityStatus +} + +// GetHostCapabilities returns the latest capability snapshot per host (from the most recent +// host_reports row). Hosts whose latest report carries no capabilities array (a pre-v0.44.0 agent) +// yield an empty slice — the checker treats that as "ok/unknown" and never alerts, so an old agent +// can't trip a false degraded. +func (s *Store) GetHostCapabilities() ([]HostCapabilityRow, error) { + rows, err := s.db.Query(` + SELECT hr.host_id, hr.customer_id, hr.report_json + FROM host_reports hr + JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest + ON hr.id = latest.mx`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []HostCapabilityRow + for rows.Next() { + var r HostCapabilityRow + var reportJSON string + if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { + return nil, err + } + var body struct { + Capabilities []CapabilityStatus `json:"capabilities"` + } + _ = json.Unmarshal([]byte(reportJSON), &body) // a malformed/old body → nil caps → no alert + r.Capabilities = body.Capabilities + out = append(out, r) + } + return out, rows.Err() +}