hub: HostCapabilityChecker — operator alert on agent capability-degraded (v0.19.0)
Companion to felhom-agent v0.44.0. New monitor.HostCapabilityChecker (sibling of HostStalenessChecker) reads the capabilities snapshot from the latest host report and emits agent_capability_degraded/recovered (operator-only, 1h cooldown) on ok<->degraded transitions for any Critical capability. store.GetHostCapabilities (MAX(id), no migration). Goldens mirror the new capabilities field. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt
This commit is contained in:
@@ -1,5 +1,29 @@
|
||||
# Felhom Hub — Changelog
|
||||
|
||||
## v0.19.0 — Agent capability-degraded operator alert (HostCapabilityChecker) (2026-06-29)
|
||||
|
||||
Companion to felhom-agent v0.44.0's privileged-capability self-probe: the agent now rides a
|
||||
`capabilities` snapshot on its host report (each required `sudo -n` grant: ok/degraded), and the hub
|
||||
alerts the operator when a host transitions into a degraded state — closing the loop that let five
|
||||
non-root-cutover regressions go undetected until user-visible breakage.
|
||||
|
||||
- **`monitor.HostCapabilityChecker` (NEW):** a deliberate SIBLING of `HostStalenessChecker` — same
|
||||
per-host state map (`ok`/`degraded`), seed-without-event, emit-only-on-transition shape. A host is
|
||||
`degraded` iff its latest report has any **Critical** capability with `status:"degraded"`;
|
||||
non-critical degradations ride the report but never alert. Runs on the existing 60s sweep next to
|
||||
the staleness checkers.
|
||||
- **Events:** `agent_capability_degraded` (warning) on ok→degraded, naming the degraded capabilities
|
||||
+ gated features in the message and details JSON; `agent_capability_recovered` (info) on
|
||||
degraded→ok. Routed through the existing `Dispatcher.ProcessEvent` — **operator-only** (the type
|
||||
is not a customer notification toggle, same as `host_stale`) with the standard 1 h operator
|
||||
cooldown (no per-cycle re-alert).
|
||||
- **`store.GetHostCapabilities` (NEW):** reads the capability snapshot from the latest host-report's
|
||||
`report_json` per host (keyed on `MAX(id)` — within-second `received_at` ties would otherwise
|
||||
return multiple rows). **No schema migration** — the array rides the existing report body. A
|
||||
pre-v0.44.0 agent (no `capabilities`) reads as `ok`, so an old agent can't trip a false alert.
|
||||
- Cross-repo `host-report.golden.json` mirrors the new `capabilities: []` field (byte-identical with
|
||||
the agent copy). Version `0.18.0 → 0.19.0`.
|
||||
|
||||
## v0.18.0 — App-email passthrough: POST /api/v1/mail → Resend SMTP (2026-06-29)
|
||||
|
||||
The hub can now relay a customer box's outbound app email to Resend, re-emitting the raw MIME
|
||||
|
||||
@@ -316,6 +316,9 @@ func main() {
|
||||
// v0.7.0: host-domain dead-man's-switch (sibling; the controller checker above is
|
||||
// unchanged and keeps running until the slice-10 cutover). Same 60s cadence.
|
||||
hostStalenessChecker := monitor.NewHostStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger)
|
||||
// v0.44.0-agent: operator alert when an agent reports a degraded privileged capability (a missing
|
||||
// `sudo -n` grant — the 2026-06-28 cutover class). Same 60s sweep + transition/cooldown plumbing.
|
||||
hostCapabilityChecker := monitor.NewHostCapabilityChecker(dataStore, dispatcher.ProcessEvent, logger)
|
||||
go func() {
|
||||
ticker := time.NewTicker(60 * time.Second)
|
||||
defer ticker.Stop()
|
||||
@@ -326,6 +329,7 @@ func main() {
|
||||
case <-ticker.C:
|
||||
stalenessChecker.Check()
|
||||
hostStalenessChecker.Check()
|
||||
hostCapabilityChecker.Check()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -132,6 +132,7 @@
|
||||
],
|
||||
"cloudflared": { "status": "active" },
|
||||
"audit_tail": [],
|
||||
"capabilities": [],
|
||||
"dr_recipe": {
|
||||
"recipe_version": 1,
|
||||
"guests": [
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// HostCapabilityChecker is the operator-facing alert for the agent's privileged-capability
|
||||
// self-check (felhom-agent v0.44.0). It is a deliberate SIBLING of HostStalenessChecker: same
|
||||
// state-transition shape (per-host ok/degraded map, seed-without-event, emit only on change), same
|
||||
// dispatcher/event plumbing — but it keys on the capability snapshot the agent rides in its host
|
||||
// report (whether each required `sudo -n` grant is usable) rather than report recency.
|
||||
//
|
||||
// A host is "degraded" iff its latest report has any CRITICAL capability with status "degraded"
|
||||
// (a missing user-facing grant — the multi-drive-flapping class). Non-critical degradations ride
|
||||
// the report + the agent's own logs but do NOT alert the operator (avoid noise). The event is
|
||||
// attributed to the host's customer, so the existing operator notification UX picks it up
|
||||
// unchanged. Customer is NEVER notified — internal capability health is operator-only.
|
||||
type HostCapabilityChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
onEvent EventNotifyFunc
|
||||
|
||||
mu sync.Mutex
|
||||
states map[string]string // hostID → "ok" | "degraded"
|
||||
customerOf map[string]string // hostID → customerID
|
||||
}
|
||||
|
||||
// NewHostCapabilityChecker creates the checker and seeds state from the current snapshots. No
|
||||
// events are generated during initialization (mirrors the staleness checker).
|
||||
func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostCapabilityChecker {
|
||||
cc := &HostCapabilityChecker{
|
||||
store: s,
|
||||
logger: logger,
|
||||
onEvent: onEvent,
|
||||
states: make(map[string]string),
|
||||
customerOf: make(map[string]string),
|
||||
}
|
||||
rows, err := s.GetHostCapabilities()
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] Host capability checker: failed to seed states: %v", err)
|
||||
return cc
|
||||
}
|
||||
var okCount, degCount int
|
||||
for _, row := range rows {
|
||||
if s.IsCustomerBlocked(row.CustomerID) {
|
||||
continue
|
||||
}
|
||||
cc.customerOf[row.HostID] = row.CustomerID
|
||||
st, _, _ := capabilityState(row.Capabilities)
|
||||
cc.states[row.HostID] = st
|
||||
if st == "degraded" {
|
||||
degCount++
|
||||
} else {
|
||||
okCount++
|
||||
}
|
||||
}
|
||||
logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded", okCount, degCount)
|
||||
return cc
|
||||
}
|
||||
|
||||
// Check evaluates all hosts and emits an operator event on each ok↔degraded transition. Call on the
|
||||
// same sweep as the staleness checker (every 60s).
|
||||
func (cc *HostCapabilityChecker) Check() {
|
||||
rows, err := cc.store.GetHostCapabilities()
|
||||
if err != nil {
|
||||
cc.logger.Printf("[WARN] Host capability check failed: %v", err)
|
||||
return
|
||||
}
|
||||
cc.mu.Lock()
|
||||
defer cc.mu.Unlock()
|
||||
|
||||
seen := make(map[string]bool, len(rows))
|
||||
for _, row := range rows {
|
||||
seen[row.HostID] = true
|
||||
if cc.store.IsCustomerBlocked(row.CustomerID) {
|
||||
delete(cc.states, row.HostID)
|
||||
continue
|
||||
}
|
||||
cc.customerOf[row.HostID] = row.CustomerID
|
||||
|
||||
newState, names, features := capabilityState(row.Capabilities)
|
||||
oldState := cc.states[row.HostID]
|
||||
if oldState == "" {
|
||||
cc.states[row.HostID] = newState // first observation — no event
|
||||
continue
|
||||
}
|
||||
if oldState == newState {
|
||||
continue
|
||||
}
|
||||
cc.states[row.HostID] = newState
|
||||
cc.emitTransition(row.HostID, row.CustomerID, oldState, newState, names, features)
|
||||
}
|
||||
|
||||
for id := range cc.states {
|
||||
if !seen[id] {
|
||||
delete(cc.states, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetState returns the current capability state for a host ("unknown" if unseen).
|
||||
func (cc *HostCapabilityChecker) GetState(hostID string) string {
|
||||
cc.mu.Lock()
|
||||
defer cc.mu.Unlock()
|
||||
s := cc.states[hostID]
|
||||
if s == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// capabilityState reduces a snapshot to "ok"/"degraded" plus the degraded CRITICAL capability names
|
||||
// + features (for the event detail). An empty/absent snapshot (pre-v0.44.0 agent) is "ok" — an old
|
||||
// agent can't trip a false alert.
|
||||
func capabilityState(caps []store.CapabilityStatus) (state string, names, features []string) {
|
||||
for _, c := range caps {
|
||||
if c.Critical && c.Status == "degraded" {
|
||||
names = append(names, c.Name)
|
||||
features = append(features, c.Feature)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
if len(names) > 0 {
|
||||
return "degraded", names, features
|
||||
}
|
||||
return "ok", nil, nil
|
||||
}
|
||||
|
||||
func (cc *HostCapabilityChecker) emitTransition(hostID, customerID, oldState, newState string, names, features []string) {
|
||||
var eventType, severity, message string
|
||||
switch {
|
||||
case newState == "degraded":
|
||||
eventType = "agent_capability_degraded"
|
||||
severity = "warning"
|
||||
message = "Host " + hostID + ": agent privileged capability degraded — " +
|
||||
strings.Join(names, ", ") + " (impairs: " + strings.Join(dedup(features), "; ") + ")"
|
||||
case newState == "ok" && oldState == "degraded":
|
||||
eventType = "agent_capability_recovered"
|
||||
severity = "info"
|
||||
message = "Host " + hostID + ": agent privileged capabilities recovered (all required grants restored)"
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
details, _ := json.Marshal(map[string]any{
|
||||
"host_id": hostID,
|
||||
"degraded_capabilities": names,
|
||||
"gated_features": dedup(features),
|
||||
})
|
||||
|
||||
cc.logger.Printf("[INFO] Host capability: %s %s → %s (%s)", hostID, oldState, newState, eventType)
|
||||
|
||||
if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
|
||||
cc.logger.Printf("[WARN] Failed to save host capability event for %s: %v", hostID, err)
|
||||
return
|
||||
}
|
||||
if cc.onEvent != nil {
|
||||
cc.onEvent(customerID, eventType, severity, message, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
func dedup(in []string) []string {
|
||||
seen := make(map[string]bool, len(in))
|
||||
var out []string
|
||||
for _, s := range in {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// reportWith builds a host-report body carrying the given capability snapshot (the only field the
|
||||
// capability checker reads). One critical-degraded entry makes the host "degraded".
|
||||
func reportWith(caps string) []byte {
|
||||
return []byte(`{"host_id":"h1","capabilities":` + caps + `}`)
|
||||
}
|
||||
|
||||
const capAllOK = `[{"name":"guest-init-pid","feature":"drive-gate","critical":true,"status":"ok"}]`
|
||||
const capCritDegraded = `[{"name":"guest-init-pid","feature":"drive-gate guest-sees","critical":true,"status":"degraded","reason":"sudo policy denied"}]`
|
||||
const capNonCritDegraded = `[{"name":"disk-smart","feature":"SMART","critical":false,"status":"degraded","reason":"binary not found"}]`
|
||||
|
||||
func newCapStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
||||
return st
|
||||
}
|
||||
|
||||
// TestHostCapabilityChecker covers the full transition contract (§10): seed (no event), ok→degraded
|
||||
// (one event), steady degraded (none), degraded→ok (recovered), and that a non-critical degradation
|
||||
// never trips the host state.
|
||||
func TestHostCapabilityChecker(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
||||
|
||||
var events []string
|
||||
onEvent := func(_, eventType, _, _, _, _ string) { events = append(events, eventType) }
|
||||
|
||||
// Seed ok → no event on init.
|
||||
cc := NewHostCapabilityChecker(st, onEvent, log.New(io.Discard, "", 0))
|
||||
if cc.GetState("h1") != "ok" {
|
||||
t.Fatalf("seed state = %s, want ok", cc.GetState("h1"))
|
||||
}
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("seed must not emit, got %v", events)
|
||||
}
|
||||
|
||||
// ok → degraded: exactly ONE operator event.
|
||||
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
|
||||
cc.Check()
|
||||
if cc.GetState("h1") != "degraded" {
|
||||
t.Fatalf("state = %s, want degraded", cc.GetState("h1"))
|
||||
}
|
||||
if len(events) != 1 || events[0] != "agent_capability_degraded" {
|
||||
t.Fatalf("want one agent_capability_degraded, got %v", events)
|
||||
}
|
||||
|
||||
// Steady degraded: NO duplicate event.
|
||||
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
|
||||
cc.Check()
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("steady degraded must not re-emit, got %v", events)
|
||||
}
|
||||
|
||||
// degraded → ok: recovered event.
|
||||
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
||||
cc.Check()
|
||||
if cc.GetState("h1") != "ok" {
|
||||
t.Fatalf("state = %s, want ok", cc.GetState("h1"))
|
||||
}
|
||||
if len(events) != 2 || events[1] != "agent_capability_recovered" {
|
||||
t.Fatalf("want recovered event, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// A NON-critical degradation must NOT flip the host to degraded (no operator noise).
|
||||
func TestHostCapabilityChecker_NonCriticalIgnored(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
||||
var events []string
|
||||
cc := NewHostCapabilityChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
|
||||
st.SaveHostReport("h1", "c1", reportWith(capNonCritDegraded), store.HostReportDenorm{})
|
||||
cc.Check()
|
||||
if cc.GetState("h1") != "ok" {
|
||||
t.Fatalf("non-critical degradation flipped state to %s, want ok", cc.GetState("h1"))
|
||||
}
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("non-critical degradation must not alert, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// A pre-v0.44.0 agent (no capabilities array) is "ok" — an old agent can't trip a false alert.
|
||||
func TestHostCapabilityChecker_OldAgentNoCaps(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
||||
cc := NewHostCapabilityChecker(st, func(_, _, _, _, _, _ string) {}, log.New(io.Discard, "", 0))
|
||||
if cc.GetState("h1") != "ok" {
|
||||
t.Fatalf("old agent (no caps) state = %s, want ok", cc.GetState("h1"))
|
||||
}
|
||||
}
|
||||
@@ -1624,3 +1624,52 @@ func (s *Store) GetHostStaleness() ([]HostStaleRow, error) {
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CapabilityStatus mirrors the agent's capability.Status wire shape (felhom-agent v0.44.0): one
|
||||
// privileged `sudo -n` grant the non-root agent depends on, and whether it is currently usable.
|
||||
type CapabilityStatus struct {
|
||||
Name string `json:"name"`
|
||||
Feature string `json:"feature"`
|
||||
Critical bool `json:"critical"`
|
||||
Status string `json:"status"` // "ok" | "degraded"
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// HostCapabilityRow is the per-host capability snapshot the HostCapabilityChecker reads — extracted
|
||||
// from the latest host-report's report_json (no dedicated column; the array rides the report body).
|
||||
type HostCapabilityRow struct {
|
||||
HostID string
|
||||
CustomerID string
|
||||
Capabilities []CapabilityStatus
|
||||
}
|
||||
|
||||
// GetHostCapabilities returns the latest capability snapshot per host (from the most recent
|
||||
// host_reports row). Hosts whose latest report carries no capabilities array (a pre-v0.44.0 agent)
|
||||
// yield an empty slice — the checker treats that as "ok/unknown" and never alerts, so an old agent
|
||||
// can't trip a false degraded.
|
||||
func (s *Store) GetHostCapabilities() ([]HostCapabilityRow, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT hr.host_id, hr.customer_id, hr.report_json
|
||||
FROM host_reports hr
|
||||
JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
|
||||
ON hr.id = latest.mx`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []HostCapabilityRow
|
||||
for rows.Next() {
|
||||
var r HostCapabilityRow
|
||||
var reportJSON string
|
||||
if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body struct {
|
||||
Capabilities []CapabilityStatus `json:"capabilities"`
|
||||
}
|
||||
_ = json.Unmarshal([]byte(reportJSON), &body) // a malformed/old body → nil caps → no alert
|
||||
r.Capabilities = body.Capabilities
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user