From 9c5cf2975fcc191411a154496c41c5ac00583b9d Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 29 Jun 2026 23:15:22 +0200 Subject: [PATCH] =?UTF-8?q?hub:=20HostLeafChecker=20=E2=80=94=20proactive?= =?UTF-8?q?=20agent=20re-key=20detection=20(host=5Fleaf=5Fchanged)=20v0.22?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Watches each host's reported local-API leaf fp; alerts on change (trust-on-first-report). Sibling of HostCapabilityChecker; store.GetHostLeafFingerprints reads report_json (no migration); hub-generated event (no allowlist change). Change red-proof + first-obs-seed + empty-skip + blocked-drop tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ --- hub/CHANGELOG.md | 24 +++ .../api/testdata/host-report.golden.json | 1 + hub/internal/monitor/host_leaf.go | 131 ++++++++++++++++ hub/internal/monitor/host_leaf_test.go | 145 ++++++++++++++++++ hub/internal/store/store.go | 38 +++++ 5 files changed, 339 insertions(+) create mode 100644 hub/internal/monitor/host_leaf.go create mode 100644 hub/internal/monitor/host_leaf_test.go diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 6ea5617..380da05 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,29 @@ # Felhom Hub — Changelog +## v0.22.0 — proactive agent re-key detection: HostLeafChecker (host_leaf_changed) (2026-06-29) + +Companion to felhom-agent v0.48.0 (which now reports its served local-API leaf fp). The hub watches +each host's leaf fingerprint and raises an operator alert when it **changes** (an agent re-key) — +proactive, fleet-wide, independent of any controller's channel-health check. The last self-health leg. + +- **`monitor.HostLeafChecker` (NEW):** sibling of `HostCapabilityChecker`. Trust-on-first-report — the + first fp per host is the baseline; a later change emits **`host_leaf_changed`** (operator, English; + details carry old+new fp) and advances the baseline. First-obs seeds silently (a "change" needs a + prior value, so no F2 issue). An empty reported fp (pre-v0.48.0 / local-API-disabled) is unknown — + never seeds, never alerts, never overwrites a baseline. Customer-blocked hosts dropped; unseen pruned. + Runs on the existing 60s sweep. **KNOWN LIMITATION (documented):** trust-on-first-report can't detect + a re-key that happened before the hub's first report — but the controller channel-check catches the + downstream pin mismatch, so this is defense-in-depth. +- **`store.GetHostLeafFingerprints` (NEW):** latest reported fp per host, parsed from `report_json` + (mirrors `GetHostCapabilities` — `MAX(id)`, **no schema migration**). +- **No allowlist change:** `host_leaf_changed` is hub-GENERATED (via `SaveEvent` + `dispatcher.ProcessEvent`), + not controller-pushed, so it bypasses the `/api/v1/event` `allowedEventTypes` gate — same as the + host_* events. The generic operator template relays it (no template change). +- Tests: change **red-proof** (A→B → one event + baseline advanced; companion: unchanged → none), + first-obs seeds silently, change-back re-alerts, empty fp skipped, customer-blocked dropped. Cross-repo + golden mirrors `leaf_fingerprint`. Version `0.21.0 → 0.22.0`. + + ## v0.21.0 — F2: alert on a host already degraded/stale at hub (re)start (2026-06-29) Mirror of the controller's F2 fix, for the hub checkers: a host that was already **degraded** diff --git a/hub/internal/api/testdata/host-report.golden.json b/hub/internal/api/testdata/host-report.golden.json index 4392845..5943e36 100644 --- a/hub/internal/api/testdata/host-report.golden.json +++ b/hub/internal/api/testdata/host-report.golden.json @@ -133,6 +133,7 @@ "cloudflared": { "status": "active" }, "audit_tail": [], "capabilities": [], + "leaf_fingerprint": "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245", "dr_recipe": { "recipe_version": 1, "guests": [ diff --git a/hub/internal/monitor/host_leaf.go b/hub/internal/monitor/host_leaf.go new file mode 100644 index 0000000..2dbd070 --- /dev/null +++ b/hub/internal/monitor/host_leaf.go @@ -0,0 +1,131 @@ +package monitor + +import ( + "encoding/json" + "log" + "sync" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// HostLeafChecker raises an operator alert when a host's agent local-API LEAF FINGERPRINT changes — a +// proactive, fleet-wide agent-re-key detector (the last self-health leg). It is a sibling of +// HostCapabilityChecker and independent of any controller's channel-health check: it catches a re-key +// even if a controller is down / hasn't probed, and gives the operator one fleet view. +// +// Design (Option 1 — trust-on-first-report): the first leaf fp seen per host is the baseline; any later +// change alerts and advances the baseline. KNOWN LIMITATION (v1, acceptable): if the agent re-keyed +// BEFORE the hub's first report, the hub trusts the wrong baseline — but the controller channel-check +// still catches the downstream pin mismatch, so this is defense-in-depth, not the sole guard. (The +// authoritative served-fp-vs-pinned-fp cross-check is a deliberate future enhancement.) +// +// An empty reported fp (pre-v0.48.0 agent, or the local API disabled) is "unknown" — never an alert and +// never overwrites a baseline. +type HostLeafChecker struct { + store *store.Store + logger *log.Logger + onEvent EventNotifyFunc + + mu sync.Mutex + states map[string]string // hostID → last-seen leaf fp (the baseline) + customerOf map[string]string // hostID → customerID (event attribution) +} + +// NewHostLeafChecker seeds the per-host baseline from the latest reported fps. No events on init. +func NewHostLeafChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostLeafChecker { + lc := &HostLeafChecker{ + store: s, + logger: logger, + onEvent: onEvent, + states: make(map[string]string), + customerOf: make(map[string]string), + } + rows, err := s.GetHostLeafFingerprints() + if err != nil { + logger.Printf("[WARN] Host leaf checker: failed to seed: %v", err) + return lc + } + seeded := 0 + for _, row := range rows { + if s.IsCustomerBlocked(row.CustomerID) || row.LeafFP == "" { + continue + } + lc.customerOf[row.HostID] = row.CustomerID + lc.states[row.HostID] = row.LeafFP + seeded++ + } + logger.Printf("[INFO] Host leaf checker initialized: %d host fingerprint(s) seeded", seeded) + return lc +} + +// Check evaluates all hosts and emits host_leaf_changed on a fingerprint change. Call on the same 60s +// sweep as the staleness/capability checkers. +func (lc *HostLeafChecker) Check() { + rows, err := lc.store.GetHostLeafFingerprints() + if err != nil { + lc.logger.Printf("[WARN] Host leaf check failed: %v", err) + return + } + lc.mu.Lock() + defer lc.mu.Unlock() + + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + if lc.store.IsCustomerBlocked(row.CustomerID) { + delete(lc.states, row.HostID) + continue + } + seen[row.HostID] = true // the host exists; keep its baseline even if this report's fp is empty + if row.LeafFP == "" { + continue // unknown fp → no alert, no baseline change + } + lc.customerOf[row.HostID] = row.CustomerID + old := lc.states[row.HostID] + if old == "" { + lc.states[row.HostID] = row.LeafFP // first observation → baseline, no event + continue + } + if old == row.LeafFP { + continue + } + lc.states[row.HostID] = row.LeafFP // advance the baseline (a change-back later re-alerts) + lc.emit(row.HostID, row.CustomerID, old, row.LeafFP) + } + + for id := range lc.states { + if !seen[id] { + delete(lc.states, id) + } + } +} + +// GetState returns the current baseline fp for a host ("" if unseen). +func (lc *HostLeafChecker) GetState(hostID string) string { + lc.mu.Lock() + defer lc.mu.Unlock() + return lc.states[hostID] +} + +func (lc *HostLeafChecker) emit(hostID, customerID, oldFP, newFP string) { + msg := "Host " + hostID + ": agent local-API leaf fingerprint CHANGED (agent re-keyed) — controllers will fail the pin check until re-bootstrapped / re-pinned" + details, _ := json.Marshal(map[string]string{ + "host_id": hostID, + "old_fingerprint": oldFP, + "new_fingerprint": newFP, + }) + lc.logger.Printf("[WARN] Host leaf: %s fp %s… → %s… (host_leaf_changed)", hostID, shortFP(oldFP), shortFP(newFP)) + if _, err := lc.store.SaveEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub"); err != nil { + lc.logger.Printf("[WARN] save host_leaf_changed for %s: %v", hostID, err) + return + } + if lc.onEvent != nil { + lc.onEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub") + } +} + +func shortFP(fp string) string { + if len(fp) > 12 { + return fp[:12] + } + return fp +} diff --git a/hub/internal/monitor/host_leaf_test.go b/hub/internal/monitor/host_leaf_test.go new file mode 100644 index 0000000..e87fc52 --- /dev/null +++ b/hub/internal/monitor/host_leaf_test.go @@ -0,0 +1,145 @@ +package monitor + +import ( + "io" + "log" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +const ( + fpA = "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245" + fpB = "911d703c9cb4cf54d7aba9d9a768e587e9d0757ea638b1db595939623c675738" +) + +func reportWithLeaf(fp string) []byte { + return []byte(`{"host_id":"h1","leaf_fingerprint":"` + fp + `"}`) +} + +func newLeafStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}) + return st +} + +// B.2 change RED-PROOF: seed fp A, then fp B → exactly one host_leaf_changed (A→B), baseline advanced +// to B. Companion (TestHostLeafChecker_NoChangeNoEvent): fp stays A → no event — proving the CHANGE, +// not the cycle, fires it. +func TestHostLeafChecker_ChangeAlertsOnce(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if lc.GetState("h1") != fpA { + t.Fatalf("seed baseline = %q, want %q", lc.GetState("h1"), fpA) + } + if len(events) != 0 { + t.Fatalf("construction must not emit, got %v", events) + } + + st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{}) + lc.Check() + if len(events) != 1 || events[0] != "host_leaf_changed" { + t.Fatalf("fp change → one host_leaf_changed, got %v", events) + } + if lc.GetState("h1") != fpB { + t.Fatalf("baseline not advanced, got %q want %q", lc.GetState("h1"), fpB) + } + lc.Check() // steady at B → no duplicate + if len(events) != 1 { + t.Fatalf("steady fp must not re-emit, got %v", events) + } +} + +func TestHostLeafChecker_NoChangeNoEvent(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) // same fp + lc.Check() + if len(events) != 0 { + t.Fatalf("unchanged fp must not alert, got %v", events) + } +} + +// First observation seeds the baseline silently. +func TestHostLeafChecker_FirstObsSeedsNoEvent(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + // construction already seeded; a Check with no change stays silent. + lc.Check() + if len(events) != 0 { + t.Fatalf("first-obs/seed must be silent, got %v", events) + } + if lc.GetState("h1") != fpA { + t.Fatalf("baseline = %q, want %q", lc.GetState("h1"), fpA) + } +} + +// A change-BACK (B→A) is also a change → another event (informative). +func TestHostLeafChecker_ChangeBackReAlerts(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{}) + lc.Check() // A→B + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + lc.Check() // B→A + if len(events) != 2 { + t.Fatalf("change-back should re-alert (want 2), got %d", len(events)) + } +} + +// An empty reported fp (pre-v0.48.0 / local-API disabled) is unknown — never seeded, never an alert, +// and never overwrites an existing baseline. +func TestHostLeafChecker_EmptyFpSkipped(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{}) // no leaf_fingerprint + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if lc.GetState("h1") != "" { + t.Fatalf("empty fp must not seed a baseline, got %q", lc.GetState("h1")) + } + lc.Check() + if len(events) != 0 { + t.Fatalf("empty fp must not alert, got %v", events) + } + // A real fp later → seeds (no alert, it's the first real baseline). + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + lc.Check() + if len(events) != 0 || lc.GetState("h1") != fpA { + t.Fatalf("first real fp should seed silently, events=%v state=%q", events, lc.GetState("h1")) + } +} + +// A blocked customer's host is dropped — no event. +func TestHostLeafChecker_CustomerBlockedDropped(t *testing.T) { + st := newLeafStore(t) + st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) + var events []string + lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if err := st.SetCustomerConfigStatus("c1", "blocked"); err != nil { + t.Fatalf("block customer: %v", err) + } + st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{}) // would-be change + lc.Check() + if len(events) != 0 { + t.Fatalf("blocked customer must not alert, got %v", events) + } + if lc.GetState("h1") != "" { + t.Fatalf("blocked host should be dropped, got %q", lc.GetState("h1")) + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index c874fce..e058e86 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -1673,3 +1673,41 @@ func (s *Store) GetHostCapabilities() ([]HostCapabilityRow, error) { } return out, rows.Err() } + +// HostLeafRow is the per-host served-leaf-fingerprint the HostLeafChecker reads — extracted from the +// latest host-report's report_json (no dedicated column; the fp rides the report body, same as the +// capabilities snapshot). LeafFP is "" for a pre-v0.48.0 agent / local-API-disabled host. +type HostLeafRow struct { + HostID string + CustomerID string + LeafFP string +} + +// GetHostLeafFingerprints returns the latest reported local-API leaf fp per host (mirrors +// GetHostCapabilities — MAX(id) per host, parsed from report_json so there is no schema migration). +func (s *Store) GetHostLeafFingerprints() ([]HostLeafRow, error) { + rows, err := s.db.Query(` + SELECT hr.host_id, hr.customer_id, hr.report_json + FROM host_reports hr + JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest + ON hr.id = latest.mx`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []HostLeafRow + for rows.Next() { + var r HostLeafRow + var reportJSON string + if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { + return nil, err + } + var body struct { + LeafFingerprint string `json:"leaf_fingerprint"` + } + _ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old body → "" → no alert + r.LeafFP = body.LeafFingerprint + out = append(out, r) + } + return out, rows.Err() +}