hub: HostLeafChecker — proactive agent re-key detection (host_leaf_changed) v0.22.0

Watches each host's reported local-API leaf fp; alerts on change (trust-on-first-report). Sibling of
HostCapabilityChecker; store.GetHostLeafFingerprints reads report_json (no migration); hub-generated
event (no allowlist change). Change red-proof + first-obs-seed + empty-skip + blocked-drop tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ

hub/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Felhom Hub — Changelog
 
+## v0.22.0 — proactive agent re-key detection: HostLeafChecker (host_leaf_changed) (2026-06-29)
+
+Companion to felhom-agent v0.48.0 (which now reports its served local-API leaf fp). The hub watches
+each host's leaf fingerprint and raises an operator alert when it **changes** (an agent re-key) —
+proactive, fleet-wide, independent of any controller's channel-health check. The last self-health leg.
+
+- **`monitor.HostLeafChecker` (NEW):** sibling of `HostCapabilityChecker`. Trust-on-first-report — the
+  first fp per host is the baseline; a later change emits **`host_leaf_changed`** (operator, English;
+  details carry old+new fp) and advances the baseline. First-obs seeds silently (a "change" needs a
+  prior value, so no F2 issue). An empty reported fp (pre-v0.48.0 / local-API-disabled) is unknown —
+  never seeds, never alerts, never overwrites a baseline. Customer-blocked hosts dropped; unseen pruned.
+  Runs on the existing 60s sweep. **KNOWN LIMITATION (documented):** trust-on-first-report can't detect
+  a re-key that happened before the hub's first report — but the controller channel-check catches the
+  downstream pin mismatch, so this is defense-in-depth.
+- **`store.GetHostLeafFingerprints` (NEW):** latest reported fp per host, parsed from `report_json`
+  (mirrors `GetHostCapabilities` — `MAX(id)`, **no schema migration**).
+- **No allowlist change:** `host_leaf_changed` is hub-GENERATED (via `SaveEvent` + `dispatcher.ProcessEvent`),
+  not controller-pushed, so it bypasses the `/api/v1/event` `allowedEventTypes` gate — same as the
+  host_* events. The generic operator template relays it (no template change).
+- Tests: change **red-proof** (A→B → one event + baseline advanced; companion: unchanged → none),
+  first-obs seeds silently, change-back re-alerts, empty fp skipped, customer-blocked dropped. Cross-repo
+  golden mirrors `leaf_fingerprint`. Version `0.21.0 → 0.22.0`.
+
+
 ## v0.21.0 — F2: alert on a host already degraded/stale at hub (re)start (2026-06-29)
 
 Mirror of the controller's F2 fix, for the hub checkers: a host that was already **degraded**

hub/internal/api/testdata/host-report.golden.json

@@ -133,6 +133,7 @@
   "cloudflared": { "status": "active" },
   "audit_tail": [],
   "capabilities": [],
+  "leaf_fingerprint": "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245",
   "dr_recipe": {
     "recipe_version": 1,
     "guests": [

hub/internal/monitor/host_leaf.go

@@ -0,0 +1,131 @@
+package monitor
+
+import (
+	"encoding/json"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// HostLeafChecker raises an operator alert when a host's agent local-API LEAF FINGERPRINT changes — a
+// proactive, fleet-wide agent-re-key detector (the last self-health leg). It is a sibling of
+// HostCapabilityChecker and independent of any controller's channel-health check: it catches a re-key
+// even if a controller is down / hasn't probed, and gives the operator one fleet view.
+//
+// Design (Option 1 — trust-on-first-report): the first leaf fp seen per host is the baseline; any later
+// change alerts and advances the baseline. KNOWN LIMITATION (v1, acceptable): if the agent re-keyed
+// BEFORE the hub's first report, the hub trusts the wrong baseline — but the controller channel-check
+// still catches the downstream pin mismatch, so this is defense-in-depth, not the sole guard. (The
+// authoritative served-fp-vs-pinned-fp cross-check is a deliberate future enhancement.)
+//
+// An empty reported fp (pre-v0.48.0 agent, or the local API disabled) is "unknown" — never an alert and
+// never overwrites a baseline.
+type HostLeafChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	states     map[string]string // hostID → last-seen leaf fp (the baseline)
+	customerOf map[string]string // hostID → customerID (event attribution)
+}
+
+// NewHostLeafChecker seeds the per-host baseline from the latest reported fps. No events on init.
+func NewHostLeafChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostLeafChecker {
+	lc := &HostLeafChecker{
+		store:      s,
+		logger:     logger,
+		onEvent:    onEvent,
+		states:     make(map[string]string),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostLeafFingerprints()
+	if err != nil {
+		logger.Printf("[WARN] Host leaf checker: failed to seed: %v", err)
+		return lc
+	}
+	seeded := 0
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) || row.LeafFP == "" {
+			continue
+		}
+		lc.customerOf[row.HostID] = row.CustomerID
+		lc.states[row.HostID] = row.LeafFP
+		seeded++
+	}
+	logger.Printf("[INFO] Host leaf checker initialized: %d host fingerprint(s) seeded", seeded)
+	return lc
+}
+
+// Check evaluates all hosts and emits host_leaf_changed on a fingerprint change. Call on the same 60s
+// sweep as the staleness/capability checkers.
+func (lc *HostLeafChecker) Check() {
+	rows, err := lc.store.GetHostLeafFingerprints()
+	if err != nil {
+		lc.logger.Printf("[WARN] Host leaf check failed: %v", err)
+		return
+	}
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if lc.store.IsCustomerBlocked(row.CustomerID) {
+			delete(lc.states, row.HostID)
+			continue
+		}
+		seen[row.HostID] = true // the host exists; keep its baseline even if this report's fp is empty
+		if row.LeafFP == "" {
+			continue // unknown fp → no alert, no baseline change
+		}
+		lc.customerOf[row.HostID] = row.CustomerID
+		old := lc.states[row.HostID]
+		if old == "" {
+			lc.states[row.HostID] = row.LeafFP // first observation → baseline, no event
+			continue
+		}
+		if old == row.LeafFP {
+			continue
+		}
+		lc.states[row.HostID] = row.LeafFP // advance the baseline (a change-back later re-alerts)
+		lc.emit(row.HostID, row.CustomerID, old, row.LeafFP)
+	}
+
+	for id := range lc.states {
+		if !seen[id] {
+			delete(lc.states, id)
+		}
+	}
+}
+
+// GetState returns the current baseline fp for a host ("" if unseen).
+func (lc *HostLeafChecker) GetState(hostID string) string {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	return lc.states[hostID]
+}
+
+func (lc *HostLeafChecker) emit(hostID, customerID, oldFP, newFP string) {
+	msg := "Host " + hostID + ": agent local-API leaf fingerprint CHANGED (agent re-keyed) — controllers will fail the pin check until re-bootstrapped / re-pinned"
+	details, _ := json.Marshal(map[string]string{
+		"host_id":         hostID,
+		"old_fingerprint": oldFP,
+		"new_fingerprint": newFP,
+	})
+	lc.logger.Printf("[WARN] Host leaf: %s fp %s… → %s… (host_leaf_changed)", hostID, shortFP(oldFP), shortFP(newFP))
+	if _, err := lc.store.SaveEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub"); err != nil {
+		lc.logger.Printf("[WARN] save host_leaf_changed for %s: %v", hostID, err)
+		return
+	}
+	if lc.onEvent != nil {
+		lc.onEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub")
+	}
+}
+
+func shortFP(fp string) string {
+	if len(fp) > 12 {
+		return fp[:12]
+	}
+	return fp
+}

hub/internal/monitor/host_leaf_test.go

@@ -0,0 +1,145 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+const (
+	fpA = "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245"
+	fpB = "911d703c9cb4cf54d7aba9d9a768e587e9d0757ea638b1db595939623c675738"
+)
+
+func reportWithLeaf(fp string) []byte {
+	return []byte(`{"host_id":"h1","leaf_fingerprint":"` + fp + `"}`)
+}
+
+func newLeafStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
+	return st
+}
+
+// B.2 change RED-PROOF: seed fp A, then fp B → exactly one host_leaf_changed (A→B), baseline advanced
+// to B. Companion (TestHostLeafChecker_NoChangeNoEvent): fp stays A → no event — proving the CHANGE,
+// not the cycle, fires it.
+func TestHostLeafChecker_ChangeAlertsOnce(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if lc.GetState("h1") != fpA {
+		t.Fatalf("seed baseline = %q, want %q", lc.GetState("h1"), fpA)
+	}
+	if len(events) != 0 {
+		t.Fatalf("construction must not emit, got %v", events)
+	}
+
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{})
+	lc.Check()
+	if len(events) != 1 || events[0] != "host_leaf_changed" {
+		t.Fatalf("fp change → one host_leaf_changed, got %v", events)
+	}
+	if lc.GetState("h1") != fpB {
+		t.Fatalf("baseline not advanced, got %q want %q", lc.GetState("h1"), fpB)
+	}
+	lc.Check() // steady at B → no duplicate
+	if len(events) != 1 {
+		t.Fatalf("steady fp must not re-emit, got %v", events)
+	}
+}
+
+func TestHostLeafChecker_NoChangeNoEvent(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{}) // same fp
+	lc.Check()
+	if len(events) != 0 {
+		t.Fatalf("unchanged fp must not alert, got %v", events)
+	}
+}
+
+// First observation seeds the baseline silently.
+func TestHostLeafChecker_FirstObsSeedsNoEvent(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	// construction already seeded; a Check with no change stays silent.
+	lc.Check()
+	if len(events) != 0 {
+		t.Fatalf("first-obs/seed must be silent, got %v", events)
+	}
+	if lc.GetState("h1") != fpA {
+		t.Fatalf("baseline = %q, want %q", lc.GetState("h1"), fpA)
+	}
+}
+
+// A change-BACK (B→A) is also a change → another event (informative).
+func TestHostLeafChecker_ChangeBackReAlerts(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{})
+	lc.Check() // A→B
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	lc.Check() // B→A
+	if len(events) != 2 {
+		t.Fatalf("change-back should re-alert (want 2), got %d", len(events))
+	}
+}
+
+// An empty reported fp (pre-v0.48.0 / local-API disabled) is unknown — never seeded, never an alert,
+// and never overwrites an existing baseline.
+func TestHostLeafChecker_EmptyFpSkipped(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{}) // no leaf_fingerprint
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if lc.GetState("h1") != "" {
+		t.Fatalf("empty fp must not seed a baseline, got %q", lc.GetState("h1"))
+	}
+	lc.Check()
+	if len(events) != 0 {
+		t.Fatalf("empty fp must not alert, got %v", events)
+	}
+	// A real fp later → seeds (no alert, it's the first real baseline).
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	lc.Check()
+	if len(events) != 0 || lc.GetState("h1") != fpA {
+		t.Fatalf("first real fp should seed silently, events=%v state=%q", events, lc.GetState("h1"))
+	}
+}
+
+// A blocked customer's host is dropped — no event.
+func TestHostLeafChecker_CustomerBlockedDropped(t *testing.T) {
+	st := newLeafStore(t)
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpA), store.HostReportDenorm{})
+	var events []string
+	lc := NewHostLeafChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if err := st.SetCustomerConfigStatus("c1", "blocked"); err != nil {
+		t.Fatalf("block customer: %v", err)
+	}
+	st.SaveHostReport("h1", "c1", reportWithLeaf(fpB), store.HostReportDenorm{}) // would-be change
+	lc.Check()
+	if len(events) != 0 {
+		t.Fatalf("blocked customer must not alert, got %v", events)
+	}
+	if lc.GetState("h1") != "" {
+		t.Fatalf("blocked host should be dropped, got %q", lc.GetState("h1"))
+	}
+}

hub/internal/store/store.go

@@ -1673,3 +1673,41 @@ func (s *Store) GetHostCapabilities() ([]HostCapabilityRow, error) {
 	}
 	return out, rows.Err()
 }
+
+// HostLeafRow is the per-host served-leaf-fingerprint the HostLeafChecker reads — extracted from the
+// latest host-report's report_json (no dedicated column; the fp rides the report body, same as the
+// capabilities snapshot). LeafFP is "" for a pre-v0.48.0 agent / local-API-disabled host.
+type HostLeafRow struct {
+	HostID     string
+	CustomerID string
+	LeafFP     string
+}
+
+// GetHostLeafFingerprints returns the latest reported local-API leaf fp per host (mirrors
+// GetHostCapabilities — MAX(id) per host, parsed from report_json so there is no schema migration).
+func (s *Store) GetHostLeafFingerprints() ([]HostLeafRow, error) {
+	rows, err := s.db.Query(`
+		SELECT hr.host_id, hr.customer_id, hr.report_json
+		FROM host_reports hr
+		JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
+		  ON hr.id = latest.mx`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []HostLeafRow
+	for rows.Next() {
+		var r HostLeafRow
+		var reportJSON string
+		if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
+			return nil, err
+		}
+		var body struct {
+			LeafFingerprint string `json:"leaf_fingerprint"`
+		}
+		_ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old body → "" → no alert
+		r.LeafFP = body.LeafFingerprint
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}