hub: HostLeafChecker — proactive agent re-key detection (host_leaf_changed) v0.22.0

Watches each host's reported local-API leaf fp; alerts on change (trust-on-first-report). Sibling of
HostCapabilityChecker; store.GetHostLeafFingerprints reads report_json (no migration); hub-generated
event (no allowlist change). Change red-proof + first-obs-seed + empty-skip + blocked-drop tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ

hub/internal/monitor/host_leaf.go

@@ -0,0 +1,131 @@
+package monitor
+
+import (
+	"encoding/json"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// HostLeafChecker raises an operator alert when a host's agent local-API LEAF FINGERPRINT changes — a
+// proactive, fleet-wide agent-re-key detector (the last self-health leg). It is a sibling of
+// HostCapabilityChecker and independent of any controller's channel-health check: it catches a re-key
+// even if a controller is down / hasn't probed, and gives the operator one fleet view.
+//
+// Design (Option 1 — trust-on-first-report): the first leaf fp seen per host is the baseline; any later
+// change alerts and advances the baseline. KNOWN LIMITATION (v1, acceptable): if the agent re-keyed
+// BEFORE the hub's first report, the hub trusts the wrong baseline — but the controller channel-check
+// still catches the downstream pin mismatch, so this is defense-in-depth, not the sole guard. (The
+// authoritative served-fp-vs-pinned-fp cross-check is a deliberate future enhancement.)
+//
+// An empty reported fp (pre-v0.48.0 agent, or the local API disabled) is "unknown" — never an alert and
+// never overwrites a baseline.
+type HostLeafChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	states     map[string]string // hostID → last-seen leaf fp (the baseline)
+	customerOf map[string]string // hostID → customerID (event attribution)
+}
+
+// NewHostLeafChecker seeds the per-host baseline from the latest reported fps. No events on init.
+func NewHostLeafChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostLeafChecker {
+	lc := &HostLeafChecker{
+		store:      s,
+		logger:     logger,
+		onEvent:    onEvent,
+		states:     make(map[string]string),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostLeafFingerprints()
+	if err != nil {
+		logger.Printf("[WARN] Host leaf checker: failed to seed: %v", err)
+		return lc
+	}
+	seeded := 0
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) || row.LeafFP == "" {
+			continue
+		}
+		lc.customerOf[row.HostID] = row.CustomerID
+		lc.states[row.HostID] = row.LeafFP
+		seeded++
+	}
+	logger.Printf("[INFO] Host leaf checker initialized: %d host fingerprint(s) seeded", seeded)
+	return lc
+}
+
+// Check evaluates all hosts and emits host_leaf_changed on a fingerprint change. Call on the same 60s
+// sweep as the staleness/capability checkers.
+func (lc *HostLeafChecker) Check() {
+	rows, err := lc.store.GetHostLeafFingerprints()
+	if err != nil {
+		lc.logger.Printf("[WARN] Host leaf check failed: %v", err)
+		return
+	}
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if lc.store.IsCustomerBlocked(row.CustomerID) {
+			delete(lc.states, row.HostID)
+			continue
+		}
+		seen[row.HostID] = true // the host exists; keep its baseline even if this report's fp is empty
+		if row.LeafFP == "" {
+			continue // unknown fp → no alert, no baseline change
+		}
+		lc.customerOf[row.HostID] = row.CustomerID
+		old := lc.states[row.HostID]
+		if old == "" {
+			lc.states[row.HostID] = row.LeafFP // first observation → baseline, no event
+			continue
+		}
+		if old == row.LeafFP {
+			continue
+		}
+		lc.states[row.HostID] = row.LeafFP // advance the baseline (a change-back later re-alerts)
+		lc.emit(row.HostID, row.CustomerID, old, row.LeafFP)
+	}
+
+	for id := range lc.states {
+		if !seen[id] {
+			delete(lc.states, id)
+		}
+	}
+}
+
+// GetState returns the current baseline fp for a host ("" if unseen).
+func (lc *HostLeafChecker) GetState(hostID string) string {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	return lc.states[hostID]
+}
+
+func (lc *HostLeafChecker) emit(hostID, customerID, oldFP, newFP string) {
+	msg := "Host " + hostID + ": agent local-API leaf fingerprint CHANGED (agent re-keyed) — controllers will fail the pin check until re-bootstrapped / re-pinned"
+	details, _ := json.Marshal(map[string]string{
+		"host_id":         hostID,
+		"old_fingerprint": oldFP,
+		"new_fingerprint": newFP,
+	})
+	lc.logger.Printf("[WARN] Host leaf: %s fp %s… → %s… (host_leaf_changed)", hostID, shortFP(oldFP), shortFP(newFP))
+	if _, err := lc.store.SaveEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub"); err != nil {
+		lc.logger.Printf("[WARN] save host_leaf_changed for %s: %v", hostID, err)
+		return
+	}
+	if lc.onEvent != nil {
+		lc.onEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub")
+	}
+}
+
+func shortFP(fp string) string {
+	if len(fp) > 12 {
+		return fp[:12]
+	}
+	return fp
+}