9c5cf2975f
Watches each host's reported local-API leaf fp; alerts on change (trust-on-first-report). Sibling of HostCapabilityChecker; store.GetHostLeafFingerprints reads report_json (no migration); hub-generated event (no allowlist change). Change red-proof + first-obs-seed + empty-skip + blocked-drop tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
132 lines
4.4 KiB
Go
132 lines
4.4 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log"
|
|
"sync"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// HostLeafChecker raises an operator alert when a host's agent local-API LEAF FINGERPRINT changes — a
|
|
// proactive, fleet-wide agent-re-key detector (the last self-health leg). It is a sibling of
|
|
// HostCapabilityChecker and independent of any controller's channel-health check: it catches a re-key
|
|
// even if a controller is down / hasn't probed, and gives the operator one fleet view.
|
|
//
|
|
// Design (Option 1 — trust-on-first-report): the first leaf fp seen per host is the baseline; any later
|
|
// change alerts and advances the baseline. KNOWN LIMITATION (v1, acceptable): if the agent re-keyed
|
|
// BEFORE the hub's first report, the hub trusts the wrong baseline — but the controller channel-check
|
|
// still catches the downstream pin mismatch, so this is defense-in-depth, not the sole guard. (The
|
|
// authoritative served-fp-vs-pinned-fp cross-check is a deliberate future enhancement.)
|
|
//
|
|
// An empty reported fp (pre-v0.48.0 agent, or the local API disabled) is "unknown" — never an alert and
|
|
// never overwrites a baseline.
|
|
type HostLeafChecker struct {
|
|
store *store.Store
|
|
logger *log.Logger
|
|
onEvent EventNotifyFunc
|
|
|
|
mu sync.Mutex
|
|
states map[string]string // hostID → last-seen leaf fp (the baseline)
|
|
customerOf map[string]string // hostID → customerID (event attribution)
|
|
}
|
|
|
|
// NewHostLeafChecker seeds the per-host baseline from the latest reported fps. No events on init.
|
|
func NewHostLeafChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostLeafChecker {
|
|
lc := &HostLeafChecker{
|
|
store: s,
|
|
logger: logger,
|
|
onEvent: onEvent,
|
|
states: make(map[string]string),
|
|
customerOf: make(map[string]string),
|
|
}
|
|
rows, err := s.GetHostLeafFingerprints()
|
|
if err != nil {
|
|
logger.Printf("[WARN] Host leaf checker: failed to seed: %v", err)
|
|
return lc
|
|
}
|
|
seeded := 0
|
|
for _, row := range rows {
|
|
if s.IsCustomerBlocked(row.CustomerID) || row.LeafFP == "" {
|
|
continue
|
|
}
|
|
lc.customerOf[row.HostID] = row.CustomerID
|
|
lc.states[row.HostID] = row.LeafFP
|
|
seeded++
|
|
}
|
|
logger.Printf("[INFO] Host leaf checker initialized: %d host fingerprint(s) seeded", seeded)
|
|
return lc
|
|
}
|
|
|
|
// Check evaluates all hosts and emits host_leaf_changed on a fingerprint change. Call on the same 60s
|
|
// sweep as the staleness/capability checkers.
|
|
func (lc *HostLeafChecker) Check() {
|
|
rows, err := lc.store.GetHostLeafFingerprints()
|
|
if err != nil {
|
|
lc.logger.Printf("[WARN] Host leaf check failed: %v", err)
|
|
return
|
|
}
|
|
lc.mu.Lock()
|
|
defer lc.mu.Unlock()
|
|
|
|
seen := make(map[string]bool, len(rows))
|
|
for _, row := range rows {
|
|
if lc.store.IsCustomerBlocked(row.CustomerID) {
|
|
delete(lc.states, row.HostID)
|
|
continue
|
|
}
|
|
seen[row.HostID] = true // the host exists; keep its baseline even if this report's fp is empty
|
|
if row.LeafFP == "" {
|
|
continue // unknown fp → no alert, no baseline change
|
|
}
|
|
lc.customerOf[row.HostID] = row.CustomerID
|
|
old := lc.states[row.HostID]
|
|
if old == "" {
|
|
lc.states[row.HostID] = row.LeafFP // first observation → baseline, no event
|
|
continue
|
|
}
|
|
if old == row.LeafFP {
|
|
continue
|
|
}
|
|
lc.states[row.HostID] = row.LeafFP // advance the baseline (a change-back later re-alerts)
|
|
lc.emit(row.HostID, row.CustomerID, old, row.LeafFP)
|
|
}
|
|
|
|
for id := range lc.states {
|
|
if !seen[id] {
|
|
delete(lc.states, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// GetState returns the current baseline fp for a host ("" if unseen).
|
|
func (lc *HostLeafChecker) GetState(hostID string) string {
|
|
lc.mu.Lock()
|
|
defer lc.mu.Unlock()
|
|
return lc.states[hostID]
|
|
}
|
|
|
|
func (lc *HostLeafChecker) emit(hostID, customerID, oldFP, newFP string) {
|
|
msg := "Host " + hostID + ": agent local-API leaf fingerprint CHANGED (agent re-keyed) — controllers will fail the pin check until re-bootstrapped / re-pinned"
|
|
details, _ := json.Marshal(map[string]string{
|
|
"host_id": hostID,
|
|
"old_fingerprint": oldFP,
|
|
"new_fingerprint": newFP,
|
|
})
|
|
lc.logger.Printf("[WARN] Host leaf: %s fp %s… → %s… (host_leaf_changed)", hostID, shortFP(oldFP), shortFP(newFP))
|
|
if _, err := lc.store.SaveEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub"); err != nil {
|
|
lc.logger.Printf("[WARN] save host_leaf_changed for %s: %v", hostID, err)
|
|
return
|
|
}
|
|
if lc.onEvent != nil {
|
|
lc.onEvent(customerID, "host_leaf_changed", "warning", msg, string(details), "hub")
|
|
}
|
|
}
|
|
|
|
func shortFP(fp string) string {
|
|
if len(fp) > 12 {
|
|
return fp[:12]
|
|
}
|
|
return fp
|
|
}
|