fix(hub): mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1) — v0.34.1
A heal marker is an event, not a baseline: construction seeds pre-existing markers (startup false-alarm guard) but a newly-observed marker now raises the warning, so the first auto-heal surfaces (matches the live drill). Added tests for both halves. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -14,12 +14,13 @@ import (
|
||||
// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
|
||||
// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
|
||||
//
|
||||
// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
|
||||
// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
|
||||
// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
|
||||
// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
|
||||
// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
|
||||
// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
|
||||
// Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites
|
||||
// the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE
|
||||
// HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed
|
||||
// marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub
|
||||
// startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest
|
||||
// reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one
|
||||
// warning. An empty timestamp (healthy host / old agent) never alerts.
|
||||
type HostMgmtPlaneChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
@@ -74,20 +75,16 @@ func (mc *HostMgmtPlaneChecker) Check() {
|
||||
continue
|
||||
}
|
||||
seen[row.HostID] = true
|
||||
if row.PrivsepHealedAt == "" {
|
||||
continue // no heal marker → healthy / old agent → no alert, no baseline change
|
||||
}
|
||||
mc.customerOf[row.HostID] = row.CustomerID
|
||||
old := mc.states[row.HostID]
|
||||
if old == "" {
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
|
||||
continue
|
||||
newHealed := row.PrivsepHealedAt
|
||||
if newHealed == mc.states[row.HostID] {
|
||||
continue // unchanged from last-seen (incl. both empty) → nothing to do
|
||||
}
|
||||
if old == row.PrivsepHealedAt {
|
||||
continue // same heal already alerted
|
||||
mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it)
|
||||
if newHealed != "" {
|
||||
// A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert.
|
||||
mc.emit(row.HostID, row.CustomerID, newHealed)
|
||||
}
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt
|
||||
mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
|
||||
}
|
||||
|
||||
for id := range mc.states {
|
||||
|
||||
Reference in New Issue
Block a user