fix(hub): mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1) — v0.34.1
A heal marker is an event, not a baseline: construction seeds pre-existing markers (startup false-alarm guard) but a newly-observed marker now raises the warning, so the first auto-heal surfaces (matches the live drill). Added tests for both halves. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -1,5 +1,12 @@
|
||||
# Felhom Hub — Changelog
|
||||
|
||||
## v0.34.1 — mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1 fix) (2026-07-05)
|
||||
|
||||
The mgmt-plane checker seeded a heal marker silently on first observation (copied from HostLeafChecker's
|
||||
trust-on-first-report). A heal is an EVENT, not a baseline: construction still seeds pre-existing markers
|
||||
(startup false-alarm guard), but a newly-observed marker now raises the warning — so the FIRST auto-heal
|
||||
surfaces, matching the live drill. Added tests for both halves.
|
||||
|
||||
## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05)
|
||||
|
||||
The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half
|
||||
|
||||
@@ -14,12 +14,13 @@ import (
|
||||
// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
|
||||
// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
|
||||
//
|
||||
// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
|
||||
// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
|
||||
// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
|
||||
// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
|
||||
// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
|
||||
// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
|
||||
// Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites
|
||||
// the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE
|
||||
// HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed
|
||||
// marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub
|
||||
// startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest
|
||||
// reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one
|
||||
// warning. An empty timestamp (healthy host / old agent) never alerts.
|
||||
type HostMgmtPlaneChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
@@ -74,20 +75,16 @@ func (mc *HostMgmtPlaneChecker) Check() {
|
||||
continue
|
||||
}
|
||||
seen[row.HostID] = true
|
||||
if row.PrivsepHealedAt == "" {
|
||||
continue // no heal marker → healthy / old agent → no alert, no baseline change
|
||||
}
|
||||
mc.customerOf[row.HostID] = row.CustomerID
|
||||
old := mc.states[row.HostID]
|
||||
if old == "" {
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
|
||||
continue
|
||||
newHealed := row.PrivsepHealedAt
|
||||
if newHealed == mc.states[row.HostID] {
|
||||
continue // unchanged from last-seen (incl. both empty) → nothing to do
|
||||
}
|
||||
if old == row.PrivsepHealedAt {
|
||||
continue // same heal already alerted
|
||||
mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it)
|
||||
if newHealed != "" {
|
||||
// A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert.
|
||||
mc.emit(row.HostID, row.CustomerID, newHealed)
|
||||
}
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt
|
||||
mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
|
||||
}
|
||||
|
||||
for id := range mc.states {
|
||||
|
||||
@@ -58,6 +58,41 @@ func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The live-drill path: the hub started while the host was HEALTHY (no marker seeded), then the
|
||||
// watchdog auto-heals and the marker appears in a later report → the FIRST observation alerts (a heal
|
||||
// is an event, not a baseline). This is the case a pure trust-on-first-report would have missed.
|
||||
func TestHostMgmtPlaneChecker_FirstHealAfterHealthyAlerts(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy at construction
|
||||
var events []string
|
||||
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
if len(events) != 0 || mc.GetState("h1") != "" {
|
||||
t.Fatalf("healthy construction: no seed/event, got state=%q events=%v", mc.GetState("h1"), events)
|
||||
}
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T17:07:39Z"), store.HostReportDenorm{}) // heal happens
|
||||
mc.Check()
|
||||
if len(events) != 1 || events[0] != "mgmt_plane_healed" {
|
||||
t.Fatalf("first heal after healthy must alert once, got %v", events)
|
||||
}
|
||||
mc.Check() // same marker → no duplicate
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("no duplicate on the same marker, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// A marker PRESENT at construction is seeded silently (avoid a false alarm on hub restart for a heal
|
||||
// that predates the hub watching) — the construction-no-event half of NewHealAlertsOnce, isolated.
|
||||
func TestHostMgmtPlaneChecker_PreexistingMarkerSeededSilently(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T09:00:00Z"), store.HostReportDenorm{})
|
||||
var events []string
|
||||
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
mc.Check() // same marker still latest → no alert (it was seeded)
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("a marker present at construction must NOT alert (startup false-alarm guard), got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: hub
|
||||
image: gitea.dooplex.hu/admin/felhom-hub:0.34.0
|
||||
image: gitea.dooplex.hu/admin/felhom-hub:0.34.1
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
|
||||
Reference in New Issue
Block a user