fix(hub): mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1) — v0.34.1

A heal marker is an event, not a baseline: construction seeds pre-existing markers
(startup false-alarm guard) but a newly-observed marker now raises the warning, so
the first auto-heal surfaces (matches the live drill). Added tests for both halves.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-05 19:12:50 +02:00
parent 1bd054602d
commit 012e5f3ecc
4 changed files with 57 additions and 18 deletions
+7
View File
@@ -1,5 +1,12 @@
# Felhom Hub — Changelog
## v0.34.1 — mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1 fix) (2026-07-05)
The mgmt-plane checker seeded a heal marker silently on first observation (copied from HostLeafChecker's
trust-on-first-report). A heal is an EVENT, not a baseline: construction still seeds pre-existing markers
(startup false-alarm guard), but a newly-observed marker now raises the warning — so the FIRST auto-heal
surfaces, matching the live drill. Added tests for both halves.
## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05)
The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half
+14 -17
View File
@@ -14,12 +14,13 @@ import (
// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
//
// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
// Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites
// the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE
// HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed
// marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub
// startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest
// reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one
// warning. An empty timestamp (healthy host / old agent) never alerts.
type HostMgmtPlaneChecker struct {
store *store.Store
logger *log.Logger
@@ -74,20 +75,16 @@ func (mc *HostMgmtPlaneChecker) Check() {
continue
}
seen[row.HostID] = true
if row.PrivsepHealedAt == "" {
continue // no heal marker → healthy / old agent → no alert, no baseline change
}
mc.customerOf[row.HostID] = row.CustomerID
old := mc.states[row.HostID]
if old == "" {
mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
continue
newHealed := row.PrivsepHealedAt
if newHealed == mc.states[row.HostID] {
continue // unchanged from last-seen (incl. both empty) → nothing to do
}
if old == row.PrivsepHealedAt {
continue // same heal already alerted
mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it)
if newHealed != "" {
// A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert.
mc.emit(row.HostID, row.CustomerID, newHealed)
}
mc.states[row.HostID] = row.PrivsepHealedAt
mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
}
for id := range mc.states {
@@ -58,6 +58,41 @@ func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
}
}
// The live-drill path: the hub started while the host was HEALTHY (no marker seeded), then the
// watchdog auto-heals and the marker appears in a later report → the FIRST observation alerts (a heal
// is an event, not a baseline). This is the case a pure trust-on-first-report would have missed.
func TestHostMgmtPlaneChecker_FirstHealAfterHealthyAlerts(t *testing.T) {
st := newMgmtStore(t)
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy at construction
var events []string
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
if len(events) != 0 || mc.GetState("h1") != "" {
t.Fatalf("healthy construction: no seed/event, got state=%q events=%v", mc.GetState("h1"), events)
}
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T17:07:39Z"), store.HostReportDenorm{}) // heal happens
mc.Check()
if len(events) != 1 || events[0] != "mgmt_plane_healed" {
t.Fatalf("first heal after healthy must alert once, got %v", events)
}
mc.Check() // same marker → no duplicate
if len(events) != 1 {
t.Fatalf("no duplicate on the same marker, got %v", events)
}
}
// A marker PRESENT at construction is seeded silently (avoid a false alarm on hub restart for a heal
// that predates the hub watching) — the construction-no-event half of NewHealAlertsOnce, isolated.
func TestHostMgmtPlaneChecker_PreexistingMarkerSeededSilently(t *testing.T) {
st := newMgmtStore(t)
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T09:00:00Z"), store.HostReportDenorm{})
var events []string
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
mc.Check() // same marker still latest → no alert (it was seeded)
if len(events) != 0 {
t.Fatalf("a marker present at construction must NOT alert (startup false-alarm guard), got %v", events)
}
}
func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
st := newMgmtStore(t)
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
+1 -1
View File
@@ -126,7 +126,7 @@ spec:
spec:
containers:
- name: hub
image: gitea.dooplex.hu/admin/felhom-hub:0.34.0
image: gitea.dooplex.hu/admin/felhom-hub:0.34.1
ports:
- containerPort: 8080
name: http