From 012e5f3ecc6b82895d3c2c3af84ea7004691f9a2 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 5 Jul 2026 19:12:50 +0200 Subject: [PATCH] =?UTF-8?q?fix(hub):=20mgmt=5Fplane=5Fhealed=20alerts=20on?= =?UTF-8?q?=20the=20FIRST=20auto-heal=20(TASK=20G1)=20=E2=80=94=20v0.34.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A heal marker is an event, not a baseline: construction seeds pre-existing markers (startup false-alarm guard) but a newly-observed marker now raises the warning, so the first auto-heal surfaces (matches the live drill). Added tests for both halves. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- hub/CHANGELOG.md | 7 +++++ hub/internal/monitor/host_mgmtplane.go | 31 +++++++++--------- hub/internal/monitor/host_mgmtplane_test.go | 35 +++++++++++++++++++++ manifests/hub.yaml | 2 +- 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 2bcc543..bfee25e 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,12 @@ # Felhom Hub — Changelog +## v0.34.1 — mgmt_plane_healed alerts on the FIRST auto-heal (TASK G1 fix) (2026-07-05) + +The mgmt-plane checker seeded a heal marker silently on first observation (copied from HostLeafChecker's +trust-on-first-report). A heal is an EVENT, not a baseline: construction still seeds pre-existing markers +(startup false-alarm guard), but a newly-observed marker now raises the warning — so the FIRST auto-heal +surfaces, matching the live drill. Added tests for both halves. + ## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05) The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half diff --git a/hub/internal/monitor/host_mgmtplane.go b/hub/internal/monitor/host_mgmtplane.go index 63b080d..33130c3 100644 --- a/hub/internal/monitor/host_mgmtplane.go +++ b/hub/internal/monitor/host_mgmtplane.go @@ -14,12 +14,13 @@ import ( // it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box // gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep. // -// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen -// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different -// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the -// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on -// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty -// timestamp (healthy host / old agent) never alerts and never overwrites a baseline. +// Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites +// the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE +// HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed +// marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub +// startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest +// reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one +// warning. An empty timestamp (healthy host / old agent) never alerts. type HostMgmtPlaneChecker struct { store *store.Store logger *log.Logger @@ -74,20 +75,16 @@ func (mc *HostMgmtPlaneChecker) Check() { continue } seen[row.HostID] = true - if row.PrivsepHealedAt == "" { - continue // no heal marker → healthy / old agent → no alert, no baseline change - } mc.customerOf[row.HostID] = row.CustomerID - old := mc.states[row.HostID] - if old == "" { - mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event - continue + newHealed := row.PrivsepHealedAt + if newHealed == mc.states[row.HostID] { + continue // unchanged from last-seen (incl. both empty) → nothing to do } - if old == row.PrivsepHealedAt { - continue // same heal already alerted + mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it) + if newHealed != "" { + // A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert. + mc.emit(row.HostID, row.CustomerID, newHealed) } - mc.states[row.HostID] = row.PrivsepHealedAt - mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt) } for id := range mc.states { diff --git a/hub/internal/monitor/host_mgmtplane_test.go b/hub/internal/monitor/host_mgmtplane_test.go index be4637e..2c08193 100644 --- a/hub/internal/monitor/host_mgmtplane_test.go +++ b/hub/internal/monitor/host_mgmtplane_test.go @@ -58,6 +58,41 @@ func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) { } } +// The live-drill path: the hub started while the host was HEALTHY (no marker seeded), then the +// watchdog auto-heals and the marker appears in a later report → the FIRST observation alerts (a heal +// is an event, not a baseline). This is the case a pure trust-on-first-report would have missed. +func TestHostMgmtPlaneChecker_FirstHealAfterHealthyAlerts(t *testing.T) { + st := newMgmtStore(t) + st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy at construction + var events []string + mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if len(events) != 0 || mc.GetState("h1") != "" { + t.Fatalf("healthy construction: no seed/event, got state=%q events=%v", mc.GetState("h1"), events) + } + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T17:07:39Z"), store.HostReportDenorm{}) // heal happens + mc.Check() + if len(events) != 1 || events[0] != "mgmt_plane_healed" { + t.Fatalf("first heal after healthy must alert once, got %v", events) + } + mc.Check() // same marker → no duplicate + if len(events) != 1 { + t.Fatalf("no duplicate on the same marker, got %v", events) + } +} + +// A marker PRESENT at construction is seeded silently (avoid a false alarm on hub restart for a heal +// that predates the hub watching) — the construction-no-event half of NewHealAlertsOnce, isolated. +func TestHostMgmtPlaneChecker_PreexistingMarkerSeededSilently(t *testing.T) { + st := newMgmtStore(t) + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T09:00:00Z"), store.HostReportDenorm{}) + var events []string + mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + mc.Check() // same marker still latest → no alert (it was seeded) + if len(events) != 0 { + t.Fatalf("a marker present at construction must NOT alert (startup false-alarm guard), got %v", events) + } +} + func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) { st := newMgmtStore(t) st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker diff --git a/manifests/hub.yaml b/manifests/hub.yaml index 108a40b..f24b0cd 100644 --- a/manifests/hub.yaml +++ b/manifests/hub.yaml @@ -126,7 +126,7 @@ spec: spec: containers: - name: hub - image: gitea.dooplex.hu/admin/felhom-hub:0.34.0 + image: gitea.dooplex.hu/admin/felhom-hub:0.34.1 ports: - containerPort: 8080 name: http