From 2f7e836f6818b718bf7a7d7f4928445b93e12d8e Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 2 Nov 2026 23:16:16 +0100 Subject: [PATCH] R-1352: the does-not-answer alarm is not mailed after the restart mail of the same episode. Hub commit only. --- hub/CHANGELOG.md | 8 +++ hub/internal/monitor/controller_supervisor.go | 16 +++++- .../monitor/controller_supervisor_test.go | 51 +++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 70b41bb8..4255f8f8 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,11 @@ +## Unreleased — next hub release; commit only, 2026-11-02 night + +- **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2 + minutes apart and reports 15, so both reached the hub in one report; the operator read the restart first and then „the agent + restarts it in 2 minutes" (seen live 2026-10-30, R-1344's proof). The restart's mail says all of it (since when, restarted when), so + the alarm of that episode is not emitted. A new episode that began after the restart still alarms, and a time that cannot be read + counts as a new episode: the alarm goes out. Tests `TestR1352_*`, the first seen failing before the fix. + ## v0.178.0 (2026-10-30) — the hub's database speed (R-1346); the agent's restart of a controller that does not answer (R-1344) - **R-1346 — measured, then fixed (evidence `documentation/audits/release-2026-10-30/p2-r1346/`).** The hub pod has diff --git a/hub/internal/monitor/controller_supervisor.go b/hub/internal/monitor/controller_supervisor.go index d04eb7be..58c81f01 100644 --- a/hub/internal/monitor/controller_supervisor.go +++ b/hub/internal/monitor/controller_supervisor.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "sync" + "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) @@ -140,11 +141,24 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests c.emit(customerID, hostID, g, "controller_slow_crashloop") } if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince { - c.emit(customerID, hostID, g, "controller_unresponsive") + // R-1352: the alarm and the restart of ONE episode lie 2 minutes apart and reports 15, so both can arrive together. + // The restart's mail says all of it (since when, restarted when); an alarm after it („restarts it in 2 minutes") + // reads backwards. A new episode after that restart has had no restart yet, and alarms. + if !(unrespRestart && episodeRestarted(g.UnresponsiveSince, g.UnresponsiveRestartAt)) { + c.emit(customerID, hostID, g, "controller_unresponsive") + } } } } +// episodeRestarted reports whether the unresponsive episode that began at since is the one restarted at restartAt. A time +// that cannot be read counts as „no": the alarm is mailed, the safe direction. +func episodeRestarted(since, restartAt string) bool { + s, err1 := time.Parse(time.RFC3339, since) + r, err2 := time.Parse(time.RFC3339, restartAt) + return err1 == nil && err2 == nil && !s.After(r) +} + func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) { var severity, message string switch eventType { diff --git a/hub/internal/monitor/controller_supervisor_test.go b/hub/internal/monitor/controller_supervisor_test.go index f793c18a..be3ef0fe 100644 --- a/hub/internal/monitor/controller_supervisor_test.go +++ b/hub/internal/monitor/controller_supervisor_test.go @@ -221,3 +221,54 @@ func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing. } } } + +// R-1352: the alarm and the restart of ONE episode reach the hub in one report (they lie 2 minutes apart, reports 15). The +// restart mail says all of it; the alarm „the agent restarts it in 2 minutes" after it reads backwards and is not mailed. +// Seen live 2026-10-30 (R-1344's proof on Tester 1). +// +// RED-PROOF (2026-11-02): before the fix this fails on „got [controller_unresponsive_restart/warning +// controller_unresponsive/warning]". +func TestR1352_TheAlarmOfAnEpisodeIsNotMailedAfterItsRestart(t *testing.T) { + st := newCapStore(t) + rec := &evRec{} + c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{}) + c.Check() + rec.take() + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{}) + c.Check() + if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" { + t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs) + } +} + +// A NEW episode that began after the restart, in the same report, still alarms: it has had no restart yet. +func TestR1352_ANewEpisodeAfterTheRestartStillAlarms(t *testing.T) { + st := newCapStore(t) + rec := &evRec{} + c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{}) + c.Check() + rec.take() + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:40:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{}) + c.Check() + evs := strings.Join(rec.take(), " ") + if !strings.Contains(evs, "controller_unresponsive_restart/warning") || !strings.Contains(evs, "controller_unresponsive/warning") { + t.Fatalf("got %q, want the restart AND the new episode's alarm", evs) + } +} + +// An alarm with no restart is unchanged (control). +func TestR1352_AnAlarmAloneStillMails(t *testing.T) { + st := newCapStore(t) + rec := &evRec{} + c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{}) + c.Check() + rec.take() + st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{}) + c.Check() + if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" { + t.Fatalf("got %v, want exactly the alarm", evs) + } +}