R-1352: the does-not-answer alarm is not mailed after the restart mail of the same episode. Hub commit only.
gates / gates (push) Successful in 1m56s

This commit is contained in:
admin committed 2026-11-02 23:16:16 +01:00
1 parent a560e69954
commit 2f7e836f68
3 files changed
+74 -1

No files matched your search

+8
View File
@@ -1,3 +1,11 @@
## Unreleased — next hub release; commit only, 2026-11-02 night
- **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2
minutes apart and reports 15, so both reached the hub in one report; the operator read the restart first and then „the agent
restarts it in 2 minutes" (seen live 2026-10-30, R-1344's proof). The restart's mail says all of it (since when, restarted when), so
the alarm of that episode is not emitted. A new episode that began after the restart still alarms, and a time that cannot be read
counts as a new episode: the alarm goes out. Tests `TestR1352_*`, the first seen failing before the fix.
## v0.178.0 (2026-10-30) — the hub's database speed (R-1346); the agent's restart of a controller that does not answer (R-1344)
- **R-1346 — measured, then fixed (evidence `documentation/audits/release-2026-10-30/p2-r1346/`).** The hub pod has
+15 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
@@ -140,11 +141,24 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
c.emit(customerID, hostID, g, "controller_slow_crashloop")
}
if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince {
c.emit(customerID, hostID, g, "controller_unresponsive")
// R-1352: the alarm and the restart of ONE episode lie 2 minutes apart and reports 15, so both can arrive together.
// The restart's mail says all of it (since when, restarted when); an alarm after it („restarts it in 2 minutes")
// reads backwards. A new episode after that restart has had no restart yet, and alarms.
if !(unrespRestart && episodeRestarted(g.UnresponsiveSince, g.UnresponsiveRestartAt)) {
c.emit(customerID, hostID, g, "controller_unresponsive")
}
}
}
}
// episodeRestarted reports whether the unresponsive episode that began at since is the one restarted at restartAt. A time
// that cannot be read counts as „no": the alarm is mailed, the safe direction.
func episodeRestarted(since, restartAt string) bool {
s, err1 := time.Parse(time.RFC3339, since)
r, err2 := time.Parse(time.RFC3339, restartAt)
return err1 == nil && err2 == nil && !s.After(r)
}
func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) {
var severity, message string
switch eventType {
@@ -221,3 +221,54 @@ func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.
}
}
}
// R-1352: the alarm and the restart of ONE episode reach the hub in one report (they lie 2 minutes apart, reports 15). The
// restart mail says all of it; the alarm „the agent restarts it in 2 minutes" after it reads backwards and is not mailed.
// Seen live 2026-10-30 (R-1344's proof on Tester 1).
//
// RED-PROOF (2026-11-02): before the fix this fails on „got [controller_unresponsive_restart/warning
// controller_unresponsive/warning]".
func TestR1352_TheAlarmOfAnEpisodeIsNotMailedAfterItsRestart(t *testing.T) {
st := newCapStore(t)
rec := &evRec{}
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
c.Check()
rec.take()
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{})
c.Check()
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
}
}
// A NEW episode that began after the restart, in the same report, still alarms: it has had no restart yet.
func TestR1352_ANewEpisodeAfterTheRestartStillAlarms(t *testing.T) {
st := newCapStore(t)
rec := &evRec{}
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
c.Check()
rec.take()
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:40:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{})
c.Check()
evs := strings.Join(rec.take(), " ")
if !strings.Contains(evs, "controller_unresponsive_restart/warning") || !strings.Contains(evs, "controller_unresponsive/warning") {
t.Fatalf("got %q, want the restart AND the new episode's alarm", evs)
}
}
// An alarm with no restart is unchanged (control).
func TestR1352_AnAlarmAloneStillMails(t *testing.T) {
st := newCapStore(t)
rec := &evRec{}
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
c.Check()
rec.take()
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
c.Check()
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" {
t.Fatalf("got %v, want exactly the alarm", evs)
}
}