R-1352: the does-not-answer alarm is not mailed after the restart mail of the same episode. Hub commit only.
gates / gates (push) Successful in 1m56s
gates / gates (push) Successful in 1m56s
This commit is contained in:
1 parent
a560e69954
commit
2f7e836f68
3 files changed
+74
-1
No files matched your search
@@ -1,3 +1,11 @@
|
||||
## Unreleased — next hub release; commit only, 2026-11-02 night
|
||||
|
||||
- **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2
|
||||
minutes apart and reports 15, so both reached the hub in one report; the operator read the restart first and then „the agent
|
||||
restarts it in 2 minutes" (seen live 2026-10-30, R-1344's proof). The restart's mail says all of it (since when, restarted when), so
|
||||
the alarm of that episode is not emitted. A new episode that began after the restart still alarms, and a time that cannot be read
|
||||
counts as a new episode: the alarm goes out. Tests `TestR1352_*`, the first seen failing before the fix.
|
||||
|
||||
## v0.178.0 (2026-10-30) — the hub's database speed (R-1346); the agent's restart of a controller that does not answer (R-1344)
|
||||
|
||||
- **R-1346 — measured, then fixed (evidence `documentation/audits/release-2026-10-30/p2-r1346/`).** The hub pod has
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
@@ -140,11 +141,24 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
|
||||
c.emit(customerID, hostID, g, "controller_slow_crashloop")
|
||||
}
|
||||
if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince {
|
||||
c.emit(customerID, hostID, g, "controller_unresponsive")
|
||||
// R-1352: the alarm and the restart of ONE episode lie 2 minutes apart and reports 15, so both can arrive together.
|
||||
// The restart's mail says all of it (since when, restarted when); an alarm after it („restarts it in 2 minutes")
|
||||
// reads backwards. A new episode after that restart has had no restart yet, and alarms.
|
||||
if !(unrespRestart && episodeRestarted(g.UnresponsiveSince, g.UnresponsiveRestartAt)) {
|
||||
c.emit(customerID, hostID, g, "controller_unresponsive")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// episodeRestarted reports whether the unresponsive episode that began at since is the one restarted at restartAt. A time
|
||||
// that cannot be read counts as „no": the alarm is mailed, the safe direction.
|
||||
func episodeRestarted(since, restartAt string) bool {
|
||||
s, err1 := time.Parse(time.RFC3339, since)
|
||||
r, err2 := time.Parse(time.RFC3339, restartAt)
|
||||
return err1 == nil && err2 == nil && !s.After(r)
|
||||
}
|
||||
|
||||
func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) {
|
||||
var severity, message string
|
||||
switch eventType {
|
||||
|
||||
@@ -221,3 +221,54 @@ func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R-1352: the alarm and the restart of ONE episode reach the hub in one report (they lie 2 minutes apart, reports 15). The
|
||||
// restart mail says all of it; the alarm „the agent restarts it in 2 minutes" after it reads backwards and is not mailed.
|
||||
// Seen live 2026-10-30 (R-1344's proof on Tester 1).
|
||||
//
|
||||
// RED-PROOF (2026-11-02): before the fix this fails on „got [controller_unresponsive_restart/warning
|
||||
// controller_unresponsive/warning]".
|
||||
func TestR1352_TheAlarmOfAnEpisodeIsNotMailedAfterItsRestart(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
rec := &evRec{}
|
||||
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
rec.take()
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
|
||||
t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
|
||||
}
|
||||
}
|
||||
|
||||
// A NEW episode that began after the restart, in the same report, still alarms: it has had no restart yet.
|
||||
func TestR1352_ANewEpisodeAfterTheRestartStillAlarms(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
rec := &evRec{}
|
||||
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
rec.take()
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T09:31:22Z","last_reason":"did not answer","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:40:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T09:31:22Z"}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
evs := strings.Join(rec.take(), " ")
|
||||
if !strings.Contains(evs, "controller_unresponsive_restart/warning") || !strings.Contains(evs, "controller_unresponsive/warning") {
|
||||
t.Fatalf("got %q, want the restart AND the new episode's alarm", evs)
|
||||
}
|
||||
}
|
||||
|
||||
// An alarm with no restart is unchanged (control).
|
||||
func TestR1352_AnAlarmAloneStillMails(t *testing.T) {
|
||||
st := newCapStore(t)
|
||||
rec := &evRec{}
|
||||
c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
rec.take()
|
||||
st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T09:29:21Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
|
||||
c.Check()
|
||||
if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive/warning" {
|
||||
t.Fatalf("got %v, want exactly the alarm", evs)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user