75 lines
3.0 KiB
Go
75 lines
3.0 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
// R-1344 (operator brief 2026-10-29, Part 2): a controller that RUNS but whose Docker health check reads „unhealthy" on
|
|
// controllerUnresponsiveConfirm consecutive sweeps is reported unresponsive (the stanza; the hub raises
|
|
// controller_unresponsive). ALARM ONLY: it is never restarted for it until the measurement licenses that.
|
|
// RED-PROOF: seen failing before the stanza carried it.
|
|
|
|
func unrespServer(t *testing.T, health string) (*Server, *opExec, *fakeGuestPowerCtl) {
|
|
t.Helper()
|
|
s, ex, ctl, _, _, _ := opServer(t, "running")
|
|
ex.health = health
|
|
return s, ex, ctl
|
|
}
|
|
|
|
func sweep(s *Server, n int) {
|
|
for i := 0; i < n; i++ {
|
|
s.ControllerSupervisorTick(context.Background())
|
|
}
|
|
}
|
|
|
|
func TestSupervisor_UnresponsiveIsAlarmedNeverRestarted(t *testing.T) {
|
|
s, ex, _ := unrespServer(t, "unhealthy")
|
|
sweep(s, controllerUnresponsiveConfirm-1)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) != 1 || g[0].Unresponsive {
|
|
t.Fatalf("unresponsive before %d sweeps: %+v", controllerUnresponsiveConfirm, g)
|
|
}
|
|
sweep(s, 1)
|
|
g := s.ControllerSupervisorStatus(context.Background()).Guests
|
|
if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveSince == "" {
|
|
t.Fatalf("after %d unhealthy sweeps: %+v", controllerUnresponsiveConfirm, g)
|
|
}
|
|
since := g[0].UnresponsiveSince
|
|
sweep(s, 10)
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("an unresponsive controller was RESTARTED (%d) — the restart waits for the measurement", n)
|
|
}
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].UnresponsiveSince != since {
|
|
t.Fatal("unresponsive_since moved while it stayed unresponsive — the hub would mail again")
|
|
}
|
|
// it answers again
|
|
ex.health = "healthy"
|
|
sweep(s, 1)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].Unresponsive || g[0].UnresponsiveSince != since {
|
|
t.Fatalf("after it answered: %+v (unresponsive must clear; since stays as the record)", g[0])
|
|
}
|
|
}
|
|
|
|
// Slow but answering (healthy / starting), a swap, a vzdump or a parked guest never counts.
|
|
func TestSupervisor_UnresponsiveGuards(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
setup func(s *Server, ex *opExec, ctl *fakeGuestPowerCtl)
|
|
}{
|
|
{"healthy", func(_ *Server, ex *opExec, _ *fakeGuestPowerCtl) { ex.health = "healthy" }},
|
|
{"starting", func(_ *Server, ex *opExec, _ *fakeGuestPowerCtl) { ex.health = "starting" }},
|
|
{"swap", func(s *Server, _ *opExec, _ *fakeGuestPowerCtl) { s.swapInFlight[9201] = true }},
|
|
{"vzdump", func(_ *Server, _ *opExec, ctl *fakeGuestPowerCtl) { ctl.backupRun = map[int]bool{9201: true} }},
|
|
{"locked", func(_ *Server, _ *opExec, ctl *fakeGuestPowerCtl) { ctl.locks[9201] = "backup" }},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, ex, ctl := unrespServer(t, "unhealthy")
|
|
tc.setup(s, ex, ctl)
|
|
sweep(s, controllerUnresponsiveConfirm+2)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) == 1 && g[0].Unresponsive {
|
|
t.Fatalf("%s counted as unresponsive: %+v", tc.name, g[0])
|
|
}
|
|
})
|
|
}
|
|
}
|