157 lines
6.9 KiB
Go
157 lines
6.9 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// R-1344 (operator brief 2026-10-29, Part 2; decided 2026-10-30 from the night's measurement): a controller that RUNS
|
|
// but whose Docker health check reads „unhealthy" on controllerUnresponsiveConfirm consecutive sweeps is reported
|
|
// unresponsive (the stanza; the hub raises controller_unresponsive). If it still reads „unhealthy" at
|
|
// controllerUnresponsiveRestart sweeps, the agent restarts it — with the not-running restart's guards and brake, plus
|
|
// the controller's own busy file. RED-PROOF: the restart tests seen failing before the restart existed.
|
|
|
|
func unrespServer(t *testing.T, health string) (*Server, *rbExec, *fakeGuestPowerCtl, *supClock) {
|
|
t.Helper()
|
|
s, ex, ctl, clk, _, _ := opServer(t, "running")
|
|
ex.health = health
|
|
rb := &rbExec{opExec: ex}
|
|
s.guestExec = rb
|
|
return s, rb, ctl, clk
|
|
}
|
|
|
|
func sweep(s *Server, n int) {
|
|
for i := 0; i < n; i++ {
|
|
s.ControllerSupervisorTick(context.Background())
|
|
}
|
|
}
|
|
|
|
func TestSupervisor_UnresponsiveIsAlarmedThenRestarted(t *testing.T) {
|
|
s, ex, _, _ := unrespServer(t, "unhealthy")
|
|
sweep(s, controllerUnresponsiveConfirm-1)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) != 1 || g[0].Unresponsive {
|
|
t.Fatalf("unresponsive before %d sweeps: %+v", controllerUnresponsiveConfirm, g)
|
|
}
|
|
sweep(s, 1)
|
|
g := s.ControllerSupervisorStatus(context.Background()).Guests
|
|
if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveSince == "" {
|
|
t.Fatalf("after %d unhealthy sweeps: %+v", controllerUnresponsiveConfirm, g)
|
|
}
|
|
if g[0].UnresponsiveRestartAfterS != (controllerUnresponsiveRestart-controllerUnresponsiveConfirm)*int(controllerSupervisorInterval/time.Second) {
|
|
t.Fatalf("the stanza must tell the hub when the restart comes: %+v", g[0])
|
|
}
|
|
since := g[0].UnresponsiveSince
|
|
sweep(s, controllerUnresponsiveRestart-controllerUnresponsiveConfirm-1)
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("restarted after %d unhealthy sweeps — the restart waits for %d", controllerUnresponsiveRestart-1, controllerUnresponsiveRestart)
|
|
}
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].UnresponsiveSince != since {
|
|
t.Fatal("unresponsive_since moved while it stayed unresponsive — the hub would mail again")
|
|
}
|
|
sweep(s, 1)
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("after %d unhealthy sweeps: %d restarts, want 1", controllerUnresponsiveRestart, n)
|
|
}
|
|
g = s.ControllerSupervisorStatus(context.Background()).Guests
|
|
if g[0].UnresponsiveRestartAt == "" || g[0].UnresponsiveRestartAt != g[0].LastRestartAt || g[0].Restarts24h != 1 ||
|
|
!strings.Contains(g[0].LastReason, "health check") {
|
|
t.Fatalf("the restart is not on the record the hub reads: %+v", g[0])
|
|
}
|
|
// A restart that did not help (still „unhealthy" on the next sweep) is not repeated at once: the count starts again.
|
|
sweep(s, 1)
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("restarted again on the sweep after a restart: %d restarts", n)
|
|
}
|
|
// The restarted controller starts, then answers: the episode ends, its record stays.
|
|
ex.health = "starting"
|
|
sweep(s, 1)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].Unresponsive || g[0].UnresponsiveSince != since {
|
|
t.Fatalf("after the restart: %+v (unresponsive must clear; since stays as the record)", g[0])
|
|
}
|
|
// A second episode needs the whole count again — never a restart on the first unhealthy read after one.
|
|
ex.health = "unhealthy"
|
|
sweep(s, controllerUnresponsiveRestart-1)
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("a second episode restarted early: %d restarts", n)
|
|
}
|
|
}
|
|
|
|
// Slow but answering (healthy / starting), a swap, a vzdump, a locked or a parked guest never counts — and so is never
|
|
// restarted.
|
|
func TestSupervisor_UnresponsiveGuards(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
setup func(s *Server, ex *rbExec, ctl *fakeGuestPowerCtl)
|
|
}{
|
|
{"healthy", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "healthy" }},
|
|
{"starting", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "starting" }},
|
|
{"swap", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl) { s.swapInFlight[9201] = true }},
|
|
{"vzdump", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.backupRun = map[int]bool{9201: true} }},
|
|
{"locked", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.locks[9201] = "backup" }},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, ex, ctl, _ := unrespServer(t, "unhealthy")
|
|
tc.setup(s, ex, ctl)
|
|
sweep(s, controllerUnresponsiveRestart+2)
|
|
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) == 1 && g[0].Unresponsive {
|
|
t.Fatalf("%s counted as unresponsive: %+v", tc.name, g[0])
|
|
}
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("%s: restarted %d times", tc.name, n)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// At the restart moment: the controller's busy file (fresh, or unreadable) and the brake refuse it; the alarm stays.
|
|
// RED-PROOF: each seen failing by deleting its guard.
|
|
func TestSupervisor_UnresponsiveRestartRefusals(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
setup func(s *Server, ex *rbExec, clk *supClock)
|
|
crashloop bool
|
|
}{
|
|
{"busy file fresh", func(_ *Server, ex *rbExec, clk *supClock) {
|
|
ex.busy = clk.t.Add(-20*time.Second).Format(time.RFC3339) + "\na backup, a restore or an off-site run is running\n"
|
|
}, false},
|
|
{"busy file unreadable", func(_ *Server, ex *rbExec, _ *supClock) { ex.busyErr = errors.New("pct exec: exit status 255") }, false},
|
|
{"brake: 3 restarts in 15 minutes", func(s *Server, _ *rbExec, clk *supClock) {
|
|
st := s.supState(9201)
|
|
st.restarts = []time.Time{clk.t.Add(-10 * time.Minute), clk.t.Add(-5 * time.Minute), clk.t.Add(-time.Minute)}
|
|
}, true},
|
|
{"crash-loop pause in force", func(s *Server, _ *rbExec, clk *supClock) {
|
|
s.supState(9201).crashloopSince = clk.t.Add(-5 * time.Minute)
|
|
}, true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, ex, _, clk := unrespServer(t, "unhealthy")
|
|
tc.setup(s, ex, clk)
|
|
sweep(s, controllerUnresponsiveRestart+3)
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("%s: restarted %d times", tc.name, n)
|
|
}
|
|
g := s.ControllerSupervisorStatus(context.Background()).Guests
|
|
if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveRestartAt != "" {
|
|
t.Fatalf("%s: the alarm must stay and no restart be recorded: %+v", tc.name, g)
|
|
}
|
|
if tc.crashloop && !g[0].Crashloop {
|
|
t.Fatalf("%s: the brake did not raise the crash-loop: %+v", tc.name, g[0])
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A busy file the controller stopped writing (older than controllerBusyFresh) is not „busy" — a hung controller must
|
|
// not shield itself with its last word.
|
|
func TestSupervisor_UnresponsiveStaleBusyFileDoesNotShield(t *testing.T) {
|
|
s, ex, _, clk := unrespServer(t, "unhealthy")
|
|
ex.busy = clk.t.Add(-10*time.Minute).Format(time.RFC3339) + "\nan app update is running\n"
|
|
sweep(s, controllerUnresponsiveRestart)
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("a stale busy file blocked the restart: %d restarts", n)
|
|
}
|
|
}
|