R-1344 decided from the night's measurement: a controller that runs but does not answer is restarted at 10 unhealthy sweeps (alarm at 6) — busy file + the brake refuse; stanza carries unresponsive_restart_after_s / unresponsive_restart_at
gates / gates (push) Successful in 33s
gates / gates (push) Successful in 33s
This commit is contained in:
admin committed
1 parent
98bd1b71e9
commit
bd18347c1c
6 files changed
+237
-44
No files matched your search
@@ -2,19 +2,25 @@ package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-1344 (operator brief 2026-10-29, Part 2): a controller that RUNS but whose Docker health check reads „unhealthy" on
|
||||
// controllerUnresponsiveConfirm consecutive sweeps is reported unresponsive (the stanza; the hub raises
|
||||
// controller_unresponsive). ALARM ONLY: it is never restarted for it until the measurement licenses that.
|
||||
// RED-PROOF: seen failing before the stanza carried it.
|
||||
// R-1344 (operator brief 2026-10-29, Part 2; decided 2026-10-30 from the night's measurement): a controller that RUNS
|
||||
// but whose Docker health check reads „unhealthy" on controllerUnresponsiveConfirm consecutive sweeps is reported
|
||||
// unresponsive (the stanza; the hub raises controller_unresponsive). If it still reads „unhealthy" at
|
||||
// controllerUnresponsiveRestart sweeps, the agent restarts it — with the not-running restart's guards and brake, plus
|
||||
// the controller's own busy file. RED-PROOF: the restart tests seen failing before the restart existed.
|
||||
|
||||
func unrespServer(t *testing.T, health string) (*Server, *opExec, *fakeGuestPowerCtl) {
|
||||
func unrespServer(t *testing.T, health string) (*Server, *rbExec, *fakeGuestPowerCtl, *supClock) {
|
||||
t.Helper()
|
||||
s, ex, ctl, _, _, _ := opServer(t, "running")
|
||||
s, ex, ctl, clk, _, _ := opServer(t, "running")
|
||||
ex.health = health
|
||||
return s, ex, ctl
|
||||
rb := &rbExec{opExec: ex}
|
||||
s.guestExec = rb
|
||||
return s, rb, ctl, clk
|
||||
}
|
||||
|
||||
func sweep(s *Server, n int) {
|
||||
@@ -23,8 +29,8 @@ func sweep(s *Server, n int) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupervisor_UnresponsiveIsAlarmedNeverRestarted(t *testing.T) {
|
||||
s, ex, _ := unrespServer(t, "unhealthy")
|
||||
func TestSupervisor_UnresponsiveIsAlarmedThenRestarted(t *testing.T) {
|
||||
s, ex, _, _ := unrespServer(t, "unhealthy")
|
||||
sweep(s, controllerUnresponsiveConfirm-1)
|
||||
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) != 1 || g[0].Unresponsive {
|
||||
t.Fatalf("unresponsive before %d sweeps: %+v", controllerUnresponsiveConfirm, g)
|
||||
@@ -34,41 +40,117 @@ func TestSupervisor_UnresponsiveIsAlarmedNeverRestarted(t *testing.T) {
|
||||
if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveSince == "" {
|
||||
t.Fatalf("after %d unhealthy sweeps: %+v", controllerUnresponsiveConfirm, g)
|
||||
}
|
||||
if g[0].UnresponsiveRestartAfterS != (controllerUnresponsiveRestart-controllerUnresponsiveConfirm)*int(controllerSupervisorInterval/time.Second) {
|
||||
t.Fatalf("the stanza must tell the hub when the restart comes: %+v", g[0])
|
||||
}
|
||||
since := g[0].UnresponsiveSince
|
||||
sweep(s, 10)
|
||||
sweep(s, controllerUnresponsiveRestart-controllerUnresponsiveConfirm-1)
|
||||
if n := ex.count(9201); n != 0 {
|
||||
t.Fatalf("an unresponsive controller was RESTARTED (%d) — the restart waits for the measurement", n)
|
||||
t.Fatalf("restarted after %d unhealthy sweeps — the restart waits for %d", controllerUnresponsiveRestart-1, controllerUnresponsiveRestart)
|
||||
}
|
||||
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].UnresponsiveSince != since {
|
||||
t.Fatal("unresponsive_since moved while it stayed unresponsive — the hub would mail again")
|
||||
}
|
||||
// it answers again
|
||||
ex.health = "healthy"
|
||||
sweep(s, 1)
|
||||
if n := ex.count(9201); n != 1 {
|
||||
t.Fatalf("after %d unhealthy sweeps: %d restarts, want 1", controllerUnresponsiveRestart, n)
|
||||
}
|
||||
g = s.ControllerSupervisorStatus(context.Background()).Guests
|
||||
if g[0].UnresponsiveRestartAt == "" || g[0].UnresponsiveRestartAt != g[0].LastRestartAt || g[0].Restarts24h != 1 ||
|
||||
!strings.Contains(g[0].LastReason, "health check") {
|
||||
t.Fatalf("the restart is not on the record the hub reads: %+v", g[0])
|
||||
}
|
||||
// A restart that did not help (still „unhealthy" on the next sweep) is not repeated at once: the count starts again.
|
||||
sweep(s, 1)
|
||||
if n := ex.count(9201); n != 1 {
|
||||
t.Fatalf("restarted again on the sweep after a restart: %d restarts", n)
|
||||
}
|
||||
// The restarted controller starts, then answers: the episode ends, its record stays.
|
||||
ex.health = "starting"
|
||||
sweep(s, 1)
|
||||
if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].Unresponsive || g[0].UnresponsiveSince != since {
|
||||
t.Fatalf("after it answered: %+v (unresponsive must clear; since stays as the record)", g[0])
|
||||
t.Fatalf("after the restart: %+v (unresponsive must clear; since stays as the record)", g[0])
|
||||
}
|
||||
// A second episode needs the whole count again — never a restart on the first unhealthy read after one.
|
||||
ex.health = "unhealthy"
|
||||
sweep(s, controllerUnresponsiveRestart-1)
|
||||
if n := ex.count(9201); n != 1 {
|
||||
t.Fatalf("a second episode restarted early: %d restarts", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Slow but answering (healthy / starting), a swap, a vzdump or a parked guest never counts.
|
||||
// Slow but answering (healthy / starting), a swap, a vzdump, a locked or a parked guest never counts — and so is never
|
||||
// restarted.
|
||||
func TestSupervisor_UnresponsiveGuards(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
setup func(s *Server, ex *opExec, ctl *fakeGuestPowerCtl)
|
||||
setup func(s *Server, ex *rbExec, ctl *fakeGuestPowerCtl)
|
||||
}{
|
||||
{"healthy", func(_ *Server, ex *opExec, _ *fakeGuestPowerCtl) { ex.health = "healthy" }},
|
||||
{"starting", func(_ *Server, ex *opExec, _ *fakeGuestPowerCtl) { ex.health = "starting" }},
|
||||
{"swap", func(s *Server, _ *opExec, _ *fakeGuestPowerCtl) { s.swapInFlight[9201] = true }},
|
||||
{"vzdump", func(_ *Server, _ *opExec, ctl *fakeGuestPowerCtl) { ctl.backupRun = map[int]bool{9201: true} }},
|
||||
{"locked", func(_ *Server, _ *opExec, ctl *fakeGuestPowerCtl) { ctl.locks[9201] = "backup" }},
|
||||
{"healthy", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "healthy" }},
|
||||
{"starting", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "starting" }},
|
||||
{"swap", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl) { s.swapInFlight[9201] = true }},
|
||||
{"vzdump", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.backupRun = map[int]bool{9201: true} }},
|
||||
{"locked", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.locks[9201] = "backup" }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, ex, ctl := unrespServer(t, "unhealthy")
|
||||
s, ex, ctl, _ := unrespServer(t, "unhealthy")
|
||||
tc.setup(s, ex, ctl)
|
||||
sweep(s, controllerUnresponsiveConfirm+2)
|
||||
sweep(s, controllerUnresponsiveRestart+2)
|
||||
if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) == 1 && g[0].Unresponsive {
|
||||
t.Fatalf("%s counted as unresponsive: %+v", tc.name, g[0])
|
||||
}
|
||||
if n := ex.count(9201); n != 0 {
|
||||
t.Fatalf("%s: restarted %d times", tc.name, n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// At the restart moment: the controller's busy file (fresh, or unreadable) and the brake refuse it; the alarm stays.
|
||||
// RED-PROOF: each seen failing by deleting its guard.
|
||||
func TestSupervisor_UnresponsiveRestartRefusals(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
setup func(s *Server, ex *rbExec, clk *supClock)
|
||||
crashloop bool
|
||||
}{
|
||||
{"busy file fresh", func(_ *Server, ex *rbExec, clk *supClock) {
|
||||
ex.busy = clk.t.Add(-20*time.Second).Format(time.RFC3339) + "\na backup, a restore or an off-site run is running\n"
|
||||
}, false},
|
||||
{"busy file unreadable", func(_ *Server, ex *rbExec, _ *supClock) { ex.busyErr = errors.New("pct exec: exit status 255") }, false},
|
||||
{"brake: 3 restarts in 15 minutes", func(s *Server, _ *rbExec, clk *supClock) {
|
||||
st := s.supState(9201)
|
||||
st.restarts = []time.Time{clk.t.Add(-10 * time.Minute), clk.t.Add(-5 * time.Minute), clk.t.Add(-time.Minute)}
|
||||
}, true},
|
||||
{"crash-loop pause in force", func(s *Server, _ *rbExec, clk *supClock) {
|
||||
s.supState(9201).crashloopSince = clk.t.Add(-5 * time.Minute)
|
||||
}, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, ex, _, clk := unrespServer(t, "unhealthy")
|
||||
tc.setup(s, ex, clk)
|
||||
sweep(s, controllerUnresponsiveRestart+3)
|
||||
if n := ex.count(9201); n != 0 {
|
||||
t.Fatalf("%s: restarted %d times", tc.name, n)
|
||||
}
|
||||
g := s.ControllerSupervisorStatus(context.Background()).Guests
|
||||
if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveRestartAt != "" {
|
||||
t.Fatalf("%s: the alarm must stay and no restart be recorded: %+v", tc.name, g)
|
||||
}
|
||||
if tc.crashloop && !g[0].Crashloop {
|
||||
t.Fatalf("%s: the brake did not raise the crash-loop: %+v", tc.name, g[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A busy file the controller stopped writing (older than controllerBusyFresh) is not „busy" — a hung controller must
|
||||
// not shield itself with its last word.
|
||||
func TestSupervisor_UnresponsiveStaleBusyFileDoesNotShield(t *testing.T) {
|
||||
s, ex, _, clk := unrespServer(t, "unhealthy")
|
||||
ex.busy = clk.t.Add(-10*time.Minute).Format(time.RFC3339) + "\nan app update is running\n"
|
||||
sweep(s, controllerUnresponsiveRestart)
|
||||
if n := ex.count(9201); n != 1 {
|
||||
t.Fatalf("a stale busy file blocked the restart: %d restarts", n)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user