package localapi import ( "context" "errors" "strings" "testing" "time" ) // R-1344 (operator brief 2026-10-29, Part 2; decided 2026-10-30 from the night's measurement): a controller that RUNS // but whose Docker health check reads „unhealthy" on controllerUnresponsiveConfirm consecutive sweeps is reported // unresponsive (the stanza; the hub raises controller_unresponsive). If it still reads „unhealthy" at // controllerUnresponsiveRestart sweeps, the agent restarts it — with the not-running restart's guards and brake, plus // the controller's own busy file. RED-PROOF: the restart tests seen failing before the restart existed. func unrespServer(t *testing.T, health string) (*Server, *rbExec, *fakeGuestPowerCtl, *supClock) { t.Helper() s, ex, ctl, clk, _, _ := opServer(t, "running") ex.health = health rb := &rbExec{opExec: ex} s.guestExec = rb return s, rb, ctl, clk } func sweep(s *Server, n int) { for i := 0; i < n; i++ { s.ControllerSupervisorTick(context.Background()) } } func TestSupervisor_UnresponsiveIsAlarmedThenRestarted(t *testing.T) { s, ex, _, _ := unrespServer(t, "unhealthy") sweep(s, controllerUnresponsiveConfirm-1) if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) != 1 || g[0].Unresponsive { t.Fatalf("unresponsive before %d sweeps: %+v", controllerUnresponsiveConfirm, g) } sweep(s, 1) g := s.ControllerSupervisorStatus(context.Background()).Guests if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveSince == "" { t.Fatalf("after %d unhealthy sweeps: %+v", controllerUnresponsiveConfirm, g) } if g[0].UnresponsiveRestartAfterS != (controllerUnresponsiveRestart-controllerUnresponsiveConfirm)*int(controllerSupervisorInterval/time.Second) { t.Fatalf("the stanza must tell the hub when the restart comes: %+v", g[0]) } since := g[0].UnresponsiveSince sweep(s, controllerUnresponsiveRestart-controllerUnresponsiveConfirm-1) if n := ex.count(9201); n != 0 { t.Fatalf("restarted after %d unhealthy sweeps — the restart waits for %d", controllerUnresponsiveRestart-1, controllerUnresponsiveRestart) } if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].UnresponsiveSince != since { t.Fatal("unresponsive_since moved while it stayed unresponsive — the hub would mail again") } sweep(s, 1) if n := ex.count(9201); n != 1 { t.Fatalf("after %d unhealthy sweeps: %d restarts, want 1", controllerUnresponsiveRestart, n) } g = s.ControllerSupervisorStatus(context.Background()).Guests if g[0].UnresponsiveRestartAt == "" || g[0].UnresponsiveRestartAt != g[0].LastRestartAt || g[0].Restarts24h != 1 || !strings.Contains(g[0].LastReason, "health check") { t.Fatalf("the restart is not on the record the hub reads: %+v", g[0]) } // A restart that did not help (still „unhealthy" on the next sweep) is not repeated at once: the count starts again. sweep(s, 1) if n := ex.count(9201); n != 1 { t.Fatalf("restarted again on the sweep after a restart: %d restarts", n) } // The restarted controller starts, then answers: the episode ends, its record stays. ex.health = "starting" sweep(s, 1) if g := s.ControllerSupervisorStatus(context.Background()).Guests; g[0].Unresponsive || g[0].UnresponsiveSince != since { t.Fatalf("after the restart: %+v (unresponsive must clear; since stays as the record)", g[0]) } // A second episode needs the whole count again — never a restart on the first unhealthy read after one. ex.health = "unhealthy" sweep(s, controllerUnresponsiveRestart-1) if n := ex.count(9201); n != 1 { t.Fatalf("a second episode restarted early: %d restarts", n) } } // Slow but answering (healthy / starting), a swap, a vzdump, a locked or a parked guest never counts — and so is never // restarted. func TestSupervisor_UnresponsiveGuards(t *testing.T) { for _, tc := range []struct { name string setup func(s *Server, ex *rbExec, ctl *fakeGuestPowerCtl) }{ {"healthy", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "healthy" }}, {"starting", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl) { ex.health = "starting" }}, {"swap", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl) { s.swapInFlight[9201] = true }}, {"vzdump", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.backupRun = map[int]bool{9201: true} }}, {"locked", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl) { ctl.locks[9201] = "backup" }}, } { t.Run(tc.name, func(t *testing.T) { s, ex, ctl, _ := unrespServer(t, "unhealthy") tc.setup(s, ex, ctl) sweep(s, controllerUnresponsiveRestart+2) if g := s.ControllerSupervisorStatus(context.Background()).Guests; len(g) == 1 && g[0].Unresponsive { t.Fatalf("%s counted as unresponsive: %+v", tc.name, g[0]) } if n := ex.count(9201); n != 0 { t.Fatalf("%s: restarted %d times", tc.name, n) } }) } } // At the restart moment: the controller's busy file (fresh, or unreadable) and the brake refuse it; the alarm stays. // RED-PROOF: each seen failing by deleting its guard. func TestSupervisor_UnresponsiveRestartRefusals(t *testing.T) { for _, tc := range []struct { name string setup func(s *Server, ex *rbExec, clk *supClock) crashloop bool }{ {"busy file fresh", func(_ *Server, ex *rbExec, clk *supClock) { ex.busy = clk.t.Add(-20*time.Second).Format(time.RFC3339) + "\na backup, a restore or an off-site run is running\n" }, false}, {"busy file unreadable", func(_ *Server, ex *rbExec, _ *supClock) { ex.busyErr = errors.New("pct exec: exit status 255") }, false}, {"brake: 3 restarts in 15 minutes", func(s *Server, _ *rbExec, clk *supClock) { st := s.supState(9201) st.restarts = []time.Time{clk.t.Add(-10 * time.Minute), clk.t.Add(-5 * time.Minute), clk.t.Add(-time.Minute)} }, true}, {"crash-loop pause in force", func(s *Server, _ *rbExec, clk *supClock) { s.supState(9201).crashloopSince = clk.t.Add(-5 * time.Minute) }, true}, } { t.Run(tc.name, func(t *testing.T) { s, ex, _, clk := unrespServer(t, "unhealthy") tc.setup(s, ex, clk) sweep(s, controllerUnresponsiveRestart+3) if n := ex.count(9201); n != 0 { t.Fatalf("%s: restarted %d times", tc.name, n) } g := s.ControllerSupervisorStatus(context.Background()).Guests if len(g) != 1 || !g[0].Unresponsive || g[0].UnresponsiveRestartAt != "" { t.Fatalf("%s: the alarm must stay and no restart be recorded: %+v", tc.name, g) } if tc.crashloop && !g[0].Crashloop { t.Fatalf("%s: the brake did not raise the crash-loop: %+v", tc.name, g[0]) } }) } } // A busy file the controller stopped writing (older than controllerBusyFresh) is not „busy" — a hung controller must // not shield itself with its last word. func TestSupervisor_UnresponsiveStaleBusyFileDoesNotShield(t *testing.T) { s, ex, _, clk := unrespServer(t, "unhealthy") ex.busy = clk.t.Add(-10*time.Minute).Format(time.RFC3339) + "\nan app update is running\n" sweep(s, controllerUnresponsiveRestart) if n := ex.count(9201); n != 1 { t.Fatalf("a stale busy file blocked the restart: %d restarts", n) } }