package localapi import ( "context" "errors" "os" "path/filepath" "reflect" "strings" "sync/atomic" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // Operator brief 2026-10-29, Part 1: „Restart controller" from the hub. The agent's closed list holds one verb; the act // is the supervisor's own restart with the supervisor's guards, and the brake counts operator restarts too. // opExec answers the supervisor's fake plus the health read the operator restart waits on. type opExec struct { *supExec health string // what {{.State.Health.Status}} reads after the restart } func (f *opExec) GuestExec(ctx context.Context, vmid int, args ...string) (string, error) { if len(args) >= 5 && args[0] == "docker" && args[1] == "inspect" && args[3] == opHealthTemplate { f.mu.Lock() st := f.status[vmid] f.mu.Unlock() if st == "" { return "", errors.New("pct exec: exit status 1: Error: No such object: felhom-controller") } return map[bool]string{true: "true", false: "false"}[st == "running"] + "|" + f.health + "\n", nil } return f.supExec.GuestExec(ctx, vmid, args...) } func opServer(t *testing.T, status string) (*Server, *opExec, *fakeGuestPowerCtl, *supClock, string, *atomic.Int32) { t.Helper() ex := &opExec{supExec: &supExec{status: map[int]string{9201: status}, onRestart: "running"}, health: "healthy"} ctl := runningGuest(9201) s, clk, dir := supServer(t, nil, ctl, 9201) s.guestExec = ex s.opHealthPoll = time.Millisecond s.opHealthWait = 2 * time.Second var triggered atomic.Int32 s.SetOperatorActionTrigger(func() { triggered.Add(1) }) return s, ex, ctl, clk, dir, &triggered } func env(acts ...hub.OperatorAction) *hub.ControlEnvelope { return &hub.ControlEnvelope{OperatorActions: acts} } // waitResult waits for the action's result (the act runs in the background). func waitResult(t *testing.T, s *Server, id int64) hub.OperatorActionResult { t.Helper() deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { for _, r := range s.OperatorActionResults() { if r.ID == id { return r } } time.Sleep(2 * time.Millisecond) } t.Fatalf("no result for #%d", id) return hub.OperatorActionResult{} } // RED-PROOF: seen failing before the list existed (build failure, then the names). func TestOpActions_AgentClosedList(t *testing.T) { if got, want := OperatorActionNames(), []string{"restart_controller"}; !reflect.DeepEqual(got, want) { t.Fatalf("the agent's operator actions = %v, want exactly %v — a new verb needs the operator's word", got, want) } } // Anything outside the list — and a bad argument — is refused and NOTHING is called. func TestOpActions_UnknownOrBadArgIsRefused(t *testing.T) { s, ex, _, _, _, _ := opServer(t, "running") s.OnEnvelope(context.Background(), env( hub.OperatorAction{ID: 1, Action: "reboot_guest", Arg: "9201"}, hub.OperatorAction{ID: 2, Action: "restart_controller", Arg: "abc"}, hub.OperatorAction{ID: 3, Action: "restart_controller", Arg: "9202"}, // not provisioned here )) for _, id := range []int64{1, 2, 3} { if r := waitResult(t, s, id); r.Outcome != "refused" { t.Errorf("#%d: %+v, want refused", id, r) } } if n := ex.count(9201); n != 0 { t.Fatalf("a refused action restarted the controller (%d)", n) } } // The press restarts ONCE, says how long the controller took to answer, asks for a report at once, and re-sends the // result until the hub stops listing the id. RED-PROOF: drop the GuestExec restart → restarts 0 → FAILS. func TestOpActions_RestartReportsTheAnswerTime(t *testing.T) { s, ex, _, _, _, trig := opServer(t, "running") a := hub.OperatorAction{ID: 7, Action: "restart_controller", Arg: "9201"} s.OnEnvelope(context.Background(), env(a)) r := waitResult(t, s, 7) if r.Outcome != "done" || !strings.Contains(r.Message, "answered after") { t.Fatalf("result %+v", r) } if n := ex.count(9201); n != 1 { t.Fatalf("restarts = %d, want 1", n) } if trig.Load() == 0 { t.Fatal("the agent did not ask for a report — the result would wait 15 minutes") } s.OnEnvelope(context.Background(), env(a)) // the hub still lists it: re-send, never act again time.Sleep(100 * time.Millisecond) // a wrong second act runs in the background — give it time to show if n := ex.count(9201); n != 1 { t.Fatalf("a re-listed id acted again (%d)", n) } if len(s.OperatorActionResults()) != 1 { t.Fatal("the result must be re-sent while listed") } s.OnEnvelope(context.Background(), env()) // the hub recorded it if left := s.OperatorActionResults(); len(left) != 0 { t.Fatalf("a recorded result is still sent: %+v", left) } } // The point of the agent doing it: a STOPPED controller is restarted too (the controller cannot restart itself). func TestOpActions_WorksWithTheControllerStopped(t *testing.T) { s, ex, _, _, _, _ := opServer(t, "exited") s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 8, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, 8); r.Outcome != "done" || ex.count(9201) != 1 { t.Fatalf("result %+v, restarts %d", r, ex.count(9201)) } } // Each supervisor guard refuses the press, with the reason. RED-PROOF: drop the lock check → the locked case // restarts → FAILS (each guard seen failing by its own mutation). func TestOpActions_GuardsRefuse(t *testing.T) { cases := []struct { name, want string setup func(s *Server, ctl *fakeGuestPowerCtl, dir string) }{ {"parked", "parked", func(_ *Server, _ *fakeGuestPowerCtl, dir string) { if err := os.WriteFile(filepath.Join(dir, "9201", ControllerParkedMarker), nil, 0o600); err != nil { panic(err) } }}, {"swap", "swap", func(s *Server, _ *fakeGuestPowerCtl, _ string) { s.swapInFlight[9201] = true }}, {"locked", "locked", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.locks[9201] = "backup" }}, {"vzdump", "backup", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.backupRun = map[int]bool{9201: true} }}, {"vzdump unknown", "backup", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.backupErr = errors.New("tasks unreadable") }}, {"guest stopped", "not running", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.guests[0].Status = "stopped" }}, {"guest list", "guest list", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.guestsErr = errors.New("api down") }}, } for i, tc := range cases { t.Run(tc.name, func(t *testing.T) { s, ex, ctl, _, dir, _ := opServer(t, "running") tc.setup(s, ctl, dir) id := int64(100 + i) s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"})) r := waitResult(t, s, id) if r.Outcome != "refused" || !strings.Contains(strings.ToLower(r.Message), tc.want) { t.Fatalf("result %+v, want refused naming %q", r, tc.want) } if n := ex.count(9201); n != 0 { t.Fatalf("guard %q did not hold: restarted %d", tc.name, n) } }) } } // The thrash brake counts operator restarts AND the supervisor's: 3 in 15 minutes → the next press is refused, and // presses stay refused for 30 minutes. RED-PROOF: drop the brake → the 4th press restarts → FAILS. func TestOpActions_TheBrakeCountsEveryRestart(t *testing.T) { s, ex, _, clk, _, _ := opServer(t, "running") // one restart by the supervisor itself ex.mu.Lock() ex.status[9201] = "exited" ex.mu.Unlock() s.ControllerSupervisorTick(context.Background()) s.ControllerSupervisorTick(context.Background()) if ex.count(9201) != 1 { t.Fatalf("setup: the supervisor did not restart (%d)", ex.count(9201)) } for id := int64(1); id <= 2; id++ { clk.t = clk.t.Add(time.Minute) s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, id); r.Outcome != "done" { t.Fatalf("press %d: %+v", id, r) } } clk.t = clk.t.Add(time.Minute) s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 3, Action: "restart_controller", Arg: "9201"})) r := waitResult(t, s, 3) if r.Outcome != "refused" || !strings.Contains(r.Message, "3 restarts in 15 minutes") { t.Fatalf("the 4th restart in 15 minutes: %+v", r) } if n := ex.count(9201); n != 3 { t.Fatalf("restarts = %d, want 3", n) } clk.t = clk.t.Add(20 * time.Minute) // the window has passed, but the 30-minute refusal has not s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 4, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, 4); r.Outcome != "refused" { t.Fatalf("inside the 30-minute refusal: %+v", r) } clk.t = clk.t.Add(11 * time.Minute) s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 5, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, 5); r.Outcome != "done" { t.Fatalf("after the refusal: %+v", r) } } // A controller that does not come back healthy is a FAILED press, said plainly. func TestOpActions_NoHealthyAnswerIsFailed(t *testing.T) { s, ex, _, _, _, _ := opServer(t, "running") ex.health = "unhealthy" s.opHealthWait = 30 * time.Millisecond s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 9, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, 9); r.Outcome != "failed" || !strings.Contains(r.Message, "no healthy answer") { t.Fatalf("result %+v", r) } }