package localapi import ( "context" "encoding/json" "errors" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/backup" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // D65 = A (operator 2026-10-29 15:50): „Reboot guest" from the hub. The agent's reboot_guest — the existing // `pct reboot` of THAT guest — with its guards, the hourly limit, the brake, and the record the household's line reads. // rbExec adds the controller's busy file to the operator-restart fake. type rbExec struct { *opExec busy string // the file's content; "" = no file busyErr error // pct exec itself failed } func (f *rbExec) GuestExec(ctx context.Context, vmid int, args ...string) (string, error) { if len(args) == 2 && args[0] == "cat" && args[1] == ControllerBusyMarkerGuestPath { if f.busyErr != nil { return "", f.busyErr } if f.busy == "" { return "", errors.New("pct exec 9201 [cat ...]: exit status 1: cat: " + ControllerBusyMarkerGuestPath + ": No such file or directory") } return f.busy, nil } return f.opExec.GuestExec(ctx, vmid, args...) } type fakeRebooter struct { mu sync.Mutex n int err error after func() } func (r *fakeRebooter) RebootGuest(context.Context, int) error { r.mu.Lock() defer r.mu.Unlock() r.n++ if r.after != nil { r.after() } return r.err } func (r *fakeRebooter) count() int { r.mu.Lock() defer r.mu.Unlock() return r.n } func rbServer(t *testing.T) (*Server, *rbExec, *fakeRebooter, *fakeGuestPowerCtl, *supClock, string) { t.Helper() s, ox, ctl, clk, dir, _ := opServer(t, "running") ex := &rbExec{opExec: ox} s.guestExec = ex rb := &fakeRebooter{} s.opRebooter = rb s.inFlight = &backup.InFlight{} return s, ex, rb, ctl, clk, dir } func pressReboot(t *testing.T, s *Server, id int64) hub.OperatorActionResult { t.Helper() s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "reboot_guest", Arg: "9201"})) return waitResult(t, s, id) } // The press reboots ONCE and says when the guest was up and when the controller answered; the record is written for // the household's line. RED-PROOF: drop the RebootGuest call → reboots 0 → FAILS. func TestOpReboot_RebootsAndReportsTheTimes(t *testing.T) { s, _, rb, _, clk, _ := rbServer(t) r := pressReboot(t, s, 1) if r.Outcome != "done" || !strings.Contains(r.Message, "the guest was up after") || !strings.Contains(r.Message, "the controller answered after") { t.Fatalf("result %+v", r) } if rb.count() != 1 { t.Fatalf("reboots = %d, want 1", rb.count()) } rec, ok, err := s.lastOperatorReboot(9201) if err != nil || !ok || !rec.At.Equal(clk.t) || rec.ActionID != 1 { t.Fatalf("record %+v ok=%v err=%v", rec, ok, err) } if s.inFlight.Busy() != "" { t.Fatalf("the heavy-operation gate is still held after the reboot: %q", s.inFlight.Busy()) } if s.rebooting(9201) { t.Fatal("the reboot-in-flight flag outlived the reboot") } } // Each guard refuses with its reason and NOTHING is rebooted. RED-PROOF: each guard seen failing by deleting it. func TestOpReboot_GuardsRefuse(t *testing.T) { fresh := func(clk *supClock) string { return clk.t.Add(-20*time.Second).Format(time.RFC3339) + "\nan off-site backup is running\n" } cases := []struct { name, want string setup func(s *Server, ex *rbExec, ctl *fakeGuestPowerCtl, clk *supClock, dir string) }{ {"parked", "parked", func(_ *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, dir string) { if err := os.WriteFile(filepath.Join(dir, "9201", ControllerParkedMarker), nil, 0o600); err != nil { panic(err) } }}, {"swap", "swap", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) { s.swapInFlight[9201] = true }}, {"locked", "locked", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) { ctl.locks[9201] = "backup" }}, {"stopped", "not running", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) { ctl.guests[0].Status = "stopped" }}, {"vzdump", "backup", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) { ctl.backupRun = map[int]bool{9201: true} }}, {"vzdump unknown", "cannot be read", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) { ctl.backupErr = errors.New("tasks unreadable") }}, {"heavy gate", "backup:felhom-pbs", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) { s.inFlight.TryAcquire("backup:felhom-pbs") }}, {"agent backup job", "a backup is running", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) { s.jobs = map[backupJobKey]*backupJob{{vmid: 9201, target: "local"}: {JobID: "j1", Phase: PhaseRunning}} }}, {"controller busy", "off-site backup", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl, clk *supClock, _ string) { ex.busy = fresh(clk) }}, {"busy file unreadable", "cannot be ruled out", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) { ex.busyErr = errors.New("pct exec 9201: exit status 255: lxc-attach failed") }}, {"brake in force", "3 restarts in 15 minutes", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, clk *supClock, _ string) { s.ctrlSup.mu.Lock() s.supState(9201).opBrakeUntil = clk.t.Add(10 * time.Minute) s.ctrlSup.mu.Unlock() }}, {"not provisioned", "not a guest this agent provisioned", func(_ *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, dir string) { if err := os.RemoveAll(filepath.Join(dir, "9201", "bootstrap")); err != nil { panic(err) } }}, } for i, tc := range cases { t.Run(tc.name, func(t *testing.T) { s, ex, rb, ctl, clk, dir := rbServer(t) tc.setup(s, ex, ctl, clk, dir) r := pressReboot(t, s, int64(100+i)) if r.Outcome != "refused" || !strings.Contains(r.Message, tc.want) { t.Fatalf("result %+v, want refused naming %q", r, tc.want) } if rb.count() != 0 { t.Fatalf("guard %q did not hold: rebooted", tc.name) } if _, ok, _ := s.lastOperatorReboot(9201); ok { t.Fatal("a refused press wrote the reboot record (the household would read a reboot that never happened)") } }) } } // A busy file the controller stopped writing (stuck, killed) is not a busy controller — the reboot exists for exactly // that box. RED-PROOF: drop the freshness check → refused → FAILS. func TestOpReboot_StaleBusyFileDoesNotBlock(t *testing.T) { s, ex, rb, _, clk, _ := rbServer(t) ex.busy = clk.t.Add(-10*time.Minute).Format(time.RFC3339) + "\na backup is running\n" if r := pressReboot(t, s, 1); r.Outcome != "done" || rb.count() != 1 { t.Fatalf("result %+v, reboots %d", r, rb.count()) } } // At most one reboot per hour per guest, and the limit survives an agent restart (the record is on disk). // RED-PROOF: drop the hourly check → the second press reboots → FAILS. func TestOpReboot_AtMostOncePerHour(t *testing.T) { s, _, rb, ctl, clk, dir := rbServer(t) if r := pressReboot(t, s, 1); r.Outcome != "done" { t.Fatalf("first press: %+v", r) } clk.t = clk.t.Add(59 * time.Minute) if r := pressReboot(t, s, 2); r.Outcome != "refused" || !strings.Contains(r.Message, "at most one reboot per hour") { t.Fatalf("second press inside the hour: %+v", r) } // a restarted agent (a fresh in-memory state over the same guests dir) still refuses s2, ox2, _, _, _, _ := opServer(t, "running") s2.guestsDir, s2.staleLock, s2.now = dir, ctl, clk.now s2.guestExec = &rbExec{opExec: ox2} rb2 := &fakeRebooter{} s2.opRebooter, s2.inFlight = rb2, &backup.InFlight{} if r := pressReboot(t, s2, 3); r.Outcome != "refused" || rb2.count() != 0 { t.Fatalf("after an agent restart, inside the hour: %+v", r) } clk.t = clk.t.Add(2 * time.Minute) if r := pressReboot(t, s, 4); r.Outcome != "done" || rb.count() != 2 { t.Fatalf("after the hour: %+v (reboots %d)", r, rb.count()) } } // The brake counts a reboot as a restart: two operator restarts and one reboot inside 15 minutes refuse the next // restart press. RED-PROOF: drop the restarts append in operatorRebootGuest → the 4th press restarts → FAILS. func TestOpReboot_TheBrakeCountsAReboot(t *testing.T) { s, ex, _, _, clk, _ := rbServer(t) for id := int64(1); id <= 2; id++ { s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, id); r.Outcome != "done" { t.Fatalf("restart %d: %+v", id, r) } clk.t = clk.t.Add(time.Minute) } if r := pressReboot(t, s, 3); r.Outcome != "done" { t.Fatalf("reboot: %+v", r) } clk.t = clk.t.Add(time.Minute) s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 4, Action: "restart_controller", Arg: "9201"})) if r := waitResult(t, s, 4); r.Outcome != "refused" || !strings.Contains(r.Message, "3 restarts in 15 minutes") { t.Fatalf("the 4th restart (2 restarts + 1 reboot) in 15 minutes: %+v", r) } if n := ex.count(9201); n != 2 { t.Fatalf("restarts = %d, want 2", n) } } // A failed `pct reboot` is a FAILED press, said plainly; the gate and the flag are given back. func TestOpReboot_FailedRebootIsFailed(t *testing.T) { s, _, rb, _, _, _ := rbServer(t) rb.err = errors.New("pct reboot 9201: exit status 255") if r := pressReboot(t, s, 1); r.Outcome != "failed" || !strings.Contains(r.Message, "failed") { t.Fatalf("result %+v", r) } if s.inFlight.Busy() != "" || s.rebooting(9201) { t.Fatal("the gate or the flag outlived a failed reboot") } } // While the reboot runs, the guest-power watchdog does not start the (momentarily stopped) guest and the supervisor // does not restart its controller. RED-PROOF: drop either rebooting() check → a start or a restart → FAILS. func TestOpReboot_TheWatchdogsLeaveARebootingGuestAlone(t *testing.T) { s, ex, rb, ctl, _, _ := rbServer(t) var started, restarted int rb.after = func() { ctl.guests[0].Status = "stopped" s.GuestPowerTick(context.Background()) ctl.guests[0].Status = "running" ex.mu.Lock() ex.status[9201] = "exited" ex.mu.Unlock() s.ControllerSupervisorTick(context.Background()) s.ControllerSupervisorTick(context.Background()) started, restarted = len(ctl.started), ex.count(9201) ex.mu.Lock() ex.status[9201] = "running" ex.mu.Unlock() } if r := pressReboot(t, s, 1); r.Outcome != "done" { t.Fatalf("result %+v", r) } if started != 0 || restarted != 0 { t.Fatalf("during the reboot: guest-power started %d, the supervisor restarted %d — both must leave it to pct reboot", started, restarted) } } // The household's line: GET /guest/operator-reboot returns the time for a day, then nothing. // RED-PROOF: drop the 24-hour check → the 25-hour read still returns the time → FAILS. func TestOpReboot_TheNoticeExpiresAfterADay(t *testing.T) { s, _, _, _, clk, _ := rbServer(t) read := func() string { w := httptest.NewRecorder() s.handleOperatorRebootNotice(w, httptest.NewRequest("GET", "/guest/operator-reboot", nil), 9201) var body struct { Data struct { At string `json:"at"` } `json:"data"` At string `json:"at"` } if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatalf("body %q: %v", w.Body.String(), err) } if body.Data.At != "" { return body.Data.At } return body.At } if got := read(); got != "" { t.Fatalf("no reboot yet, got %q", got) } at := clk.t if r := pressReboot(t, s, 1); r.Outcome != "done" { t.Fatalf("result %+v", r) } clk.t = at.Add(23 * time.Hour) if got := read(); got != at.UTC().Format(time.RFC3339) { t.Fatalf("after 23 h: %q, want %s", got, at.UTC().Format(time.RFC3339)) } clk.t = at.Add(25 * time.Hour) if got := read(); got != "" { t.Fatalf("after 25 h the line must be gone, got %q", got) } }