307 lines
11 KiB
Go
307 lines
11 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
)
|
|
|
|
// D65 = A (operator 2026-10-29 15:50): „Reboot guest" from the hub. The agent's reboot_guest — the existing
|
|
// `pct reboot` of THAT guest — with its guards, the hourly limit, the brake, and the record the household's line reads.
|
|
|
|
// rbExec adds the controller's busy file to the operator-restart fake.
|
|
type rbExec struct {
|
|
*opExec
|
|
busy string // the file's content; "" = no file
|
|
busyErr error // pct exec itself failed
|
|
}
|
|
|
|
func (f *rbExec) GuestExec(ctx context.Context, vmid int, args ...string) (string, error) {
|
|
if len(args) == 2 && args[0] == "cat" && args[1] == ControllerBusyMarkerGuestPath {
|
|
if f.busyErr != nil {
|
|
return "", f.busyErr
|
|
}
|
|
if f.busy == "" {
|
|
return "", errors.New("pct exec 9201 [cat ...]: exit status 1: cat: " + ControllerBusyMarkerGuestPath + ": No such file or directory")
|
|
}
|
|
return f.busy, nil
|
|
}
|
|
return f.opExec.GuestExec(ctx, vmid, args...)
|
|
}
|
|
|
|
type fakeRebooter struct {
|
|
mu sync.Mutex
|
|
n int
|
|
err error
|
|
after func()
|
|
}
|
|
|
|
func (r *fakeRebooter) RebootGuest(context.Context, int) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.n++
|
|
if r.after != nil {
|
|
r.after()
|
|
}
|
|
return r.err
|
|
}
|
|
|
|
func (r *fakeRebooter) count() int {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.n
|
|
}
|
|
|
|
func rbServer(t *testing.T) (*Server, *rbExec, *fakeRebooter, *fakeGuestPowerCtl, *supClock, string) {
|
|
t.Helper()
|
|
s, ox, ctl, clk, dir, _ := opServer(t, "running")
|
|
ex := &rbExec{opExec: ox}
|
|
s.guestExec = ex
|
|
rb := &fakeRebooter{}
|
|
s.opRebooter = rb
|
|
s.inFlight = &backup.InFlight{}
|
|
return s, ex, rb, ctl, clk, dir
|
|
}
|
|
|
|
func pressReboot(t *testing.T, s *Server, id int64) hub.OperatorActionResult {
|
|
t.Helper()
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "reboot_guest", Arg: "9201"}))
|
|
return waitResult(t, s, id)
|
|
}
|
|
|
|
// The press reboots ONCE and says when the guest was up and when the controller answered; the record is written for
|
|
// the household's line. RED-PROOF: drop the RebootGuest call → reboots 0 → FAILS.
|
|
func TestOpReboot_RebootsAndReportsTheTimes(t *testing.T) {
|
|
s, _, rb, _, clk, _ := rbServer(t)
|
|
r := pressReboot(t, s, 1)
|
|
if r.Outcome != "done" || !strings.Contains(r.Message, "the guest was up after") || !strings.Contains(r.Message, "the controller answered after") {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
if rb.count() != 1 {
|
|
t.Fatalf("reboots = %d, want 1", rb.count())
|
|
}
|
|
rec, ok, err := s.lastOperatorReboot(9201)
|
|
if err != nil || !ok || !rec.At.Equal(clk.t) || rec.ActionID != 1 {
|
|
t.Fatalf("record %+v ok=%v err=%v", rec, ok, err)
|
|
}
|
|
if s.inFlight.Busy() != "" {
|
|
t.Fatalf("the heavy-operation gate is still held after the reboot: %q", s.inFlight.Busy())
|
|
}
|
|
if s.rebooting(9201) {
|
|
t.Fatal("the reboot-in-flight flag outlived the reboot")
|
|
}
|
|
}
|
|
|
|
// Each guard refuses with its reason and NOTHING is rebooted. RED-PROOF: each guard seen failing by deleting it.
|
|
func TestOpReboot_GuardsRefuse(t *testing.T) {
|
|
fresh := func(clk *supClock) string {
|
|
return clk.t.Add(-20*time.Second).Format(time.RFC3339) + "\nan off-site backup is running\n"
|
|
}
|
|
cases := []struct {
|
|
name, want string
|
|
setup func(s *Server, ex *rbExec, ctl *fakeGuestPowerCtl, clk *supClock, dir string)
|
|
}{
|
|
{"parked", "parked", func(_ *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, dir string) {
|
|
if err := os.WriteFile(filepath.Join(dir, "9201", ControllerParkedMarker), nil, 0o600); err != nil {
|
|
panic(err)
|
|
}
|
|
}},
|
|
{"swap", "swap", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) { s.swapInFlight[9201] = true }},
|
|
{"locked", "locked", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) { ctl.locks[9201] = "backup" }},
|
|
{"stopped", "not running", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
ctl.guests[0].Status = "stopped"
|
|
}},
|
|
{"vzdump", "backup", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
ctl.backupRun = map[int]bool{9201: true}
|
|
}},
|
|
{"vzdump unknown", "cannot be read", func(_ *Server, _ *rbExec, ctl *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
ctl.backupErr = errors.New("tasks unreadable")
|
|
}},
|
|
{"heavy gate", "backup:felhom-pbs", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
s.inFlight.TryAcquire("backup:felhom-pbs")
|
|
}},
|
|
{"agent backup job", "a backup is running", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
s.jobs = map[backupJobKey]*backupJob{{vmid: 9201, target: "local"}: {JobID: "j1", Phase: PhaseRunning}}
|
|
}},
|
|
{"controller busy", "off-site backup", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl, clk *supClock, _ string) {
|
|
ex.busy = fresh(clk)
|
|
}},
|
|
{"busy file unreadable", "cannot be ruled out", func(_ *Server, ex *rbExec, _ *fakeGuestPowerCtl, _ *supClock, _ string) {
|
|
ex.busyErr = errors.New("pct exec 9201: exit status 255: lxc-attach failed")
|
|
}},
|
|
{"brake in force", "3 restarts in 15 minutes", func(s *Server, _ *rbExec, _ *fakeGuestPowerCtl, clk *supClock, _ string) {
|
|
s.ctrlSup.mu.Lock()
|
|
s.supState(9201).opBrakeUntil = clk.t.Add(10 * time.Minute)
|
|
s.ctrlSup.mu.Unlock()
|
|
}},
|
|
{"not provisioned", "not a guest this agent provisioned", func(_ *Server, _ *rbExec, _ *fakeGuestPowerCtl, _ *supClock, dir string) {
|
|
if err := os.RemoveAll(filepath.Join(dir, "9201", "bootstrap")); err != nil {
|
|
panic(err)
|
|
}
|
|
}},
|
|
}
|
|
for i, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, ex, rb, ctl, clk, dir := rbServer(t)
|
|
tc.setup(s, ex, ctl, clk, dir)
|
|
r := pressReboot(t, s, int64(100+i))
|
|
if r.Outcome != "refused" || !strings.Contains(r.Message, tc.want) {
|
|
t.Fatalf("result %+v, want refused naming %q", r, tc.want)
|
|
}
|
|
if rb.count() != 0 {
|
|
t.Fatalf("guard %q did not hold: rebooted", tc.name)
|
|
}
|
|
if _, ok, _ := s.lastOperatorReboot(9201); ok {
|
|
t.Fatal("a refused press wrote the reboot record (the household would read a reboot that never happened)")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A busy file the controller stopped writing (stuck, killed) is not a busy controller — the reboot exists for exactly
|
|
// that box. RED-PROOF: drop the freshness check → refused → FAILS.
|
|
func TestOpReboot_StaleBusyFileDoesNotBlock(t *testing.T) {
|
|
s, ex, rb, _, clk, _ := rbServer(t)
|
|
ex.busy = clk.t.Add(-10*time.Minute).Format(time.RFC3339) + "\na backup is running\n"
|
|
if r := pressReboot(t, s, 1); r.Outcome != "done" || rb.count() != 1 {
|
|
t.Fatalf("result %+v, reboots %d", r, rb.count())
|
|
}
|
|
}
|
|
|
|
// At most one reboot per hour per guest, and the limit survives an agent restart (the record is on disk).
|
|
// RED-PROOF: drop the hourly check → the second press reboots → FAILS.
|
|
func TestOpReboot_AtMostOncePerHour(t *testing.T) {
|
|
s, _, rb, ctl, clk, dir := rbServer(t)
|
|
if r := pressReboot(t, s, 1); r.Outcome != "done" {
|
|
t.Fatalf("first press: %+v", r)
|
|
}
|
|
clk.t = clk.t.Add(59 * time.Minute)
|
|
if r := pressReboot(t, s, 2); r.Outcome != "refused" || !strings.Contains(r.Message, "at most one reboot per hour") {
|
|
t.Fatalf("second press inside the hour: %+v", r)
|
|
}
|
|
// a restarted agent (a fresh in-memory state over the same guests dir) still refuses
|
|
s2, ox2, _, _, _, _ := opServer(t, "running")
|
|
s2.guestsDir, s2.staleLock, s2.now = dir, ctl, clk.now
|
|
s2.guestExec = &rbExec{opExec: ox2}
|
|
rb2 := &fakeRebooter{}
|
|
s2.opRebooter, s2.inFlight = rb2, &backup.InFlight{}
|
|
if r := pressReboot(t, s2, 3); r.Outcome != "refused" || rb2.count() != 0 {
|
|
t.Fatalf("after an agent restart, inside the hour: %+v", r)
|
|
}
|
|
clk.t = clk.t.Add(2 * time.Minute)
|
|
if r := pressReboot(t, s, 4); r.Outcome != "done" || rb.count() != 2 {
|
|
t.Fatalf("after the hour: %+v (reboots %d)", r, rb.count())
|
|
}
|
|
}
|
|
|
|
// The brake counts a reboot as a restart: two operator restarts and one reboot inside 15 minutes refuse the next
|
|
// restart press. RED-PROOF: drop the restarts append in operatorRebootGuest → the 4th press restarts → FAILS.
|
|
func TestOpReboot_TheBrakeCountsAReboot(t *testing.T) {
|
|
s, ex, _, _, clk, _ := rbServer(t)
|
|
for id := int64(1); id <= 2; id++ {
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, id); r.Outcome != "done" {
|
|
t.Fatalf("restart %d: %+v", id, r)
|
|
}
|
|
clk.t = clk.t.Add(time.Minute)
|
|
}
|
|
if r := pressReboot(t, s, 3); r.Outcome != "done" {
|
|
t.Fatalf("reboot: %+v", r)
|
|
}
|
|
clk.t = clk.t.Add(time.Minute)
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 4, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, 4); r.Outcome != "refused" || !strings.Contains(r.Message, "3 restarts in 15 minutes") {
|
|
t.Fatalf("the 4th restart (2 restarts + 1 reboot) in 15 minutes: %+v", r)
|
|
}
|
|
if n := ex.count(9201); n != 2 {
|
|
t.Fatalf("restarts = %d, want 2", n)
|
|
}
|
|
}
|
|
|
|
// A failed `pct reboot` is a FAILED press, said plainly; the gate and the flag are given back.
|
|
func TestOpReboot_FailedRebootIsFailed(t *testing.T) {
|
|
s, _, rb, _, _, _ := rbServer(t)
|
|
rb.err = errors.New("pct reboot 9201: exit status 255")
|
|
if r := pressReboot(t, s, 1); r.Outcome != "failed" || !strings.Contains(r.Message, "failed") {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
if s.inFlight.Busy() != "" || s.rebooting(9201) {
|
|
t.Fatal("the gate or the flag outlived a failed reboot")
|
|
}
|
|
}
|
|
|
|
// While the reboot runs, the guest-power watchdog does not start the (momentarily stopped) guest and the supervisor
|
|
// does not restart its controller. RED-PROOF: drop either rebooting() check → a start or a restart → FAILS.
|
|
func TestOpReboot_TheWatchdogsLeaveARebootingGuestAlone(t *testing.T) {
|
|
s, ex, rb, ctl, _, _ := rbServer(t)
|
|
var started, restarted int
|
|
rb.after = func() {
|
|
ctl.guests[0].Status = "stopped"
|
|
s.GuestPowerTick(context.Background())
|
|
ctl.guests[0].Status = "running"
|
|
ex.mu.Lock()
|
|
ex.status[9201] = "exited"
|
|
ex.mu.Unlock()
|
|
s.ControllerSupervisorTick(context.Background())
|
|
s.ControllerSupervisorTick(context.Background())
|
|
started, restarted = len(ctl.started), ex.count(9201)
|
|
ex.mu.Lock()
|
|
ex.status[9201] = "running"
|
|
ex.mu.Unlock()
|
|
}
|
|
if r := pressReboot(t, s, 1); r.Outcome != "done" {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
if started != 0 || restarted != 0 {
|
|
t.Fatalf("during the reboot: guest-power started %d, the supervisor restarted %d — both must leave it to pct reboot", started, restarted)
|
|
}
|
|
}
|
|
|
|
// The household's line: GET /guest/operator-reboot returns the time for a day, then nothing.
|
|
// RED-PROOF: drop the 24-hour check → the 25-hour read still returns the time → FAILS.
|
|
func TestOpReboot_TheNoticeExpiresAfterADay(t *testing.T) {
|
|
s, _, _, _, clk, _ := rbServer(t)
|
|
read := func() string {
|
|
w := httptest.NewRecorder()
|
|
s.handleOperatorRebootNotice(w, httptest.NewRequest("GET", "/guest/operator-reboot", nil), 9201)
|
|
var body struct {
|
|
Data struct {
|
|
At string `json:"at"`
|
|
} `json:"data"`
|
|
At string `json:"at"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("body %q: %v", w.Body.String(), err)
|
|
}
|
|
if body.Data.At != "" {
|
|
return body.Data.At
|
|
}
|
|
return body.At
|
|
}
|
|
if got := read(); got != "" {
|
|
t.Fatalf("no reboot yet, got %q", got)
|
|
}
|
|
at := clk.t
|
|
if r := pressReboot(t, s, 1); r.Outcome != "done" {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
clk.t = at.Add(23 * time.Hour)
|
|
if got := read(); got != at.UTC().Format(time.RFC3339) {
|
|
t.Fatalf("after 23 h: %q, want %s", got, at.UTC().Format(time.RFC3339))
|
|
}
|
|
clk.t = at.Add(25 * time.Hour)
|
|
if got := read(); got != "" {
|
|
t.Fatalf("after 25 h the line must be gone, got %q", got)
|
|
}
|
|
}
|