223 lines
9.1 KiB
Go
223 lines
9.1 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
)
|
|
|
|
// Operator brief 2026-10-29, Part 1: „Restart controller" from the hub. The agent's closed list holds one verb; the act
|
|
// is the supervisor's own restart with the supervisor's guards, and the brake counts operator restarts too.
|
|
|
|
// opExec answers the supervisor's fake plus the health read the operator restart waits on.
|
|
type opExec struct {
|
|
*supExec
|
|
health string // what {{.State.Health.Status}} reads after the restart
|
|
}
|
|
|
|
func (f *opExec) GuestExec(ctx context.Context, vmid int, args ...string) (string, error) {
|
|
if len(args) >= 5 && args[0] == "docker" && args[1] == "inspect" && args[3] == opHealthTemplate {
|
|
f.mu.Lock()
|
|
st := f.status[vmid]
|
|
f.mu.Unlock()
|
|
if st == "" {
|
|
return "", errors.New("pct exec: exit status 1: Error: No such object: felhom-controller")
|
|
}
|
|
return map[bool]string{true: "true", false: "false"}[st == "running"] + "|" + f.health + "\n", nil
|
|
}
|
|
return f.supExec.GuestExec(ctx, vmid, args...)
|
|
}
|
|
|
|
func opServer(t *testing.T, status string) (*Server, *opExec, *fakeGuestPowerCtl, *supClock, string, *atomic.Int32) {
|
|
t.Helper()
|
|
ex := &opExec{supExec: &supExec{status: map[int]string{9201: status}, onRestart: "running"}, health: "healthy"}
|
|
ctl := runningGuest(9201)
|
|
s, clk, dir := supServer(t, nil, ctl, 9201)
|
|
s.guestExec = ex
|
|
s.opHealthPoll = time.Millisecond
|
|
s.opHealthWait = 2 * time.Second
|
|
var triggered atomic.Int32
|
|
s.SetOperatorActionTrigger(func() { triggered.Add(1) })
|
|
return s, ex, ctl, clk, dir, &triggered
|
|
}
|
|
|
|
func env(acts ...hub.OperatorAction) *hub.ControlEnvelope {
|
|
return &hub.ControlEnvelope{OperatorActions: acts}
|
|
}
|
|
|
|
// waitResult waits for the action's result (the act runs in the background).
|
|
func waitResult(t *testing.T, s *Server, id int64) hub.OperatorActionResult {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
for _, r := range s.OperatorActionResults() {
|
|
if r.ID == id {
|
|
return r
|
|
}
|
|
}
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
t.Fatalf("no result for #%d", id)
|
|
return hub.OperatorActionResult{}
|
|
}
|
|
|
|
// RED-PROOF: seen failing before the list existed (build failure, then the names).
|
|
func TestOpActions_AgentClosedList(t *testing.T) {
|
|
if got, want := OperatorActionNames(), []string{"restart_controller"}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("the agent's operator actions = %v, want exactly %v — a new verb needs the operator's word", got, want)
|
|
}
|
|
}
|
|
|
|
// Anything outside the list — and a bad argument — is refused and NOTHING is called.
|
|
func TestOpActions_UnknownOrBadArgIsRefused(t *testing.T) {
|
|
s, ex, _, _, _, _ := opServer(t, "running")
|
|
s.OnEnvelope(context.Background(), env(
|
|
hub.OperatorAction{ID: 1, Action: "reboot_guest", Arg: "9201"},
|
|
hub.OperatorAction{ID: 2, Action: "restart_controller", Arg: "abc"},
|
|
hub.OperatorAction{ID: 3, Action: "restart_controller", Arg: "9202"}, // not provisioned here
|
|
))
|
|
for _, id := range []int64{1, 2, 3} {
|
|
if r := waitResult(t, s, id); r.Outcome != "refused" {
|
|
t.Errorf("#%d: %+v, want refused", id, r)
|
|
}
|
|
}
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("a refused action restarted the controller (%d)", n)
|
|
}
|
|
}
|
|
|
|
// The press restarts ONCE, says how long the controller took to answer, asks for a report at once, and re-sends the
|
|
// result until the hub stops listing the id. RED-PROOF: drop the GuestExec restart → restarts 0 → FAILS.
|
|
func TestOpActions_RestartReportsTheAnswerTime(t *testing.T) {
|
|
s, ex, _, _, _, trig := opServer(t, "running")
|
|
a := hub.OperatorAction{ID: 7, Action: "restart_controller", Arg: "9201"}
|
|
s.OnEnvelope(context.Background(), env(a))
|
|
r := waitResult(t, s, 7)
|
|
if r.Outcome != "done" || !strings.Contains(r.Message, "answered after") {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("restarts = %d, want 1", n)
|
|
}
|
|
if trig.Load() == 0 {
|
|
t.Fatal("the agent did not ask for a report — the result would wait 15 minutes")
|
|
}
|
|
s.OnEnvelope(context.Background(), env(a)) // the hub still lists it: re-send, never act again
|
|
time.Sleep(100 * time.Millisecond) // a wrong second act runs in the background — give it time to show
|
|
if n := ex.count(9201); n != 1 {
|
|
t.Fatalf("a re-listed id acted again (%d)", n)
|
|
}
|
|
if len(s.OperatorActionResults()) != 1 {
|
|
t.Fatal("the result must be re-sent while listed")
|
|
}
|
|
s.OnEnvelope(context.Background(), env()) // the hub recorded it
|
|
if left := s.OperatorActionResults(); len(left) != 0 {
|
|
t.Fatalf("a recorded result is still sent: %+v", left)
|
|
}
|
|
}
|
|
|
|
// The point of the agent doing it: a STOPPED controller is restarted too (the controller cannot restart itself).
|
|
func TestOpActions_WorksWithTheControllerStopped(t *testing.T) {
|
|
s, ex, _, _, _, _ := opServer(t, "exited")
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 8, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, 8); r.Outcome != "done" || ex.count(9201) != 1 {
|
|
t.Fatalf("result %+v, restarts %d", r, ex.count(9201))
|
|
}
|
|
}
|
|
|
|
// Each supervisor guard refuses the press, with the reason. RED-PROOF: drop the lock check → the locked case
|
|
// restarts → FAILS (each guard seen failing by its own mutation).
|
|
func TestOpActions_GuardsRefuse(t *testing.T) {
|
|
cases := []struct {
|
|
name, want string
|
|
setup func(s *Server, ctl *fakeGuestPowerCtl, dir string)
|
|
}{
|
|
{"parked", "parked", func(_ *Server, _ *fakeGuestPowerCtl, dir string) {
|
|
if err := os.WriteFile(filepath.Join(dir, "9201", ControllerParkedMarker), nil, 0o600); err != nil {
|
|
panic(err)
|
|
}
|
|
}},
|
|
{"swap", "swap", func(s *Server, _ *fakeGuestPowerCtl, _ string) { s.swapInFlight[9201] = true }},
|
|
{"locked", "locked", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.locks[9201] = "backup" }},
|
|
{"vzdump", "backup", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.backupRun = map[int]bool{9201: true} }},
|
|
{"vzdump unknown", "backup", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.backupErr = errors.New("tasks unreadable") }},
|
|
{"guest stopped", "not running", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.guests[0].Status = "stopped" }},
|
|
{"guest list", "guest list", func(_ *Server, ctl *fakeGuestPowerCtl, _ string) { ctl.guestsErr = errors.New("api down") }},
|
|
}
|
|
for i, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, ex, ctl, _, dir, _ := opServer(t, "running")
|
|
tc.setup(s, ctl, dir)
|
|
id := int64(100 + i)
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"}))
|
|
r := waitResult(t, s, id)
|
|
if r.Outcome != "refused" || !strings.Contains(strings.ToLower(r.Message), tc.want) {
|
|
t.Fatalf("result %+v, want refused naming %q", r, tc.want)
|
|
}
|
|
if n := ex.count(9201); n != 0 {
|
|
t.Fatalf("guard %q did not hold: restarted %d", tc.name, n)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The thrash brake counts operator restarts AND the supervisor's: 3 in 15 minutes → the next press is refused, and
|
|
// presses stay refused for 30 minutes. RED-PROOF: drop the brake → the 4th press restarts → FAILS.
|
|
func TestOpActions_TheBrakeCountsEveryRestart(t *testing.T) {
|
|
s, ex, _, clk, _, _ := opServer(t, "running")
|
|
// one restart by the supervisor itself
|
|
ex.mu.Lock()
|
|
ex.status[9201] = "exited"
|
|
ex.mu.Unlock()
|
|
s.ControllerSupervisorTick(context.Background())
|
|
s.ControllerSupervisorTick(context.Background())
|
|
if ex.count(9201) != 1 {
|
|
t.Fatalf("setup: the supervisor did not restart (%d)", ex.count(9201))
|
|
}
|
|
for id := int64(1); id <= 2; id++ {
|
|
clk.t = clk.t.Add(time.Minute)
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: id, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, id); r.Outcome != "done" {
|
|
t.Fatalf("press %d: %+v", id, r)
|
|
}
|
|
}
|
|
clk.t = clk.t.Add(time.Minute)
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 3, Action: "restart_controller", Arg: "9201"}))
|
|
r := waitResult(t, s, 3)
|
|
if r.Outcome != "refused" || !strings.Contains(r.Message, "3 restarts in 15 minutes") {
|
|
t.Fatalf("the 4th restart in 15 minutes: %+v", r)
|
|
}
|
|
if n := ex.count(9201); n != 3 {
|
|
t.Fatalf("restarts = %d, want 3", n)
|
|
}
|
|
clk.t = clk.t.Add(20 * time.Minute) // the window has passed, but the 30-minute refusal has not
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 4, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, 4); r.Outcome != "refused" {
|
|
t.Fatalf("inside the 30-minute refusal: %+v", r)
|
|
}
|
|
clk.t = clk.t.Add(11 * time.Minute)
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 5, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, 5); r.Outcome != "done" {
|
|
t.Fatalf("after the refusal: %+v", r)
|
|
}
|
|
}
|
|
|
|
// A controller that does not come back healthy is a FAILED press, said plainly.
|
|
func TestOpActions_NoHealthyAnswerIsFailed(t *testing.T) {
|
|
s, ex, _, _, _, _ := opServer(t, "running")
|
|
ex.health = "unhealthy"
|
|
s.opHealthWait = 30 * time.Millisecond
|
|
s.OnEnvelope(context.Background(), env(hub.OperatorAction{ID: 9, Action: "restart_controller", Arg: "9201"}))
|
|
if r := waitResult(t, s, 9); r.Outcome != "failed" || !strings.Contains(r.Message, "no healthy answer") {
|
|
t.Fatalf("result %+v", r)
|
|
}
|
|
}
|