196 lines
8.3 KiB
Go
196 lines
8.3 KiB
Go
package localapi
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// R-1008: after a power cut Proxmox keeps the dead vzdump task in `tasks?source=active` (measured on demo-hp
|
|
// 2026-10-14: still RUNNING after the boot, started 136 s before it). A task that started before this boot is
|
|
// dead, not in flight. These tests pin the helper (vzdumpInFlight), the production controller, and three of the
|
|
// four callers end to end; the fourth (the controller supervisor) by the structural pin at the bottom.
|
|
|
|
const testBoot = int64(1791967481) // the C3 boot (power-cut-fixes-2026-10-14)
|
|
|
|
func vz(vmid string, start int64) proxmox.TaskStatus {
|
|
return proxmox.TaskStatus{UPID: "UPID:demo-hp:001E6EEC:00000001:6ACF4071:vzdump:" + vmid + ":felhom-agent@pve!agent:",
|
|
Type: "vzdump", ID: vmid, Status: "RUNNING", StartTime: start}
|
|
}
|
|
|
|
// RED-PROOF: make vzdumpInFlight return true for every matching task (drop the `start >= boot` test) → this fails.
|
|
func TestVzdumpInFlight_PreBootTaskIsDead(t *testing.T) {
|
|
running, dead := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", testBoot-136)}, 9201, testBoot, true)
|
|
if running || len(dead) != 1 {
|
|
t.Fatalf("a task from before the boot must be dead: running=%v dead=%d", running, len(dead))
|
|
}
|
|
}
|
|
|
|
func TestVzdumpInFlight_PostBootTaskRuns(t *testing.T) {
|
|
for _, start := range []int64{testBoot, testBoot + 600} {
|
|
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", start)}, 9201, testBoot, true); !running {
|
|
t.Fatalf("a task started at %d (boot %d) must be in flight", start, testBoot)
|
|
}
|
|
}
|
|
}
|
|
|
|
// RED-PROOF: drop the `!bootOK` early return → a pre-boot task with an unreadable boot is called dead → fails.
|
|
func TestVzdumpInFlight_UnreadableBootFailsSafe(t *testing.T) {
|
|
// the boot value is a real one on purpose: only bootOK=false may make this „running"
|
|
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", testBoot-136)}, 9201, testBoot, false); !running {
|
|
t.Fatal("an unreadable boot time must count as running (fail safe)")
|
|
}
|
|
}
|
|
|
|
// RED-PROOF: make taskStart ignore the UPID → StartTime 0 counts as unknown → running; the first case fails.
|
|
func TestVzdumpInFlight_StartFromUPIDThenUnknownFailsSafe(t *testing.T) {
|
|
tk := vz("9201", 0) // no starttime in the list: the UPID's 6ACF4071 = 1791967345 (before the boot)
|
|
if running, dead := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, testBoot, true); running || len(dead) != 1 {
|
|
t.Fatalf("the UPID's start time must be used: running=%v", running)
|
|
}
|
|
tk.UPID = "garbage"
|
|
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, testBoot, true); !running {
|
|
t.Fatal("an unknown start time must count as running (fail safe)")
|
|
}
|
|
}
|
|
|
|
func TestVzdumpInFlight_OtherGuestOrTypeIgnored(t *testing.T) {
|
|
tasks := []proxmox.TaskStatus{vz("9202", testBoot+5), {Type: "vzsnapshot", ID: "9201", StartTime: testBoot + 5}}
|
|
if running, dead := vzdumpInFlight(tasks, 9201, testBoot, true); running || len(dead) != 0 {
|
|
t.Fatalf("other guests' and other types' tasks are not 9201's backup: running=%v dead=%d", running, len(dead))
|
|
}
|
|
}
|
|
|
|
func inflightCtrl(api *fakeStaleLockAPI, runner *recordingRunner, logger *slog.Logger) *staleLockController {
|
|
c := NewStaleLockController(api, runner, "felhom", logger).(*staleLockController)
|
|
c.bootTime = func() (int64, bool) { return testBoot, true }
|
|
return c
|
|
}
|
|
|
|
// The production controller answers through the helper, and logs a dead task ONCE.
|
|
// RED-PROOF: make BackupRunning ignore bootTime (the pre-R-1008 body) → returns true → fails.
|
|
func TestStaleLockController_BackupRunningIgnoresDeadTask(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", testBoot-136)}}
|
|
c := inflightCtrl(api, &recordingRunner{}, slog.New(slog.NewTextHandler(&buf, nil)))
|
|
for i := 0; i < 3; i++ {
|
|
if running, err := c.BackupRunning(context.Background(), 9201); err != nil || running {
|
|
t.Fatalf("dead pre-boot task reported running=%v err=%v", running, err)
|
|
}
|
|
}
|
|
if n := strings.Count(buf.String(), "backup-in-flight: a vzdump task from before this boot"); n != 1 {
|
|
t.Fatalf("the dead task must be logged exactly once, got %d", n)
|
|
}
|
|
api.tasks = []proxmox.TaskStatus{vz("9201", testBoot+30)}
|
|
if running, _ := c.BackupRunning(context.Background(), 9201); !running {
|
|
t.Fatal("a post-boot task must be running")
|
|
}
|
|
}
|
|
|
|
func poolOf(vmids ...int) proxmox.PoolInfo {
|
|
var p proxmox.PoolInfo
|
|
raw := `{"members":[`
|
|
for i, v := range vmids {
|
|
if i > 0 {
|
|
raw += ","
|
|
}
|
|
raw += `{"vmid":` + itoaT(v) + `,"type":"lxc"}`
|
|
}
|
|
raw += `]}`
|
|
_ = json.Unmarshal([]byte(raw), &p)
|
|
return p
|
|
}
|
|
|
|
// Caller 1 — the startup stale-lock recovery clears the lock a power cut left (the night of 2026-10-14 it said
|
|
// „genuinely in-flight — leaving the lock").
|
|
func TestCaller_StartupStaleLockClearsAfterPowerCut(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
start int64
|
|
cleared bool
|
|
}{{testBoot - 136, true}, {testBoot + 30, false}} {
|
|
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", tc.start)},
|
|
cfg: map[int]proxmox.GuestConfig{9201: staleCfg("backup", true)}}
|
|
runner := &recordingRunner{}
|
|
s := &Server{staleLock: inflightCtrl(api, runner, nil), logger: slog.New(slog.NewTextHandler(discardW{}, nil))}
|
|
s.recoverOneStaleLock(context.Background(), proxmox.Guest{VMID: 9201, Status: "running"})
|
|
if runner.unlocked("9201") != tc.cleared {
|
|
t.Fatalf("task start %d: unlocked=%v, want %v", tc.start, runner.unlocked("9201"), tc.cleared)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Caller 2 — the guest-power watchdog starts a stopped guest despite a dead pre-boot vzdump task.
|
|
func TestCaller_GuestPowerStartsGuestAfterPowerCut(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
start int64
|
|
started bool
|
|
}{{testBoot - 136, true}, {testBoot + 30, false}} {
|
|
api := &fakeStaleLockAPI{lxc: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, pool: poolOf(9201),
|
|
tasks: []proxmox.TaskStatus{vz("9201", tc.start)}, cfg: map[int]proxmox.GuestConfig{9201: staleCfg("", true)}}
|
|
s := gpServer(t, inflightCtrl(api, &recordingRunner{}, nil), nil)
|
|
s.GuestPowerTick(context.Background())
|
|
if (len(api.started) == 1) != tc.started {
|
|
t.Fatalf("task start %d: started=%v, want %v", tc.start, api.started, tc.started)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Caller 3 — the household power-off guard does not refuse „backup running" for a dead task.
|
|
func TestCaller_PowerOffJobBlockerIgnoresDeadTask(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
start int64
|
|
want string
|
|
}{{testBoot - 136, ""}, {testBoot + 30, PowerOffBackup}} {
|
|
rig := newPowerRig(t, 0)
|
|
rig.srv.staleLock = inflightCtrl(&fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", tc.start)}}, &recordingRunner{}, nil)
|
|
if code, _ := rig.srv.jobBlocker(context.Background(), 9201); code != tc.want {
|
|
t.Fatalf("task start %d: jobBlocker=%q, want %q", tc.start, code, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Caller 4 (the supervisor), the R-1006 cleanup (VzdumpInFlight) and the rest — structural pin: every non-test BackupRunning call in this package goes through
|
|
// s.staleLock (whose one production implementation is staleLockController, which uses vzdumpInFlight), and no
|
|
// other production type implements BackupRunning. If someone adds a second path, this fails.
|
|
func TestBackupRunning_AllCallersUseTheOneHelper(t *testing.T) {
|
|
files, _ := filepath.Glob("*.go")
|
|
call := regexp.MustCompile(`([A-Za-z_.]+)\.BackupRunning\(`)
|
|
impl := regexp.MustCompile(`func \([a-z]+ \*?([A-Za-z]+)\) BackupRunning\(`)
|
|
callers := 0
|
|
for _, f := range files {
|
|
if strings.HasSuffix(f, "_test.go") {
|
|
continue
|
|
}
|
|
b, err := os.ReadFile(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, m := range call.FindAllStringSubmatch(string(b), -1) {
|
|
if m[1] != "s.staleLock" {
|
|
t.Errorf("%s: BackupRunning called on %q, not s.staleLock", f, m[1])
|
|
}
|
|
callers++
|
|
}
|
|
for _, m := range impl.FindAllStringSubmatch(string(b), -1) {
|
|
if m[1] != "staleLockController" {
|
|
t.Errorf("%s: a second production BackupRunning on %s", f, m[1])
|
|
}
|
|
}
|
|
}
|
|
if callers != 5 {
|
|
t.Errorf("expected the five known callers (stale-lock, guest-power, power-off, supervisor, the R-1006 cleanup), found %d", callers)
|
|
}
|
|
b, _ := os.ReadFile("stalelock.go")
|
|
if !strings.Contains(string(b), "vzdumpInFlight(tasks, vmid, boot, ok)") {
|
|
t.Error("staleLockController.BackupRunning no longer answers through vzdumpInFlight")
|
|
}
|
|
}
|