package localapi import ( "context" "io" "log/slog" "testing" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // fakeStaleLock is a scripted StaleLockController: it answers the reads from per-vmid maps and records // the mutating calls (unlock/delsnapshot/start) so a test can assert the exact recovery sequence. type fakeStaleLock struct { guests []proxmox.Guest lock map[int]string onboot map[int]bool snap map[int]bool // a dangling vzdump snapshot exists running map[int]bool // a vzdump task is genuinely in-flight unlocked []int delsnap []int started []int lockErr map[int]error runningErr map[int]error } func (f *fakeStaleLock) Guests(context.Context) ([]proxmox.Guest, error) { return f.guests, nil } func (f *fakeStaleLock) Lock(_ context.Context, vmid int) (string, bool, error) { if err := f.lockErr[vmid]; err != nil { return "", false, err } return f.lock[vmid], f.onboot[vmid], nil } func (f *fakeStaleLock) BackupRunning(_ context.Context, vmid int) (bool, error) { if err := f.runningErr[vmid]; err != nil { return false, err } return f.running[vmid], nil } func (f *fakeStaleLock) HasVzdumpSnapshot(_ context.Context, vmid int) (bool, error) { return f.snap[vmid], nil } func (f *fakeStaleLock) Unlock(_ context.Context, vmid int) error { f.unlocked = append(f.unlocked, vmid) return nil } func (f *fakeStaleLock) DeleteVzdumpSnapshot(_ context.Context, vmid int) error { f.delsnap = append(f.delsnap, vmid) return nil } func (f *fakeStaleLock) Start(_ context.Context, vmid int) error { f.started = append(f.started, vmid) return nil } func staleLockServer(f *fakeStaleLock) *Server { return &Server{staleLock: f, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} } func contains(xs []int, v int) bool { for _, x := range xs { if x == v { return true } } return false } // TestStaleLock_RecoversStoppedLockedGuest is the F2-b core: a stopped, onboot guest with a // snapshot-delete lock + a dangling vzdump snapshot is unlocked, its snapshot deleted, and it is // started — the exact reboot-during-backup recovery. func TestStaleLock_RecoversStoppedLockedGuest(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, lock: map[int]string{9201: "snapshot-delete"}, onboot: map[int]bool{9201: true}, snap: map[int]bool{9201: true}, running: map[int]bool{9201: false}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if !contains(f.unlocked, 9201) { t.Fatalf("expected pct unlock for 9201, unlocked=%v", f.unlocked) } if !contains(f.delsnap, 9201) { t.Fatalf("expected dangling vzdump snapshot delete for 9201, delsnap=%v", f.delsnap) } if !contains(f.started, 9201) { t.Fatalf("expected start (onboot, stopped) for 9201, started=%v", f.started) } } // TestStaleLock_NoLockTouchesNothing is the COMPANION proof: a healthy guest with no lock has NONE of // the mutating ops invoked — recovery acts only on the stale state. func TestStaleLock_NoLockTouchesNothing(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "running"}}, lock: map[int]string{9201: ""}, // unlocked onboot: map[int]bool{9201: true}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if len(f.unlocked)+len(f.delsnap)+len(f.started) != 0 { t.Fatalf("no-lock guest must be untouched; got unlocked=%v delsnap=%v started=%v", f.unlocked, f.delsnap, f.started) } } // TestStaleLock_NonBackupLockLeftAlone: a non-vzdump lock (e.g. migrate) is NEVER cleared — scope is // strictly the two backup locks. func TestStaleLock_NonBackupLockLeftAlone(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "running"}}, lock: map[int]string{9201: "migrate"}, onboot: map[int]bool{9201: true}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if len(f.unlocked) != 0 { t.Fatalf("a migrate lock must not be cleared; unlocked=%v", f.unlocked) } } // TestStaleLock_OnbootZeroNotStarted: a stale-locked guest with onboot=0 (e.g. the golden) is unlocked // + snapshot-cleaned but NOT started — a deliberately-stopped guest stays stopped. func TestStaleLock_OnbootZeroNotStarted(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9100, Status: "stopped"}}, lock: map[int]string{9100: "snapshot-delete"}, onboot: map[int]bool{9100: false}, snap: map[int]bool{9100: true}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if !contains(f.unlocked, 9100) || !contains(f.delsnap, 9100) { t.Fatalf("onboot=0 guest should still be unlocked + snapshot-cleaned; unlocked=%v delsnap=%v", f.unlocked, f.delsnap) } if contains(f.started, 9100) { t.Fatalf("onboot=0 guest must NOT be started; started=%v", f.started) } } // TestStaleLock_DelsnapOnlyWhenSnapshotExists: a stop/suspend-mode interruption (lock but NO dangling // snapshot) is unlocked + started, but delsnapshot is NOT called. func TestStaleLock_DelsnapOnlyWhenSnapshotExists(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, lock: map[int]string{9201: "backup"}, onboot: map[int]bool{9201: true}, snap: map[int]bool{9201: false}, // no dangling snapshot } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if !contains(f.unlocked, 9201) || !contains(f.started, 9201) { t.Fatalf("expected unlock + start; unlocked=%v started=%v", f.unlocked, f.started) } if len(f.delsnap) != 0 { t.Fatalf("delsnapshot must not run without a dangling snapshot; delsnap=%v", f.delsnap) } } // TestStaleLock_RunningBackupNotCleared is the INVARIANT guard: a backup lock present WITH a genuinely // in-flight vzdump must NOT be cleared (clearing a live backup's lock corrupts it). func TestStaleLock_RunningBackupNotCleared(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "running"}}, lock: map[int]string{9201: "backup"}, onboot: map[int]bool{9201: true}, running: map[int]bool{9201: true}, // a real backup is running } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if len(f.unlocked)+len(f.delsnap)+len(f.started) != 0 { t.Fatalf("a live backup's lock must be left alone; unlocked=%v delsnap=%v started=%v", f.unlocked, f.delsnap, f.started) } } // TestStaleLock_RunningProbeErrorFailsSafe: if the no-backup-running confirmation errors, the lock is // LEFT (fail-safe) rather than blind-cleared. func TestStaleLock_RunningProbeErrorFailsSafe(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, lock: map[int]string{9201: "snapshot-delete"}, onboot: map[int]bool{9201: true}, runningErr: map[int]error{9201: context.DeadlineExceeded}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if len(f.unlocked) != 0 { t.Fatalf("an unconfirmable backup state must fail safe (no unlock); unlocked=%v", f.unlocked) } } // TestStaleLock_AlreadyRunningNotRestarted: a bare agent restart can find the guest UP behind a stale // lock — it is unlocked but NOT (re)started. func TestStaleLock_AlreadyRunningNotRestarted(t *testing.T) { f := &fakeStaleLock{ guests: []proxmox.Guest{{VMID: 9201, Status: "running"}}, lock: map[int]string{9201: "snapshot-delete"}, onboot: map[int]bool{9201: true}, snap: map[int]bool{9201: true}, } staleLockServer(f).RecoverStaleLockedGuests(context.Background()) if !contains(f.unlocked, 9201) { t.Fatalf("a running but stale-locked guest should still be unlocked; unlocked=%v", f.unlocked) } if contains(f.started, 9201) { t.Fatalf("an already-running guest must NOT be started; started=%v", f.started) } } // TestStaleLock_NilControllerNoop: an unwired controller is a safe no-op (the optional-feature contract). func TestStaleLock_NilControllerNoop(t *testing.T) { s := &Server{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} s.RecoverStaleLockedGuests(context.Background()) // must not panic }