package localapi import ( "context" "encoding/json" "errors" "log/slog" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // F-REBOOT (Campaign 8 fault 11): a guest rebooted mid-backup never came back — stopped, unlocked, // nothing retrying, 9m47s of total appliance outage. // // Scenario A (a crashed guest is restarted), B (a deliberately stopped guest is left alone) and // C (bounded retry, then escalate). B and C are what make A safe. type fakeGuestPowerCtl struct { guests []proxmox.Guest guestsErr error locks map[int]string // vmid -> lock ("" = unlocked) onboot map[int]bool backupRun map[int]bool backupErr error started []int startErr error } func (f *fakeGuestPowerCtl) Guests(context.Context) ([]proxmox.Guest, error) { return f.guests, f.guestsErr } func (f *fakeGuestPowerCtl) Lock(_ context.Context, vmid int) (string, bool, error) { return f.locks[vmid], f.onboot[vmid], nil } func (f *fakeGuestPowerCtl) BackupRunning(_ context.Context, vmid int) (bool, error) { return f.backupRun[vmid], f.backupErr } func (f *fakeGuestPowerCtl) HasVzdumpSnapshot(context.Context, int) (bool, error) { return false, nil } func (f *fakeGuestPowerCtl) Unlock(context.Context, int) error { return nil } func (f *fakeGuestPowerCtl) DeleteVzdumpSnapshot(context.Context, int) error { return nil } func (f *fakeGuestPowerCtl) Start(_ context.Context, vmid int) error { f.started = append(f.started, vmid) return f.startErr } func gpServer(t *testing.T, ctl StaleLockController, now func() time.Time) *Server { t.Helper() s := &Server{staleLock: ctl, logger: slog.New(slog.NewTextHandler(discardW{}, nil))} if now != nil { s.now = now } else { s.now = func() time.Time { return time.Now().UTC() } } return s } type discardW struct{} func (discardW) Write(p []byte) (int, error) { return len(p), nil } // Scenario A — a guest that should be running (onboot) and is stopped-and-unlocked IS started. // // RED-PROOF: delete the `s.staleLock.Start(...)` call in recoverOneStoppedGuest (or make the whole // function return before it) → started is empty and this fails with // "guest 9201 was NOT started — this is F-REBOOT". func TestGuestPower_StoppedOnbootGuestIsStarted(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, } s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 1 || ctl.started[0] != 9201 { t.Fatalf("guest 9201 was NOT started — this is F-REBOOT (started=%v)", ctl.started) } } // Scenario B — a DELIBERATELY stopped guest (onboot:0) is never started. This is the trap: fighting // the operator makes maintenance impossible and is worse than the outage being fixed. // // RED-PROOF: remove the `if !onboot { return }` guard → the golden/scratch guest is started and this // fails with "a deliberately stopped guest (onboot:0) was started". func TestGuestPower_DeliberatelyStoppedGuestIsLeftAlone(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9100, Status: "stopped"}, {VMID: 990000, Status: "stopped"}}, locks: map[int]string{9100: "", 990000: ""}, onboot: map[int]bool{9100: false, 990000: false}, // golden + scratch } s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 0 { t.Errorf("a deliberately stopped guest (onboot:0) was started: %v", ctl.started) } } // A LOCKED stopped guest belongs to the stale-lock path, which knows how to prove a lock is stale. // This watchdog must not race it. func TestGuestPower_LockedGuestIsLeftToTheStaleLockPath(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: "snapshot-delete"}, onboot: map[int]bool{9201: true}, } s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 0 { t.Errorf("started a LOCKED guest: %v — that races the stale-lock recovery", ctl.started) } } // A guest whose vzdump is genuinely in flight must be left stopped — a stop-mode backup stops the // guest ON PURPOSE, and starting it underneath would corrupt the backup. func TestGuestPower_InFlightBackupIsNotDisturbed(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, backupRun: map[int]bool{9201: true}, } s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 0 { t.Errorf("started a guest with a vzdump in flight: %v", ctl.started) } } // Fail-safe: if we cannot confirm no backup is running, do NOT start. func TestGuestPower_UnconfirmableBackupFailsSafe(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, backupErr: errors.New("task list unavailable"), } s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 0 { t.Errorf("started despite being unable to confirm no backup is running: %v", ctl.started) } } // Unknown ownership ⇒ act on nothing. Starting a co-tenant's guest is worse than leaving ours down. func TestGuestPower_UnprovenOwnershipActsOnNothing(t *testing.T) { ctl := &fakeGuestPowerCtl{guestsErr: errors.New("pool read failed")} s := gpServer(t, ctl, nil) s.GuestPowerTick(context.Background()) if len(ctl.started) != 0 { t.Errorf("acted with unproven ownership: %v", ctl.started) } } // Scenario C — bounded retry, then escalate. A guest that will not start must NOT be retried forever. // // RED-PROOF: remove the `if st.attempts >= guestPowerMaxAttempts` branch → the sweep keeps starting // on every tick and this fails with "start attempted N times, want at most 3 — infinite retry loop". func TestGuestPower_RetryIsBoundedThenEscalates(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, startErr: errors.New("cannot start: storage offline"), } base := time.Now().UTC() now := base s := gpServer(t, ctl, func() time.Time { return now }) // Drive many ticks, advancing well past every backoff each time. for i := 0; i < 12; i++ { s.GuestPowerTick(context.Background()) now = now.Add(10 * time.Minute) } if len(ctl.started) > guestPowerMaxAttempts { t.Errorf("start attempted %d times, want at most %d — this is an infinite retry loop", len(ctl.started), guestPowerMaxAttempts) } if len(ctl.started) != guestPowerMaxAttempts { t.Errorf("start attempted %d times, want exactly %d before giving up", len(ctl.started), guestPowerMaxAttempts) } } // The backoff must actually hold a retry back — otherwise "bounded" is only bounded by luck. func TestGuestPower_BackoffDefersTheNextAttempt(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, startErr: errors.New("boom"), } base := time.Now().UTC() now := base s := gpServer(t, ctl, func() time.Time { return now }) s.GuestPowerTick(context.Background()) // attempt 1, arms a 1m backoff if len(ctl.started) != 1 { t.Fatalf("precondition: want 1 attempt, got %d", len(ctl.started)) } now = base.Add(30 * time.Second) // still inside the 1m backoff s.GuestPowerTick(context.Background()) if len(ctl.started) != 1 { t.Errorf("retried inside the backoff window (%d attempts) — the bound is not being honoured", len(ctl.started)) } now = base.Add(90 * time.Second) // past it s.GuestPowerTick(context.Background()) if len(ctl.started) != 2 { t.Errorf("did not retry after the backoff lapsed (%d attempts)", len(ctl.started)) } } // A guest that comes back healthy must lose its attempt history, so it does not carry old failures // into the next incident. func TestGuestPower_RunningGuestClearsAttemptHistory(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, startErr: errors.New("boom"), } base := time.Now().UTC() now := base s := gpServer(t, ctl, func() time.Time { return now }) s.GuestPowerTick(context.Background()) if _, due := s.guestPowerDue(9201); due { t.Error("precondition: a backoff should be armed after a failed start") } // it comes back up ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}} s.GuestPowerTick(context.Background()) if _, due := s.guestPowerDue(9201); !due { t.Error("attempt history survived the guest coming back healthy") } } // R-1383 (D78): the give-up was ONE journal line while the household's apps were off. It now rides the host report: // nothing before the give-up (and an untyped nil, so the report omits the field), the guest with the time and the // attempt count after it, and nothing again once the guest runs. // COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/B/redproof-agent-guest-power.txt). func TestGuestPower_GiveUpIsReportedUntilTheGuestRuns(t *testing.T) { ctl := &fakeGuestPowerCtl{ guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, locks: map[int]string{9201: ""}, onboot: map[int]bool{9201: true}, startErr: errors.New("cannot start: storage offline"), } now := time.Date(2026, 11, 14, 2, 0, 0, 0, time.UTC) s := gpServer(t, ctl, func() time.Time { return now }) if st := s.GuestPowerGaveUpStanza(); st != nil { t.Fatalf("before any attempt the stanza is %#v, want an untyped nil (the report omits the field)", st) } for i := 0; i < guestPowerMaxAttempts; i++ { // the three failed starts s.GuestPowerTick(context.Background()) if g := s.GuestPowerGaveUp(); len(g) != 0 { t.Fatalf("after %d failed start(s) the watchdog still tries, but it reports a give-up: %+v", i+1, g) } now = now.Add(10 * time.Minute) } gaveUpAt := now s.GuestPowerTick(context.Background()) // the tick that gives up now = now.Add(time.Hour) s.GuestPowerTick(context.Background()) g := s.GuestPowerGaveUp() if len(g) != 1 || g[0].VMID != 9201 || g[0].Attempts != guestPowerMaxAttempts || g[0].Since != gaveUpAt.Format(time.RFC3339) { t.Fatalf("give-up list = %+v, want guest 9201, %d attempts, since %s (the give-up's time, not a later tick's)", g, guestPowerMaxAttempts, gaveUpAt.Format(time.RFC3339)) } b, _ := json.Marshal(s.GuestPowerGaveUpStanza()) if string(b) != `[{"vmid":9201,"since":"2026-11-14T02:30:00Z","attempts":3}]` { t.Fatalf("the stanza on the wire = %s", b) } // the operator starts the guest: the next sweep sees it running and the list is empty again ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}} s.GuestPowerTick(context.Background()) if st := s.GuestPowerGaveUpStanza(); st != nil { t.Fatalf("the guest runs again and the stanza is still %#v", st) } }