admin
a1135018c7
gates / gates (push) Successful in 42s
The job runner clears a job with its outcome, a closed reason code and the op's name. The guest-power watchdog's give-up rides the host report until the guest runs. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
284 lines
11 KiB
Go
284 lines
11 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// F-REBOOT (Campaign 8 fault 11): a guest rebooted mid-backup never came back — stopped, unlocked,
|
|
// nothing retrying, 9m47s of total appliance outage.
|
|
//
|
|
// Scenario A (a crashed guest is restarted), B (a deliberately stopped guest is left alone) and
|
|
// C (bounded retry, then escalate). B and C are what make A safe.
|
|
|
|
type fakeGuestPowerCtl struct {
|
|
guests []proxmox.Guest
|
|
guestsErr error
|
|
locks map[int]string // vmid -> lock ("" = unlocked)
|
|
onboot map[int]bool
|
|
backupRun map[int]bool
|
|
backupErr error
|
|
started []int
|
|
startErr error
|
|
}
|
|
|
|
func (f *fakeGuestPowerCtl) Guests(context.Context) ([]proxmox.Guest, error) {
|
|
return f.guests, f.guestsErr
|
|
}
|
|
func (f *fakeGuestPowerCtl) Lock(_ context.Context, vmid int) (string, bool, error) {
|
|
return f.locks[vmid], f.onboot[vmid], nil
|
|
}
|
|
func (f *fakeGuestPowerCtl) BackupRunning(_ context.Context, vmid int) (bool, error) {
|
|
return f.backupRun[vmid], f.backupErr
|
|
}
|
|
func (f *fakeGuestPowerCtl) HasVzdumpSnapshot(context.Context, int) (bool, error) { return false, nil }
|
|
func (f *fakeGuestPowerCtl) Unlock(context.Context, int) error { return nil }
|
|
func (f *fakeGuestPowerCtl) DeleteVzdumpSnapshot(context.Context, int) error { return nil }
|
|
func (f *fakeGuestPowerCtl) Start(_ context.Context, vmid int) error {
|
|
f.started = append(f.started, vmid)
|
|
return f.startErr
|
|
}
|
|
|
|
func gpServer(t *testing.T, ctl StaleLockController, now func() time.Time) *Server {
|
|
t.Helper()
|
|
s := &Server{staleLock: ctl, logger: slog.New(slog.NewTextHandler(discardW{}, nil))}
|
|
if now != nil {
|
|
s.now = now
|
|
} else {
|
|
s.now = func() time.Time { return time.Now().UTC() }
|
|
}
|
|
return s
|
|
}
|
|
|
|
type discardW struct{}
|
|
|
|
func (discardW) Write(p []byte) (int, error) { return len(p), nil }
|
|
|
|
// Scenario A — a guest that should be running (onboot) and is stopped-and-unlocked IS started.
|
|
//
|
|
// RED-PROOF: delete the `s.staleLock.Start(...)` call in recoverOneStoppedGuest (or make the whole
|
|
// function return before it) → started is empty and this fails with
|
|
// "guest 9201 was NOT started — this is F-REBOOT".
|
|
func TestGuestPower_StoppedOnbootGuestIsStarted(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
|
|
s.GuestPowerTick(context.Background())
|
|
|
|
if len(ctl.started) != 1 || ctl.started[0] != 9201 {
|
|
t.Fatalf("guest 9201 was NOT started — this is F-REBOOT (started=%v)", ctl.started)
|
|
}
|
|
}
|
|
|
|
// Scenario B — a DELIBERATELY stopped guest (onboot:0) is never started. This is the trap: fighting
|
|
// the operator makes maintenance impossible and is worse than the outage being fixed.
|
|
//
|
|
// RED-PROOF: remove the `if !onboot { return }` guard → the golden/scratch guest is started and this
|
|
// fails with "a deliberately stopped guest (onboot:0) was started".
|
|
func TestGuestPower_DeliberatelyStoppedGuestIsLeftAlone(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9100, Status: "stopped"}, {VMID: 990000, Status: "stopped"}},
|
|
locks: map[int]string{9100: "", 990000: ""},
|
|
onboot: map[int]bool{9100: false, 990000: false}, // golden + scratch
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
|
|
s.GuestPowerTick(context.Background())
|
|
|
|
if len(ctl.started) != 0 {
|
|
t.Errorf("a deliberately stopped guest (onboot:0) was started: %v", ctl.started)
|
|
}
|
|
}
|
|
|
|
// A LOCKED stopped guest belongs to the stale-lock path, which knows how to prove a lock is stale.
|
|
// This watchdog must not race it.
|
|
func TestGuestPower_LockedGuestIsLeftToTheStaleLockPath(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: "snapshot-delete"},
|
|
onboot: map[int]bool{9201: true},
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 0 {
|
|
t.Errorf("started a LOCKED guest: %v — that races the stale-lock recovery", ctl.started)
|
|
}
|
|
}
|
|
|
|
// A guest whose vzdump is genuinely in flight must be left stopped — a stop-mode backup stops the
|
|
// guest ON PURPOSE, and starting it underneath would corrupt the backup.
|
|
func TestGuestPower_InFlightBackupIsNotDisturbed(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
backupRun: map[int]bool{9201: true},
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 0 {
|
|
t.Errorf("started a guest with a vzdump in flight: %v", ctl.started)
|
|
}
|
|
}
|
|
|
|
// Fail-safe: if we cannot confirm no backup is running, do NOT start.
|
|
func TestGuestPower_UnconfirmableBackupFailsSafe(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
backupErr: errors.New("task list unavailable"),
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 0 {
|
|
t.Errorf("started despite being unable to confirm no backup is running: %v", ctl.started)
|
|
}
|
|
}
|
|
|
|
// Unknown ownership ⇒ act on nothing. Starting a co-tenant's guest is worse than leaving ours down.
|
|
func TestGuestPower_UnprovenOwnershipActsOnNothing(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{guestsErr: errors.New("pool read failed")}
|
|
s := gpServer(t, ctl, nil)
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 0 {
|
|
t.Errorf("acted with unproven ownership: %v", ctl.started)
|
|
}
|
|
}
|
|
|
|
// Scenario C — bounded retry, then escalate. A guest that will not start must NOT be retried forever.
|
|
//
|
|
// RED-PROOF: remove the `if st.attempts >= guestPowerMaxAttempts` branch → the sweep keeps starting
|
|
// on every tick and this fails with "start attempted N times, want at most 3 — infinite retry loop".
|
|
func TestGuestPower_RetryIsBoundedThenEscalates(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
startErr: errors.New("cannot start: storage offline"),
|
|
}
|
|
base := time.Now().UTC()
|
|
now := base
|
|
s := gpServer(t, ctl, func() time.Time { return now })
|
|
|
|
// Drive many ticks, advancing well past every backoff each time.
|
|
for i := 0; i < 12; i++ {
|
|
s.GuestPowerTick(context.Background())
|
|
now = now.Add(10 * time.Minute)
|
|
}
|
|
|
|
if len(ctl.started) > guestPowerMaxAttempts {
|
|
t.Errorf("start attempted %d times, want at most %d — this is an infinite retry loop",
|
|
len(ctl.started), guestPowerMaxAttempts)
|
|
}
|
|
if len(ctl.started) != guestPowerMaxAttempts {
|
|
t.Errorf("start attempted %d times, want exactly %d before giving up", len(ctl.started), guestPowerMaxAttempts)
|
|
}
|
|
}
|
|
|
|
// The backoff must actually hold a retry back — otherwise "bounded" is only bounded by luck.
|
|
func TestGuestPower_BackoffDefersTheNextAttempt(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
startErr: errors.New("boom"),
|
|
}
|
|
base := time.Now().UTC()
|
|
now := base
|
|
s := gpServer(t, ctl, func() time.Time { return now })
|
|
|
|
s.GuestPowerTick(context.Background()) // attempt 1, arms a 1m backoff
|
|
if len(ctl.started) != 1 {
|
|
t.Fatalf("precondition: want 1 attempt, got %d", len(ctl.started))
|
|
}
|
|
now = base.Add(30 * time.Second) // still inside the 1m backoff
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 1 {
|
|
t.Errorf("retried inside the backoff window (%d attempts) — the bound is not being honoured", len(ctl.started))
|
|
}
|
|
now = base.Add(90 * time.Second) // past it
|
|
s.GuestPowerTick(context.Background())
|
|
if len(ctl.started) != 2 {
|
|
t.Errorf("did not retry after the backoff lapsed (%d attempts)", len(ctl.started))
|
|
}
|
|
}
|
|
|
|
// A guest that comes back healthy must lose its attempt history, so it does not carry old failures
|
|
// into the next incident.
|
|
func TestGuestPower_RunningGuestClearsAttemptHistory(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
startErr: errors.New("boom"),
|
|
}
|
|
base := time.Now().UTC()
|
|
now := base
|
|
s := gpServer(t, ctl, func() time.Time { return now })
|
|
s.GuestPowerTick(context.Background())
|
|
|
|
if _, due := s.guestPowerDue(9201); due {
|
|
t.Error("precondition: a backoff should be armed after a failed start")
|
|
}
|
|
// it comes back up
|
|
ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}}
|
|
s.GuestPowerTick(context.Background())
|
|
if _, due := s.guestPowerDue(9201); !due {
|
|
t.Error("attempt history survived the guest coming back healthy")
|
|
}
|
|
}
|
|
|
|
// R-1383 (D78): the give-up was ONE journal line while the household's apps were off. It now rides the host report:
|
|
// nothing before the give-up (and an untyped nil, so the report omits the field), the guest with the time and the
|
|
// attempt count after it, and nothing again once the guest runs.
|
|
// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/B/redproof-agent-guest-power.txt).
|
|
func TestGuestPower_GiveUpIsReportedUntilTheGuestRuns(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
|
locks: map[int]string{9201: ""},
|
|
onboot: map[int]bool{9201: true},
|
|
startErr: errors.New("cannot start: storage offline"),
|
|
}
|
|
now := time.Date(2026, 11, 14, 2, 0, 0, 0, time.UTC)
|
|
s := gpServer(t, ctl, func() time.Time { return now })
|
|
if st := s.GuestPowerGaveUpStanza(); st != nil {
|
|
t.Fatalf("before any attempt the stanza is %#v, want an untyped nil (the report omits the field)", st)
|
|
}
|
|
for i := 0; i < guestPowerMaxAttempts; i++ { // the three failed starts
|
|
s.GuestPowerTick(context.Background())
|
|
if g := s.GuestPowerGaveUp(); len(g) != 0 {
|
|
t.Fatalf("after %d failed start(s) the watchdog still tries, but it reports a give-up: %+v", i+1, g)
|
|
}
|
|
now = now.Add(10 * time.Minute)
|
|
}
|
|
gaveUpAt := now
|
|
s.GuestPowerTick(context.Background()) // the tick that gives up
|
|
now = now.Add(time.Hour)
|
|
s.GuestPowerTick(context.Background())
|
|
g := s.GuestPowerGaveUp()
|
|
if len(g) != 1 || g[0].VMID != 9201 || g[0].Attempts != guestPowerMaxAttempts || g[0].Since != gaveUpAt.Format(time.RFC3339) {
|
|
t.Fatalf("give-up list = %+v, want guest 9201, %d attempts, since %s (the give-up's time, not a later tick's)", g, guestPowerMaxAttempts, gaveUpAt.Format(time.RFC3339))
|
|
}
|
|
b, _ := json.Marshal(s.GuestPowerGaveUpStanza())
|
|
if string(b) != `[{"vmid":9201,"since":"2026-11-14T02:30:00Z","attempts":3}]` {
|
|
t.Fatalf("the stanza on the wire = %s", b)
|
|
}
|
|
// the operator starts the guest: the next sweep sees it running and the list is empty again
|
|
ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}}
|
|
s.GuestPowerTick(context.Background())
|
|
if st := s.GuestPowerGaveUpStanza(); st != nil {
|
|
t.Fatalf("the guest runs again and the stanza is still %#v", st)
|
|
}
|
|
}
|