R-1008: in flight = the worker process lives (Proxmox's pid+pstart rule), not a wall-clock start vs btime (security review: a clock step could make a live backup look dead)
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
# REPORT — v0.156.0: after a power cut the box clears up (R-1006, R-1008) — released, not vouched (2026-10-14)
|
||||
# REPORT — v0.156.1: after a power cut the box clears up (R-1006, R-1008) (2026-10-14)
|
||||
|
||||
Evidence: felhom.eu `documentation/audits/power-cut-fixes-2026-10-14/` — the spike (C1 the night's leftover, C2 a killed
|
||||
worker, C3 a real power cut, C4 a full disk), the 17 red-proofs.
|
||||
@@ -14,6 +14,14 @@ worker, C3 a real power cut, C4 a full disk), the 17 red-proofs.
|
||||
- On „No space left" vzdump removes its own partial (both paths): the preflight's sizing stays. But a full host root breaks
|
||||
LVM's own metadata writes (a half-removed snapshot; the janitor cannot clear it — new row).
|
||||
|
||||
## 0.156.0 is superseded — do not vouch it
|
||||
|
||||
A background security review of the pushed 0.156.0 flagged „fail-open state drift": its in-flight rule compared a task's
|
||||
wall-clock start with the kernel's `btime`, which moves when the clock is corrected after the boot (an NTP step after a
|
||||
power cut) — a live backup could then look „from before the boot" and its lock be cleared. 0.156.1 uses Proxmox's own
|
||||
liveness rule instead (the UPID's pid + pstart against `/proc/<pid>/stat` start tick and state; no clock), and fails safe
|
||||
when this agent cannot see root's processes (PID 1 not visible).
|
||||
|
||||
## What changed
|
||||
|
||||
See `CHANGELOG.md` v0.156.0: `vzdumpInFlight` behind the one `BackupRunning`; `internal/vzpartial` at start-up and before
|
||||
|
||||
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/poweroff"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
@@ -144,9 +146,8 @@ type staleLockController struct {
|
||||
runner proxmox.Runner
|
||||
pool string // ownership registry: only members of this PVE pool are ever scanned (A1)
|
||||
logger *slog.Logger // the one success-path scan-summary line; nil = silent
|
||||
// bootTime is the host's boot time (unix seconds) and whether it could be read (R-1008). nil = the
|
||||
// kernel's btime (poweroff.ProcBtime — the one reader of /proc/stat btime this agent shares).
|
||||
bootTime func() (int64, bool)
|
||||
// procDir is where taskAlive reads /proc/<pid>/stat (R-1008); "" = /proc. Tests point it at a fixture.
|
||||
procDir string
|
||||
deadMu sync.Mutex
|
||||
deadSeen map[string]bool // UPIDs of dead tasks already logged (once per task)
|
||||
}
|
||||
@@ -220,29 +221,28 @@ func (c *staleLockController) Lock(ctx context.Context, vmid int) (string, bool,
|
||||
// recovery, the guest-power watchdog, the household power-off guard, the controller supervisor): every one of
|
||||
// them reaches this method through the StaleLockController interface. R-1008: Proxmox keeps a task in
|
||||
// `tasks?source=active` after a power cut — nobody is left to end it (measured on demo-hp 2026-10-14: still
|
||||
// listed RUNNING after the boot) — so a task that STARTED BEFORE THIS BOOT is dead, not in flight.
|
||||
// listed RUNNING ~30–60 s after the boot) — so a listed task whose worker process is gone is dead, not in flight.
|
||||
func (c *staleLockController) BackupRunning(ctx context.Context, vmid int) (bool, error) {
|
||||
tasks, err := c.px.ListRunningTasks(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
bt := c.bootTime
|
||||
if bt == nil {
|
||||
bt = procBootTime
|
||||
proc := c.procDir
|
||||
if proc == "" {
|
||||
proc = "/proc"
|
||||
}
|
||||
boot, ok := bt()
|
||||
running, dead := vzdumpInFlight(tasks, vmid, boot, ok)
|
||||
running, dead := vzdumpInFlight(tasks, vmid, proc)
|
||||
for _, t := range dead {
|
||||
c.logDeadOnce(t, vmid, boot)
|
||||
c.logDeadOnce(t, vmid)
|
||||
}
|
||||
return running, nil
|
||||
}
|
||||
|
||||
// vzdumpInFlight decides, from the node's active-task list, whether a vzdump for vmid is in flight. A matching
|
||||
// task is in flight when it started at or after the host's boot; one that started before the boot is dead
|
||||
// (returned in dead, for the log). FAIL SAFE on doubt — the old answer: an unreadable boot time, or a task whose
|
||||
// start time is unknown, counts as in flight. Pure; pinned by stalelock_inflight_test.go.
|
||||
func vzdumpInFlight(tasks []proxmox.TaskStatus, vmid int, boot int64, bootOK bool) (bool, []proxmox.TaskStatus) {
|
||||
// vzdumpInFlight decides, from the node's active-task list, whether a vzdump for vmid is in flight: a matching
|
||||
// task is in flight while its worker process lives (taskAlive). A task whose worker is gone is dead (returned in dead,
|
||||
// for the log). FAIL SAFE on doubt — the old answer: a UPID that does not parse, or a /proc entry that cannot be read,
|
||||
// counts as in flight. Pure over procDir; pinned by stalelock_inflight_test.go.
|
||||
func vzdumpInFlight(tasks []proxmox.TaskStatus, vmid int, procDir string) (bool, []proxmox.TaskStatus) {
|
||||
id := strconv.Itoa(vmid)
|
||||
var dead []proxmox.TaskStatus
|
||||
for _, t := range tasks {
|
||||
@@ -250,11 +250,8 @@ func vzdumpInFlight(tasks []proxmox.TaskStatus, vmid int, boot int64, bootOK boo
|
||||
if t.Type != "vzdump" || t.ID != id {
|
||||
continue
|
||||
}
|
||||
if !bootOK || boot <= 0 {
|
||||
return true, nil
|
||||
}
|
||||
start := taskStart(t)
|
||||
if start <= 0 || start >= boot {
|
||||
alive, known := taskAlive(procDir, t.UPID)
|
||||
if !known || alive {
|
||||
return true, nil
|
||||
}
|
||||
dead = append(dead, t)
|
||||
@@ -262,24 +259,47 @@ func vzdumpInFlight(tasks []proxmox.TaskStatus, vmid int, boot int64, bootOK boo
|
||||
return false, dead
|
||||
}
|
||||
|
||||
// taskStart is the task's start time: the list's starttime field, else the one encoded in its UPID; 0 = unknown.
|
||||
func taskStart(t proxmox.TaskStatus) int64 {
|
||||
if t.StartTime > 0 {
|
||||
return t.StartTime
|
||||
// taskAlive is Proxmox's own liveness rule for a task (PVE::ProcFSTools::check_process_running): the UPID names the
|
||||
// worker's pid and its start time in clock ticks since boot (pstart); the task lives while /proc/<pid>/stat exists with
|
||||
// that same start time (field 22). No wall clock is involved — a clock step after the boot (an NTP correction after a
|
||||
// power cut) cannot make a live task look dead (the 2026-10-14 security review of the first, btime-based version).
|
||||
// known=false: the UPID does not parse, or /proc/<pid>/stat exists but cannot be read or parsed.
|
||||
func taskAlive(procDir, upid string) (alive, known bool) {
|
||||
u, err := proxmox.ParseUPID(upid)
|
||||
if err != nil || u.PID == 0 {
|
||||
return false, false
|
||||
}
|
||||
if u, err := proxmox.ParseUPID(t.UPID); err == nil {
|
||||
return int64(u.StartTime)
|
||||
raw, err := os.ReadFile(filepath.Join(procDir, strconv.FormatUint(u.PID, 10), "stat"))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// „no such process" is only an answer if this agent can see other users' processes at all (a /proc mounted
|
||||
// hidepid=, or a unit with ProtectProc=, hides root's workers — every task would then look dead). PID 1 is
|
||||
// root's; not seeing it means not knowing.
|
||||
if _, err1 := os.Stat(filepath.Join(procDir, "1", "stat")); err1 != nil {
|
||||
return false, false
|
||||
}
|
||||
return false, true // no such process in this boot: the worker is gone
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
return 0
|
||||
st := string(raw)
|
||||
i := strings.LastIndexByte(st, ')') // the comm field may hold spaces and parentheses
|
||||
if i < 0 {
|
||||
return false, false
|
||||
}
|
||||
f := strings.Fields(st[i+1:]) // f[0] is field 3 (state), so field 22 (starttime) is f[19]
|
||||
if len(f) < 20 {
|
||||
return false, false
|
||||
}
|
||||
start, err := strconv.ParseUint(f[19], 10, 64)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
// Proxmox's rule exactly: the start tick matches (a pstart of 0 is not checked) and the process is not a zombie.
|
||||
return (u.PStart == 0 || start == u.PStart) && f[0] != "Z", true
|
||||
}
|
||||
|
||||
// procBootTime reads the kernel's boot time through poweroff.ProcBtime (no second /proc/stat parser).
|
||||
func procBootTime() (int64, bool) {
|
||||
b, err := strconv.ParseInt(poweroff.ProcBtime(), 10, 64)
|
||||
return b, err == nil && b > 0
|
||||
}
|
||||
|
||||
func (c *staleLockController) logDeadOnce(t proxmox.TaskStatus, vmid int, boot int64) {
|
||||
func (c *staleLockController) logDeadOnce(t proxmox.TaskStatus, vmid int) {
|
||||
c.deadMu.Lock()
|
||||
defer c.deadMu.Unlock()
|
||||
if c.deadSeen == nil {
|
||||
@@ -289,8 +309,8 @@ func (c *staleLockController) logDeadOnce(t proxmox.TaskStatus, vmid int, boot i
|
||||
return
|
||||
}
|
||||
c.deadSeen[t.UPID] = true
|
||||
c.logger.Warn("backup-in-flight: a vzdump task from before this boot is listed as active — it is dead, not running",
|
||||
"vmid", vmid, "upid", t.UPID, "task_start", taskStart(t), "host_boot", boot)
|
||||
c.logger.Warn("backup-in-flight: a vzdump task is listed as active but its worker process is gone — it is dead, not running",
|
||||
"vmid", vmid, "upid", t.UPID)
|
||||
}
|
||||
|
||||
func (c *staleLockController) HasVzdumpSnapshot(ctx context.Context, vmid int) (bool, error) {
|
||||
|
||||
@@ -15,86 +15,157 @@ import (
|
||||
)
|
||||
|
||||
// R-1008: after a power cut Proxmox keeps the dead vzdump task in `tasks?source=active` (measured on demo-hp
|
||||
// 2026-10-14: still RUNNING after the boot, started 136 s before it). A task that started before this boot is
|
||||
// dead, not in flight. These tests pin the helper (vzdumpInFlight), the production controller, and three of the
|
||||
// four callers end to end; the fourth (the controller supervisor) by the structural pin at the bottom.
|
||||
// 2026-10-14: still RUNNING ~30–60 s after the boot). A task whose worker process is gone is dead, not in flight.
|
||||
// Liveness is Proxmox's own rule (pid + pstart vs /proc/<pid>/stat field 22), never the wall clock: the first version
|
||||
// compared start time with btime and a clock step after the boot could make a live task look dead (security review
|
||||
// 2026-10-14). These tests pin the helper, the production controller, three callers end to end; the fourth (the
|
||||
// controller supervisor) by the structural pin at the bottom.
|
||||
|
||||
const testBoot = int64(1791967481) // the C3 boot (power-cut-fixes-2026-10-14)
|
||||
// A worker: pid 0x1E6EEC = 1994476, pstart 0x1B4A0D = 1788429 (UPID hex fields).
|
||||
const (
|
||||
testPID = 1994476
|
||||
testPStart = 1788429
|
||||
)
|
||||
|
||||
func vz(vmid string, start int64) proxmox.TaskStatus {
|
||||
return proxmox.TaskStatus{UPID: "UPID:demo-hp:001E6EEC:00000001:6ACF4071:vzdump:" + vmid + ":felhom-agent@pve!agent:",
|
||||
Type: "vzdump", ID: vmid, Status: "RUNNING", StartTime: start}
|
||||
func vz(vmid string) proxmox.TaskStatus {
|
||||
return proxmox.TaskStatus{UPID: "UPID:demo-hp:001E6EEC:001B4A0D:6ACF4071:vzdump:" + vmid + ":felhom-agent@pve!agent:",
|
||||
Type: "vzdump", ID: vmid, Status: "RUNNING", StartTime: 1791967345}
|
||||
}
|
||||
|
||||
// RED-PROOF: make vzdumpInFlight return true for every matching task (drop the `start >= boot` test) → this fails.
|
||||
func TestVzdumpInFlight_PreBootTaskIsDead(t *testing.T) {
|
||||
running, dead := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", testBoot-136)}, 9201, testBoot, true)
|
||||
if running || len(dead) != 1 {
|
||||
t.Fatalf("a task from before the boot must be dead: running=%v dead=%d", running, len(dead))
|
||||
// procFixture writes <dir>/<pid>/stat with the given start ticks (comm with a space and a parenthesis, as real ones can).
|
||||
func procFixture(t *testing.T, pid int, start int) string {
|
||||
return procFixtureState(t, pid, start, "S")
|
||||
}
|
||||
|
||||
func procFixtureState(t *testing.T, pid int, start int, state string) string {
|
||||
t.Helper()
|
||||
d := t.TempDir()
|
||||
// PID 1 is always visible on a real host; procFixtureHidden leaves it out
|
||||
if err := os.MkdirAll(filepath.Join(d, "1"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVzdumpInFlight_PostBootTaskRuns(t *testing.T) {
|
||||
for _, start := range []int64{testBoot, testBoot + 600} {
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", start)}, 9201, testBoot, true); !running {
|
||||
t.Fatalf("a task started at %d (boot %d) must be in flight", start, testBoot)
|
||||
if err := os.WriteFile(filepath.Join(d, "1", "stat"), []byte("1 (systemd) S 0 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 2 0 0\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pid > 0 {
|
||||
p := filepath.Join(d, itoaT(pid))
|
||||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := []string{itoaT(pid), "(task UPID:x (vzdump))", state}
|
||||
for i := 4; i <= 21; i++ {
|
||||
f = append(f, "0")
|
||||
}
|
||||
f = append(f, itoaT(start), "0", "0")
|
||||
if err := os.WriteFile(filepath.Join(p, "stat"), []byte(strings.Join(f, " ")+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// RED-PROOF: drop the `!bootOK` early return → a pre-boot task with an unreadable boot is called dead → fails.
|
||||
func TestVzdumpInFlight_UnreadableBootFailsSafe(t *testing.T) {
|
||||
// the boot value is a real one on purpose: only bootOK=false may make this „running"
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201", testBoot-136)}, 9201, testBoot, false); !running {
|
||||
t.Fatal("an unreadable boot time must count as running (fail safe)")
|
||||
// RED-PROOF: make taskAlive return (true, true) for a missing /proc entry → the dead task runs → fails.
|
||||
func TestVzdumpInFlight_WorkerGoneIsDead(t *testing.T) {
|
||||
running, dead := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, 0, 0))
|
||||
if running || len(dead) != 1 {
|
||||
t.Fatalf("a task whose worker is gone must be dead: running=%v dead=%d", running, len(dead))
|
||||
}
|
||||
}
|
||||
|
||||
// RED-PROOF: make taskStart ignore the UPID → StartTime 0 counts as unknown → running; the first case fails.
|
||||
func TestVzdumpInFlight_StartFromUPIDThenUnknownFailsSafe(t *testing.T) {
|
||||
tk := vz("9201", 0) // no starttime in the list: the UPID's 6ACF4071 = 1791967345 (before the boot)
|
||||
if running, dead := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, testBoot, true); running || len(dead) != 1 {
|
||||
t.Fatalf("the UPID's start time must be used: running=%v", running)
|
||||
// RED-PROOF: drop the `start == u.PStart` comparison (alive whenever the pid exists) → a reused pid runs → fails.
|
||||
func TestVzdumpInFlight_ReusedPidIsDead(t *testing.T) {
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, testPID, 42)); running {
|
||||
t.Fatal("the pid exists but started at another tick: another process, the task is dead")
|
||||
}
|
||||
tk.UPID = "garbage"
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, testBoot, true); !running {
|
||||
t.Fatal("an unknown start time must count as running (fail safe)")
|
||||
}
|
||||
|
||||
// RED-PROOF: drop the `f[0] != "Z"` test → a zombie worker counts as running → fails.
|
||||
func TestVzdumpInFlight_ZombieIsDead(t *testing.T) {
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixtureState(t, testPID, testPStart, "Z")); running {
|
||||
t.Fatal("a zombie worker is not running (Proxmox's own rule)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVzdumpInFlight_LiveWorkerRuns(t *testing.T) {
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, testPID, testPStart)); !running {
|
||||
t.Fatal("the worker lives (same pid, same start tick): in flight")
|
||||
}
|
||||
}
|
||||
|
||||
// The reason for the rewrite: the wall clock plays no part. A live task whose list start time lies „before the boot"
|
||||
// (a clock step after the boot) is still running.
|
||||
func TestVzdumpInFlight_ClockStepCannotKillALiveTask(t *testing.T) {
|
||||
tk := vz("9201")
|
||||
tk.StartTime = 1 // absurdly early by the wall clock
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, procFixture(t, testPID, testPStart)); !running {
|
||||
t.Fatal("a live worker must count as running whatever its wall-clock start says")
|
||||
}
|
||||
}
|
||||
|
||||
// RED-PROOF: make an unparsable UPID or unreadable stat „known dead" → fails.
|
||||
func TestVzdumpInFlight_DoubtFailsSafe(t *testing.T) {
|
||||
bad := vz("9201")
|
||||
bad.UPID = "garbage"
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{bad}, 9201, procFixture(t, 0, 0)); !running {
|
||||
t.Fatal("an unparsable UPID must count as running (fail safe)")
|
||||
}
|
||||
d := procFixture(t, testPID, testPStart)
|
||||
if err := os.WriteFile(filepath.Join(d, itoaT(testPID), "stat"), []byte("garbage"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, d); !running {
|
||||
t.Fatal("an unparsable stat must count as running (fail safe)")
|
||||
}
|
||||
}
|
||||
|
||||
// RED-PROOF: drop the PID-1 visibility check → a hidden /proc makes the live task „gone" → fails.
|
||||
func TestVzdumpInFlight_HiddenProcFailsSafe(t *testing.T) {
|
||||
d := t.TempDir() // nothing visible: not even PID 1 (hidepid / ProtectProc)
|
||||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, d); !running {
|
||||
t.Fatal("a /proc that hides other users' processes must count as running (fail safe)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVzdumpInFlight_OtherGuestOrTypeIgnored(t *testing.T) {
|
||||
tasks := []proxmox.TaskStatus{vz("9202", testBoot+5), {Type: "vzsnapshot", ID: "9201", StartTime: testBoot + 5}}
|
||||
if running, dead := vzdumpInFlight(tasks, 9201, testBoot, true); running || len(dead) != 0 {
|
||||
tasks := []proxmox.TaskStatus{vz("9202"), {UPID: vz("9201").UPID, Type: "vzsnapshot", ID: "9201"}}
|
||||
if running, dead := vzdumpInFlight(tasks, 9201, procFixture(t, testPID, testPStart)); running || len(dead) != 0 {
|
||||
t.Fatalf("other guests' and other types' tasks are not 9201's backup: running=%v dead=%d", running, len(dead))
|
||||
}
|
||||
}
|
||||
|
||||
func inflightCtrl(api *fakeStaleLockAPI, runner *recordingRunner, logger *slog.Logger) *staleLockController {
|
||||
func inflightCtrl(api *fakeStaleLockAPI, runner *recordingRunner, logger *slog.Logger, proc string) *staleLockController {
|
||||
c := NewStaleLockController(api, runner, "felhom", logger).(*staleLockController)
|
||||
c.bootTime = func() (int64, bool) { return testBoot, true }
|
||||
c.procDir = proc
|
||||
return c
|
||||
}
|
||||
|
||||
// The production controller answers through the helper, and logs a dead task ONCE.
|
||||
// RED-PROOF: make BackupRunning ignore bootTime (the pre-R-1008 body) → returns true → fails.
|
||||
// RED-PROOF: make BackupRunning ignore the helper (any listed task runs) → fails.
|
||||
func TestStaleLockController_BackupRunningIgnoresDeadTask(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", testBoot-136)}}
|
||||
c := inflightCtrl(api, &recordingRunner{}, slog.New(slog.NewTextHandler(&buf, nil)))
|
||||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")}}
|
||||
c := inflightCtrl(api, &recordingRunner{}, slog.New(slog.NewTextHandler(&buf, nil)), procFixture(t, 0, 0))
|
||||
for i := 0; i < 3; i++ {
|
||||
if running, err := c.BackupRunning(context.Background(), 9201); err != nil || running {
|
||||
t.Fatalf("dead pre-boot task reported running=%v err=%v", running, err)
|
||||
t.Fatalf("dead task reported running=%v err=%v", running, err)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(buf.String(), "backup-in-flight: a vzdump task from before this boot"); n != 1 {
|
||||
if n := strings.Count(buf.String(), "backup-in-flight: a vzdump task is listed as active but its worker process is gone"); n != 1 {
|
||||
t.Fatalf("the dead task must be logged exactly once, got %d", n)
|
||||
}
|
||||
api.tasks = []proxmox.TaskStatus{vz("9201", testBoot+30)}
|
||||
c.procDir = procFixture(t, testPID, testPStart)
|
||||
if running, _ := c.BackupRunning(context.Background(), 9201); !running {
|
||||
t.Fatal("a post-boot task must be running")
|
||||
t.Fatal("a live worker must be running")
|
||||
}
|
||||
}
|
||||
|
||||
func fixtureFor(t *testing.T, alive bool) string {
|
||||
if alive {
|
||||
return procFixture(t, testPID, testPStart)
|
||||
}
|
||||
return procFixture(t, 0, 0)
|
||||
}
|
||||
|
||||
func poolOf(vmids ...int) proxmox.PoolInfo {
|
||||
var p proxmox.PoolInfo
|
||||
raw := `{"members":[`
|
||||
@@ -113,16 +184,16 @@ func poolOf(vmids ...int) proxmox.PoolInfo {
|
||||
// „genuinely in-flight — leaving the lock").
|
||||
func TestCaller_StartupStaleLockClearsAfterPowerCut(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
start int64
|
||||
alive bool
|
||||
cleared bool
|
||||
}{{testBoot - 136, true}, {testBoot + 30, false}} {
|
||||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", tc.start)},
|
||||
}{{false, true}, {true, false}} {
|
||||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")},
|
||||
cfg: map[int]proxmox.GuestConfig{9201: staleCfg("backup", true)}}
|
||||
runner := &recordingRunner{}
|
||||
s := &Server{staleLock: inflightCtrl(api, runner, nil), logger: slog.New(slog.NewTextHandler(discardW{}, nil))}
|
||||
s := &Server{staleLock: inflightCtrl(api, runner, nil, fixtureFor(t, tc.alive)), logger: slog.New(slog.NewTextHandler(discardW{}, nil))}
|
||||
s.recoverOneStaleLock(context.Background(), proxmox.Guest{VMID: 9201, Status: "running"})
|
||||
if runner.unlocked("9201") != tc.cleared {
|
||||
t.Fatalf("task start %d: unlocked=%v, want %v", tc.start, runner.unlocked("9201"), tc.cleared)
|
||||
t.Fatalf("worker alive=%v: unlocked=%v, want %v", tc.alive, runner.unlocked("9201"), tc.cleared)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -130,15 +201,15 @@ func TestCaller_StartupStaleLockClearsAfterPowerCut(t *testing.T) {
|
||||
// Caller 2 — the guest-power watchdog starts a stopped guest despite a dead pre-boot vzdump task.
|
||||
func TestCaller_GuestPowerStartsGuestAfterPowerCut(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
start int64
|
||||
alive bool
|
||||
started bool
|
||||
}{{testBoot - 136, true}, {testBoot + 30, false}} {
|
||||
}{{false, true}, {true, false}} {
|
||||
api := &fakeStaleLockAPI{lxc: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, pool: poolOf(9201),
|
||||
tasks: []proxmox.TaskStatus{vz("9201", tc.start)}, cfg: map[int]proxmox.GuestConfig{9201: staleCfg("", true)}}
|
||||
s := gpServer(t, inflightCtrl(api, &recordingRunner{}, nil), nil)
|
||||
tasks: []proxmox.TaskStatus{vz("9201")}, cfg: map[int]proxmox.GuestConfig{9201: staleCfg("", true)}}
|
||||
s := gpServer(t, inflightCtrl(api, &recordingRunner{}, nil, fixtureFor(t, tc.alive)), nil)
|
||||
s.GuestPowerTick(context.Background())
|
||||
if (len(api.started) == 1) != tc.started {
|
||||
t.Fatalf("task start %d: started=%v, want %v", tc.start, api.started, tc.started)
|
||||
t.Fatalf("worker alive=%v: started=%v, want %v", tc.alive, api.started, tc.started)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -146,13 +217,13 @@ func TestCaller_GuestPowerStartsGuestAfterPowerCut(t *testing.T) {
|
||||
// Caller 3 — the household power-off guard does not refuse „backup running" for a dead task.
|
||||
func TestCaller_PowerOffJobBlockerIgnoresDeadTask(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
start int64
|
||||
alive bool
|
||||
want string
|
||||
}{{testBoot - 136, ""}, {testBoot + 30, PowerOffBackup}} {
|
||||
}{{false, ""}, {true, PowerOffBackup}} {
|
||||
rig := newPowerRig(t, 0)
|
||||
rig.srv.staleLock = inflightCtrl(&fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201", tc.start)}}, &recordingRunner{}, nil)
|
||||
rig.srv.staleLock = inflightCtrl(&fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")}}, &recordingRunner{}, nil, fixtureFor(t, tc.alive))
|
||||
if code, _ := rig.srv.jobBlocker(context.Background(), 9201); code != tc.want {
|
||||
t.Fatalf("task start %d: jobBlocker=%q, want %q", tc.start, code, tc.want)
|
||||
t.Fatalf("worker alive=%v: jobBlocker=%q, want %q", tc.alive, code, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,7 +260,7 @@ func TestBackupRunning_AllCallersUseTheOneHelper(t *testing.T) {
|
||||
t.Errorf("expected the five known callers (stale-lock, guest-power, power-off, supervisor, the R-1006 cleanup), found %d", callers)
|
||||
}
|
||||
b, _ := os.ReadFile("stalelock.go")
|
||||
if !strings.Contains(string(b), "vzdumpInFlight(tasks, vmid, boot, ok)") {
|
||||
if !strings.Contains(string(b), "vzdumpInFlight(tasks, vmid, proc)") {
|
||||
t.Error("staleLockController.BackupRunning no longer answers through vzdumpInFlight")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user