267 lines
11 KiB
Go
267 lines
11 KiB
Go
package localapi
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"log/slog"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||
)
|
||
|
||
// R-1008: after a power cut Proxmox keeps the dead vzdump task in `tasks?source=active` (measured on demo-hp
|
||
// 2026-10-14: still RUNNING ~30–60 s after the boot). A task whose worker process is gone is dead, not in flight.
|
||
// Liveness is Proxmox's own rule (pid + pstart vs /proc/<pid>/stat field 22), never the wall clock: the first version
|
||
// compared start time with btime and a clock step after the boot could make a live task look dead (security review
|
||
// 2026-10-14). These tests pin the helper, the production controller, three callers end to end; the fourth (the
|
||
// controller supervisor) by the structural pin at the bottom.
|
||
|
||
// A worker: pid 0x1E6EEC = 1994476, pstart 0x1B4A0D = 1788429 (UPID hex fields).
|
||
const (
|
||
testPID = 1994476
|
||
testPStart = 1788429
|
||
)
|
||
|
||
func vz(vmid string) proxmox.TaskStatus {
|
||
return proxmox.TaskStatus{UPID: "UPID:demo-hp:001E6EEC:001B4A0D:6ACF4071:vzdump:" + vmid + ":felhom-agent@pve!agent:",
|
||
Type: "vzdump", ID: vmid, Status: "RUNNING", StartTime: 1791967345}
|
||
}
|
||
|
||
// procFixture writes <dir>/<pid>/stat with the given start ticks (comm with a space and a parenthesis, as real ones can).
|
||
func procFixture(t *testing.T, pid int, start int) string {
|
||
return procFixtureState(t, pid, start, "S")
|
||
}
|
||
|
||
func procFixtureState(t *testing.T, pid int, start int, state string) string {
|
||
t.Helper()
|
||
d := t.TempDir()
|
||
// PID 1 is always visible on a real host; procFixtureHidden leaves it out
|
||
if err := os.MkdirAll(filepath.Join(d, "1"), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(filepath.Join(d, "1", "stat"), []byte("1 (systemd) S 0 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 2 0 0\n"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if pid > 0 {
|
||
p := filepath.Join(d, itoaT(pid))
|
||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
f := []string{itoaT(pid), "(task UPID:x (vzdump))", state}
|
||
for i := 4; i <= 21; i++ {
|
||
f = append(f, "0")
|
||
}
|
||
f = append(f, itoaT(start), "0", "0")
|
||
if err := os.WriteFile(filepath.Join(p, "stat"), []byte(strings.Join(f, " ")+"\n"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
return d
|
||
}
|
||
|
||
// RED-PROOF: make taskAlive return (true, true) for a missing /proc entry → the dead task runs → fails.
|
||
func TestVzdumpInFlight_WorkerGoneIsDead(t *testing.T) {
|
||
running, dead := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, 0, 0))
|
||
if running || len(dead) != 1 {
|
||
t.Fatalf("a task whose worker is gone must be dead: running=%v dead=%d", running, len(dead))
|
||
}
|
||
}
|
||
|
||
// RED-PROOF: drop the `start == u.PStart` comparison (alive whenever the pid exists) → a reused pid runs → fails.
|
||
func TestVzdumpInFlight_ReusedPidIsDead(t *testing.T) {
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, testPID, 42)); running {
|
||
t.Fatal("the pid exists but started at another tick: another process, the task is dead")
|
||
}
|
||
}
|
||
|
||
// RED-PROOF: drop the `f[0] != "Z"` test → a zombie worker counts as running → fails.
|
||
func TestVzdumpInFlight_ZombieIsDead(t *testing.T) {
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixtureState(t, testPID, testPStart, "Z")); running {
|
||
t.Fatal("a zombie worker is not running (Proxmox's own rule)")
|
||
}
|
||
}
|
||
|
||
func TestVzdumpInFlight_LiveWorkerRuns(t *testing.T) {
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, procFixture(t, testPID, testPStart)); !running {
|
||
t.Fatal("the worker lives (same pid, same start tick): in flight")
|
||
}
|
||
}
|
||
|
||
// The reason for the rewrite: the wall clock plays no part. A live task whose list start time lies „before the boot"
|
||
// (a clock step after the boot) is still running.
|
||
func TestVzdumpInFlight_ClockStepCannotKillALiveTask(t *testing.T) {
|
||
tk := vz("9201")
|
||
tk.StartTime = 1 // absurdly early by the wall clock
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{tk}, 9201, procFixture(t, testPID, testPStart)); !running {
|
||
t.Fatal("a live worker must count as running whatever its wall-clock start says")
|
||
}
|
||
}
|
||
|
||
// RED-PROOF: make an unparsable UPID or unreadable stat „known dead" → fails.
|
||
func TestVzdumpInFlight_DoubtFailsSafe(t *testing.T) {
|
||
bad := vz("9201")
|
||
bad.UPID = "garbage"
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{bad}, 9201, procFixture(t, 0, 0)); !running {
|
||
t.Fatal("an unparsable UPID must count as running (fail safe)")
|
||
}
|
||
d := procFixture(t, testPID, testPStart)
|
||
if err := os.WriteFile(filepath.Join(d, itoaT(testPID), "stat"), []byte("garbage"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, d); !running {
|
||
t.Fatal("an unparsable stat must count as running (fail safe)")
|
||
}
|
||
}
|
||
|
||
// RED-PROOF: drop the PID-1 visibility check → a hidden /proc makes the live task „gone" → fails.
|
||
func TestVzdumpInFlight_HiddenProcFailsSafe(t *testing.T) {
|
||
d := t.TempDir() // nothing visible: not even PID 1 (hidepid / ProtectProc)
|
||
if running, _ := vzdumpInFlight([]proxmox.TaskStatus{vz("9201")}, 9201, d); !running {
|
||
t.Fatal("a /proc that hides other users' processes must count as running (fail safe)")
|
||
}
|
||
}
|
||
|
||
func TestVzdumpInFlight_OtherGuestOrTypeIgnored(t *testing.T) {
|
||
tasks := []proxmox.TaskStatus{vz("9202"), {UPID: vz("9201").UPID, Type: "vzsnapshot", ID: "9201"}}
|
||
if running, dead := vzdumpInFlight(tasks, 9201, procFixture(t, testPID, testPStart)); running || len(dead) != 0 {
|
||
t.Fatalf("other guests' and other types' tasks are not 9201's backup: running=%v dead=%d", running, len(dead))
|
||
}
|
||
}
|
||
|
||
func inflightCtrl(api *fakeStaleLockAPI, runner *recordingRunner, logger *slog.Logger, proc string) *staleLockController {
|
||
c := NewStaleLockController(api, runner, "felhom", logger).(*staleLockController)
|
||
c.procDir = proc
|
||
return c
|
||
}
|
||
|
||
// The production controller answers through the helper, and logs a dead task ONCE.
|
||
// RED-PROOF: make BackupRunning ignore the helper (any listed task runs) → fails.
|
||
func TestStaleLockController_BackupRunningIgnoresDeadTask(t *testing.T) {
|
||
var buf bytes.Buffer
|
||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")}}
|
||
c := inflightCtrl(api, &recordingRunner{}, slog.New(slog.NewTextHandler(&buf, nil)), procFixture(t, 0, 0))
|
||
for i := 0; i < 3; i++ {
|
||
if running, err := c.BackupRunning(context.Background(), 9201); err != nil || running {
|
||
t.Fatalf("dead task reported running=%v err=%v", running, err)
|
||
}
|
||
}
|
||
if n := strings.Count(buf.String(), "backup-in-flight: a vzdump task is listed as active but its worker process is gone"); n != 1 {
|
||
t.Fatalf("the dead task must be logged exactly once, got %d", n)
|
||
}
|
||
c.procDir = procFixture(t, testPID, testPStart)
|
||
if running, _ := c.BackupRunning(context.Background(), 9201); !running {
|
||
t.Fatal("a live worker must be running")
|
||
}
|
||
}
|
||
|
||
func fixtureFor(t *testing.T, alive bool) string {
|
||
if alive {
|
||
return procFixture(t, testPID, testPStart)
|
||
}
|
||
return procFixture(t, 0, 0)
|
||
}
|
||
|
||
func poolOf(vmids ...int) proxmox.PoolInfo {
|
||
var p proxmox.PoolInfo
|
||
raw := `{"members":[`
|
||
for i, v := range vmids {
|
||
if i > 0 {
|
||
raw += ","
|
||
}
|
||
raw += `{"vmid":` + itoaT(v) + `,"type":"lxc"}`
|
||
}
|
||
raw += `]}`
|
||
_ = json.Unmarshal([]byte(raw), &p)
|
||
return p
|
||
}
|
||
|
||
// Caller 1 — the startup stale-lock recovery clears the lock a power cut left (the night of 2026-10-14 it said
|
||
// „genuinely in-flight — leaving the lock").
|
||
func TestCaller_StartupStaleLockClearsAfterPowerCut(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
alive bool
|
||
cleared bool
|
||
}{{false, true}, {true, false}} {
|
||
api := &fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")},
|
||
cfg: map[int]proxmox.GuestConfig{9201: staleCfg("backup", true)}}
|
||
runner := &recordingRunner{}
|
||
s := &Server{staleLock: inflightCtrl(api, runner, nil, fixtureFor(t, tc.alive)), logger: slog.New(slog.NewTextHandler(discardW{}, nil))}
|
||
s.recoverOneStaleLock(context.Background(), proxmox.Guest{VMID: 9201, Status: "running"})
|
||
if runner.unlocked("9201") != tc.cleared {
|
||
t.Fatalf("worker alive=%v: unlocked=%v, want %v", tc.alive, runner.unlocked("9201"), tc.cleared)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Caller 2 — the guest-power watchdog starts a stopped guest despite a dead pre-boot vzdump task.
|
||
func TestCaller_GuestPowerStartsGuestAfterPowerCut(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
alive bool
|
||
started bool
|
||
}{{false, true}, {true, false}} {
|
||
api := &fakeStaleLockAPI{lxc: []proxmox.Guest{{VMID: 9201, Status: "stopped"}}, pool: poolOf(9201),
|
||
tasks: []proxmox.TaskStatus{vz("9201")}, cfg: map[int]proxmox.GuestConfig{9201: staleCfg("", true)}}
|
||
s := gpServer(t, inflightCtrl(api, &recordingRunner{}, nil, fixtureFor(t, tc.alive)), nil)
|
||
s.GuestPowerTick(context.Background())
|
||
if (len(api.started) == 1) != tc.started {
|
||
t.Fatalf("worker alive=%v: started=%v, want %v", tc.alive, api.started, tc.started)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Caller 3 — the household power-off guard does not refuse „backup running" for a dead task.
|
||
func TestCaller_PowerOffJobBlockerIgnoresDeadTask(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
alive bool
|
||
want string
|
||
}{{false, ""}, {true, PowerOffBackup}} {
|
||
rig := newPowerRig(t, 0)
|
||
rig.srv.staleLock = inflightCtrl(&fakeStaleLockAPI{tasks: []proxmox.TaskStatus{vz("9201")}}, &recordingRunner{}, nil, fixtureFor(t, tc.alive))
|
||
if code, _ := rig.srv.jobBlocker(context.Background(), 9201); code != tc.want {
|
||
t.Fatalf("worker alive=%v: jobBlocker=%q, want %q", tc.alive, code, tc.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Caller 4 (the supervisor), the R-1006 cleanup (VzdumpInFlight) and the rest — structural pin: every non-test BackupRunning call in this package goes through
|
||
// s.staleLock (whose one production implementation is staleLockController, which uses vzdumpInFlight), and no
|
||
// other production type implements BackupRunning. If someone adds a second path, this fails.
|
||
func TestBackupRunning_AllCallersUseTheOneHelper(t *testing.T) {
|
||
files, _ := filepath.Glob("*.go")
|
||
call := regexp.MustCompile(`([A-Za-z_.]+)\.BackupRunning\(`)
|
||
impl := regexp.MustCompile(`func \([a-z]+ \*?([A-Za-z]+)\) BackupRunning\(`)
|
||
callers := 0
|
||
for _, f := range files {
|
||
if strings.HasSuffix(f, "_test.go") {
|
||
continue
|
||
}
|
||
b, err := os.ReadFile(f)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, m := range call.FindAllStringSubmatch(string(b), -1) {
|
||
if m[1] != "s.staleLock" {
|
||
t.Errorf("%s: BackupRunning called on %q, not s.staleLock", f, m[1])
|
||
}
|
||
callers++
|
||
}
|
||
for _, m := range impl.FindAllStringSubmatch(string(b), -1) {
|
||
if m[1] != "staleLockController" {
|
||
t.Errorf("%s: a second production BackupRunning on %s", f, m[1])
|
||
}
|
||
}
|
||
}
|
||
if callers != 5 {
|
||
t.Errorf("expected the five known callers (stale-lock, guest-power, power-off, supervisor, the R-1006 cleanup), found %d", callers)
|
||
}
|
||
b, _ := os.ReadFile("stalelock.go")
|
||
if !strings.Contains(string(b), "vzdumpInFlight(tasks, vmid, proc)") {
|
||
t.Error("staleLockController.BackupRunning no longer answers through vzdumpInFlight")
|
||
}
|
||
}
|