package reconcile import ( "context" "encoding/json" "errors" "io" "os" "path/filepath" "strings" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a // substring of the joined command — makes that one command fail (the C2 mid-swap failure driver). type fakeRunner struct { mu sync.Mutex cmds []string failOn string } func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { cmd := name + " " + strings.Join(args, " ") f.mu.Lock() f.cmds = append(f.cmds, cmd) f.mu.Unlock() if f.failOn != "" && strings.Contains(cmd, f.failOn) { return nil, []byte("boom"), errors.New("exit status 1") } return nil, nil, nil } func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) { return f.Run(ctx, name, args...) } // newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an // isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it). func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) { t.Helper() jp := filepath.Join(t.TempDir(), "journal.log") j, err := OpenJournal(jp) if err != nil { t.Fatalf("OpenJournal: %v", err) } t.Cleanup(func() { j.Close() }) q := NewQueue() t.Cleanup(q.Close) fr := &fakeRunner{} sd := t.TempDir() e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd}) return e, fr, sd, q } // setParamsFor returns the params of the (last) SetConfig call against vmid, or nil. func setParamsFor(api *fakeAPI, vmid int) map[string]string { var out map[string]string for _, s := range api.sets { if s.vmid == vmid { out = s.params } } return out } func TestRunBringUp_ProvisionHappyPath(t *testing.T) { const vmid = 8000 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} // empty lxc → vmid free; running default e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8000", Cores: 2, MemoryMB: 2048, RootfsGrowGB: 2, Mounts: []GuestMount{{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/data"}}, }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } if res.Verified != "boot+running" { t.Errorf("verified = %q", res.Verified) } // restore issued to the target vmid. if len(api.restores) != 1 || api.restores[0].VMID != vmid || api.restores[0].Archive != "local:backup/golden.tar.zst" { t.Fatalf("restore not issued correctly: %+v", api.restores) } // identity: a fresh MAC (net0 set WITHOUT hwaddr) + hostname, coalesced with sizing+mount. p := setParamsFor(api, vmid) if p == nil { t.Fatal("expected a coalesced config PUT") } if net0, ok := p["net0"]; !ok || strings.Contains(net0, "hwaddr=") { t.Errorf("provision must reset MAC: net0 must be set WITHOUT hwaddr, got %q", net0) } if p["hostname"] != "felhom-prov-8000" { t.Errorf("hostname not set: %q", p["hostname"]) } if p["cores"] != "2" || p["memory"] != "2048" { t.Errorf("sizing not coalesced: cores=%q memory=%q", p["cores"], p["memory"]) } if p["mp0"] != "local-lvm:1,mp=/mnt/data" { t.Errorf("mount not attached: mp0=%q", p["mp0"]) } // rootfs grow is a SEPARATE call (F4). if len(api.resizes) != 1 || api.resizes[0].vmid != vmid || api.resizes[0].size != "+2G" { t.Errorf("rootfs grow not issued separately: %+v", api.resizes) } // started link-up. if len(api.starts) != 1 || api.starts[0] != vmid { t.Errorf("guest not started: %+v", api.starts) } // THE key difference from the restore-test: the guest is KEPT (no teardown). if len(api.destroys) != 0 { t.Fatalf("provision success must NOT destroy the guest: %+v", api.destroys) } } // A data-bearing additive mount must carry backup=1 (so its DBs stay in PBS — storage-split B3); // a non-backup mount must NOT. Pure-function check on buildBringUpConfig. func TestBuildBringUpConfig_BackupFlagOnDataMount(t *testing.T) { params := buildBringUpConfig(BringUpSpec{ Mode: ModeProvision, Mounts: []GuestMount{ {Storage: "local-lvm", SizeGB: 2, MountPoint: "/mnt/data", Backup: true}, {Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/scratch"}, // no backup }, }, scratchCfg()) if params["mp0"] != "local-lvm:2,mp=/mnt/data,backup=1" { t.Errorf("data mount must carry backup=1: mp0=%q", params["mp0"]) } if params["mp1"] != "local-lvm:1,mp=/mnt/scratch" { t.Errorf("non-backup mount must NOT carry backup=1: mp1=%q", params["mp1"]) } } // The operator-opt-in CPU/RAM cap is emitted into the SAME pre-start config PUT (so the guest never // boots uncapped), and — critically — is OMITTED when unset (0), so an uncapped provision keeps the // golden's baked sizes instead of being shrunk to 0. Pure-function check on buildBringUpConfig. func TestBuildBringUpConfig_ResourceCaps(t *testing.T) { // caps set → both keys present, MiB integer for memory capped := buildBringUpConfig(BringUpSpec{ Mode: ModeProvision, Cores: 2, MemoryMB: 4096, }, scratchCfg()) if capped["cores"] != "2" { t.Errorf("cores cap must be emitted: cores=%q", capped["cores"]) } if capped["memory"] != "4096" { t.Errorf("memory cap must be emitted (MiB): memory=%q", capped["memory"]) } // caps unset (0) → NEITHER key present (omit-when-zero; else the guest would shrink to 0 cores) uncapped := buildBringUpConfig(BringUpSpec{ Mode: ModeProvision, Cores: 0, MemoryMB: 0, }, scratchCfg()) if _, ok := uncapped["cores"]; ok { t.Errorf("cores must be ABSENT when unset (golden default), got %q", uncapped["cores"]) } if _, ok := uncapped["memory"]; ok { t.Errorf("memory must be ABSENT when unset (golden default), got %q", uncapped["memory"]) } } // R-50: with the island configured, bring-up attaches a static net1 on the island bridge; with it // unset (or half-set), NO net1 is emitted — byte-for-byte the pre-R-50 config on non-island hosts. // Pure-function check on buildBringUpConfig (the derivation that makes fresh installs F1-immune). func TestBuildBringUpConfig_IslandNIC(t *testing.T) { // island set → net1 present, exact shape, no hwaddr (PVE mints a fresh per-guest MAC) island := buildBringUpConfig(BringUpSpec{ Mode: ModeProvision, IslandBridge: "vmbr9", IslandGuestAddr: "169.254.253.2/30", }, scratchCfg()) if got, want := island["net1"], "name=eth1,bridge=vmbr9,ip=169.254.253.2/30"; got != want { t.Errorf("island net1 mismatch:\n got %q\nwant %q", got, want) } // DR mode too — a restored customer guest must also reach the island-bound agent on the host. dr := buildBringUpConfig(BringUpSpec{ Mode: ModeDRGuestLoss, KeepMAC: true, IslandBridge: "vmbr9", IslandGuestAddr: "169.254.253.2/30", }, scratchCfg()) if _, ok := dr["net1"]; !ok { t.Errorf("DR bring-up must also attach the island net1, got none") } // island unset → NO net1 key (non-island hosts unchanged; the pre-R-50 default) none := buildBringUpConfig(BringUpSpec{Mode: ModeProvision}, scratchCfg()) if v, ok := none["net1"]; ok { t.Errorf("net1 must be ABSENT when the island is not configured, got %q", v) } // half-configured (bridge only) → still no net1 (all-or-nothing; config.Validate rejects the config too) half := buildBringUpConfig(BringUpSpec{Mode: ModeProvision, IslandBridge: "vmbr9"}, scratchCfg()) if v, ok := half["net1"]; ok { t.Errorf("net1 must be ABSENT when only the bridge is set, got %q", v) } } // Both restore sites allocate the guest INTO the felhom pool (SPIKE 3b): the provision bring-up // threads spec.Pool, and the restore-test hardcodes DefaultPool — else a pool-scoped token 403s on // the created guest's config/start/destroy. Asserts via the fakeAPI's captured RestoreLXCOptions. func TestRestoreSitesUsePool(t *testing.T) { // provision bring-up: spec.Pool must flow to the restore const vmid = 8060 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/g.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Pool: DefaultPool, }) if res.Err != nil || !res.Pass { t.Fatalf("bring-up must pass: %+v", res) } if len(api.restores) != 1 || api.restores[0].Pool != DefaultPool { t.Fatalf("provision restore must set Pool=%q, got %+v", DefaultPool, api.restores) } // restore-test: the scratch restore hardcodes DefaultPool api2 := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}} e2, _, q2 := newEngine(t, api2, EmptyProvider{}) defer q2.Close() rt := e2.RunRestoreTest(context.Background(), RestoreTestSpec{ Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local", }) if rt.Err != nil || !rt.Pass { t.Fatalf("restore-test must pass: %+v", rt) } if len(api2.restores) != 1 || api2.restores[0].Pool != DefaultPool { t.Fatalf("restore-test restore must set Pool=%q, got %+v", DefaultPool, api2.restores) } } // The golden-carried Docker-data volume is grown via a SEPARATE resize on its mpN slot (B4), // alongside (but distinct from) the rootfs grow. func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) { const vmid = 8050 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8050", RootfsGrowGB: 8, DataVolGrowGB: 240, // grows mp0 (DefaultDataVolMount) }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } // TWO resizes: rootfs +8G and the Docker-data volume mp0 +240G. if len(api.resizes) != 2 { t.Fatalf("expected rootfs + data-volume resizes, got %+v", api.resizes) } var sawRootfs, sawData bool for _, r := range api.resizes { if r.disk == "rootfs" && r.size == "+8G" { sawRootfs = true } if r.disk == "mp0" && r.size == "+240G" { sawData = true } } if !sawRootfs || !sawData { t.Errorf("want rootfs +8G AND mp0 +240G, got %+v", api.resizes) } } // R-165 RETARGETED THIS TEST, and the retarget IS the contract change. There is no longer a second // volume, so `SysDataGrowGB` no longer drives its own resize on mp1 — its GiB are FOLDED INTO the // single volume's grow. // // FOLDED, NOT DROPPED, and that is the whole point. `felhom-host-install.sh` computes and passes // `-sysdata-grow` from the thin pool's free space, and an installer and an agent do not upgrade in // the same instant. Dropping the value would silently shrink every appliance built by an older // installer by the user-data share — 42 of 250 GiB on the standard branch — which is precisely the // "a knob that silently does nothing" outcome this work was told to avoid. func TestRunBringUp_StorageSplit_SysDataGrowIsFoldedIn(t *testing.T) { const vmid = 8051 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8051", DataVolGrowGB: 240, SysDataGrowGB: 42, // ONE volume: 240 + 42 = 282 }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } // EXACTLY ONE resize. A second one would mean an mp1 the golden no longer ships. if len(api.resizes) != 1 { t.Fatalf("expected exactly ONE data-volume resize (there is no mp1 since R-165), got %+v", api.resizes) } r := api.resizes[0] if r.disk != "mp0" { t.Fatalf("resized %q, want mp0 — the single data volume", r.disk) } if r.size != "+282G" { t.Fatalf("resized %s, want +282G (240 data + 42 folded sys-data). Anything less means the "+ "retired knob's GiB were DROPPED, silently shrinking every appliance an older "+ "felhom-host-install.sh provisions", r.size) } for _, rr := range api.resizes { if rr.disk == "mp1" { t.Fatalf("an mp1 resize was issued (%+v) — the golden ships no second volume, so this "+ "would fail on a real box", rr) } } } // SysDataGrowGB=0 must issue NO mp1 resize (grow is an orthogonal knob; separateness comes from the // golden, not the grow). func TestRunBringUp_StorageSplit_SysDataGrowZeroNoResize(t *testing.T) { const vmid = 8052 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8052", SysDataGrowGB: 0, // no sys-data grow }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } for _, r := range api.resizes { if r.disk == "mp1" { t.Errorf("SysDataGrowGB=0 must NOT resize mp1, got %+v", api.resizes) } } } func TestRunBringUp_CompensatingRollback(t *testing.T) { const vmid = 8000 lockBackoffFast(t) cases := []struct { name string setup func(*fakeAPI) }{ {"restore-task failure after launch", func(a *fakeAPI) { // Scenario B (no-regression): the restore POST was ACCEPTED (UPID) but the task then // fails → we created/started it → the compensating destroy MUST still fire. a.restoreUPID = "UPID:demo:restore:8000:" a.waitFunc = func(upid string) (proxmox.TaskStatus, error) { if upid == "UPID:demo:restore:8000:" { return proxmox.TaskStatus{}, errors.New("restore task failed") } return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil } }}, {"config real error", func(a *fakeAPI) { a.setFunc = func(int, map[string]string) (string, error) { return "", &proxmox.APIError{StatusCode: 500, Body: "some non-lock internal error"} } }}, {"start-task real error", func(a *fakeAPI) { a.startUPID = "UPID:demo:start:8000:" a.waitFunc = func(upid string) (proxmox.TaskStatus, error) { if upid == "UPID:demo:start:8000:" { return proxmox.TaskStatus{}, errors.New("start task failed") } return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil } }}, {"waitRunning timeout", func(a *fakeAPI) { a.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}} }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} tc.setup(api) e, j, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h", BootTimeout: 40 * time.Millisecond, }) if res.Pass || res.Err == nil { t.Fatalf("must fail, got %+v", res) } // The adversarial point: the just-created guest was ACTUALLY destroyed. if len(api.destroys) != 1 || api.destroys[0] != vmid { t.Fatalf("compensating rollback must destroy the guest: destroys=%+v", api.destroys) } // And the owning entry is terminal (rollback complete) — not left in-flight. if len(j.InFlight()) != 0 { t.Errorf("owning entry must be terminal after rollback: %+v", j.InFlight()) } }) } } // TestRunBringUp_NoLaunchNoDestroy is the F1a red-proof (campaign pool-effects): a restore that // fails SYNCHRONOUSLY (no UPID — nothing created) must NOT arm the compensating destroy. The // headline case is PVE refusing a vmid that already holds a guest the pool-blind duplicate guard // couldn't see — the old defer would have destroyed that innocent pre-existing guest (only the // pool ACL's 403 saved the non-pool subset; an in-pool one would have been destroyed). // Red-proof: revert the `launched` gate in runBringUp and this test fails with destroys=[8000]. func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) { const vmid = 8000 cases := []struct { name string err error }{ {"PVE refuses pre-existing vmid", &proxmox.APIError{ StatusCode: 500, Method: "POST", Path: "/nodes/x/lxc", Body: `{"message":"CT 8000 already exists on node 'x'\n","data":null}`, }}, {"plain synchronous restore error", errors.New("restore boom")}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, restoreErr: tc.err} e, j, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h", }) if res.Pass || res.Err == nil { t.Fatalf("must fail, got %+v", res) } // THE point: no destroy is even attempted — the txn created nothing. if len(api.destroys) != 0 { t.Fatalf("no-launch failure must NOT destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys) } // And the owning entry is closed terminal in-process (not left for Recover). if len(j.InFlight()) != 0 { t.Errorf("owning entry must be terminal after a no-launch failure: %+v", j.InFlight()) } }) } } func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) { const vmid = 8001 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "ignored-in-dr", KeepMAC: true, Cores: 4, // a benign config PUT happens, so we can inspect it has NO identity resets }) if res.Err != nil || !res.Pass { t.Fatalf("dr bring-up must pass, got %+v", res) } p := setParamsFor(api, vmid) // DR continuity: MAC kept (no net0 reset) and hostname NOT force-reset. if p != nil { if _, ok := p["net0"]; ok { t.Errorf("dr+KeepMAC must NOT reset net0 (keep the archived MAC): %+v", p) } if _, ok := p["hostname"]; ok { t.Errorf("dr must NOT force-reset hostname (continuity): %+v", p) } if p["cores"] != "4" { t.Errorf("benign sizing should still apply: cores=%q", p["cores"]) } } // AssignedMAC reflects the kept archived MAC (from scratchCfg's net0). if res.AssignedMAC != "AA:BB:CC:DD:EE:FF" { t.Errorf("dr should keep the archived MAC, got %q", res.AssignedMAC) } if len(api.destroys) != 0 { t.Errorf("dr success must not destroy: %+v", api.destroys) } // The agent performs NO guest-internal host-key op — there is no such API call; host keys // are preserved (DR) or regenerated by the baked golden unit (provision). } func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) { const vmid = 8002 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: false, // a source guest may still be live → reset MAC even in DR }) if res.Err != nil || !res.Pass { t.Fatalf("got %+v", res) } p := setParamsFor(api, vmid) if p == nil || strings.Contains(p["net0"], "hwaddr=") { t.Errorf("dr with KeepMAC=false must reset MAC (net0 without hwaddr): %+v", p) } } func TestRunBringUp_LivenessIsTheVerdict(t *testing.T) { const vmid = 8000 const startUPID = "UPID:demo:start:8000:" mkAPI := func() *fakeAPI { return &fakeAPI{ cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, startUPID: startUPID, waitFunc: func(upid string) (proxmox.TaskStatus, error) { if upid == startUPID { return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil } return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil }, logTailFunc: func(string) ([]string, error) { return []string{"WARN: Systemd 257 detected. You may need to enable nesting."}, nil }, } } // start exits WARNINGS + guest reaches running → PASS, warnings surfaced + recognized. t.Run("warnings + running -> pass", func(t *testing.T) { api := mkAPI() e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if !res.Pass || res.Err != nil { t.Fatalf("warnings+running must pass: %+v", res) } if len(res.StartWarnings) != 1 || !res.WarningsRecognized { t.Errorf("warnings must be surfaced+recognized: %+v recognized=%v", res.StartWarnings, res.WarningsRecognized) } if len(api.destroys) != 0 { t.Errorf("a passed bring-up must not destroy: %+v", api.destroys) } }) // same warnings but guest NEVER reaches running → FAIL (verdict is liveness), guest destroyed. t.Run("warnings + not-running -> fail", func(t *testing.T) { api := mkAPI() api.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", BootTimeout: 40 * time.Millisecond, }) if res.Pass || res.Err == nil { t.Fatalf("not-running must fail regardless of warnings: %+v", res) } if len(api.destroys) != 1 { t.Errorf("a failed bring-up must roll back (destroy): %+v", api.destroys) } }) } func TestRunBringUp_F4_ConfigLockRetry(t *testing.T) { const vmid = 8000 lockBackoffFast(t) t.Run("transient lock-500 then 200 -> retries and proceeds", func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} var calls int api.setFunc = func(int, map[string]string) (string, error) { calls++ if calls == 1 { return "", &proxmox.APIError{StatusCode: 500, Body: "can't lock file '/run/lock/lxc/pve-config-8000.lock' - got timeout"} } return "", nil } e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Err != nil || !res.Pass { t.Fatalf("lock-500 then 200 must succeed: %+v", res) } if calls < 2 { t.Errorf("expected a retry on the transient lock-500, calls=%d", calls) } }) t.Run("non-lock 500 -> fails without retry", func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} var calls int api.setFunc = func(int, map[string]string) (string, error) { calls++ return "", &proxmox.APIError{StatusCode: 500, Body: "internal error: disk full"} } e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Pass || res.Err == nil { t.Fatalf("a non-lock 500 must fail: %+v", res) } if calls != 1 { t.Errorf("a real error must NOT be retried, calls=%d", calls) } }) } func TestRunBringUp_JournalsOwningEntryBeforeRestore(t *testing.T) { const vmid = 8000 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, j, q := newEngine(t, api, EmptyProvider{}) defer q.Close() // At the moment RestoreLXC is called, the owning Rollback entry must already be in-flight — // so a crash here is recoverable (Recover reaps the half-built guest). var ownedAtRestore bool api.restoreHook = func() { for _, en := range j.InFlight() { if en.VMID == vmid && en.Rollback { ownedAtRestore = true } } } res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Err != nil { t.Fatalf("setup: %+v", res) } if !ownedAtRestore { t.Fatal("the owning Rollback entry MUST be journaled before the restore (crash-safety)") } } func TestRunBringUp_RejectsReservedAndExistingVMID(t *testing.T) { e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{}) defer q.Close() for _, id := range []int{9999, 990000, 990005} { res := e.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: id, RestoreStorage: "s"}) if res.Err == nil { t.Errorf("VMID %d is reserved and must be refused", id) } } // existing VMID → refuse (restore-over-existing is a signed op, not this benign path). api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 8000}}} e2, _, q2 := newEngine(t, api, EmptyProvider{}) defer q2.Close() res := e2.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: 8000, RestoreStorage: "s"}) if res.Err == nil { t.Error("bring-up over an existing guest must be refused") } if len(api.restores) != 0 || len(api.destroys) != 0 { t.Error("a refused bring-up must not restore or destroy anything") } } func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) { const vmid = 8000 // The guest still exists at startup (agent crashed mid-bring-up AFTER the restore launched — // the entry carries the UPID = the launch proof) → Recover destroys it. api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "running"}}} e, j, _ := newEngine(t, api, EmptyProvider{}) if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()}); err != nil { t.Fatal(err) } res := e.Recover(context.Background()) if res.BringUpRolledBack != 1 { t.Fatalf("half-built bring-up must be rolled back, got %+v", res) } if len(api.destroys) != 1 || api.destroys[0] != vmid { t.Fatalf("DestroyLXC not called for the half-built guest: %+v", api.destroys) } if len(j.InFlight()) != 0 { t.Errorf("resolved rollback entry must not be in-flight: %+v", j.InFlight()) } } func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) { const vmid = 8000 // Crash after a LAUNCHED restore whose guest is already gone → idempotent clean, no destroy. api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001}}} // 8000 absent e, j, _ := newEngine(t, api, EmptyProvider{}) j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()}) res := e.Recover(context.Background()) if res.BringUpClean != 1 || len(api.destroys) != 0 { t.Fatalf("already-gone bring-up must be clean with no destroy: res=%+v destroys=%+v", res, api.destroys) } if len(j.InFlight()) != 0 { t.Errorf("entry must be resolved: %+v", j.InFlight()) } } // TestRecover_BringUpNoUPIDAbandoned is the F1c red-proof (campaign pool-effects, Scenario C): a // crash left a Rollback entry with NO UPID (the restore was never confirmed) while a guest sits // at that vmid — e.g. a PRE-EXISTING guest the pool-blind list-existence check would have called // "ours". Recover must ABANDON (fail-safe), never destroy: no journaled UPID ⇒ this transaction // created nothing. Red-proof: route the entry to recoverBringUp (pre-fix order) and this fails // with destroys=[8000]. func TestRecover_BringUpNoUPIDAbandoned(t *testing.T) { const vmid = 8000 api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "stopped"}}} // a guest IS at the vmid e, j, _ := newEngine(t, api, EmptyProvider{}) j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()}) res := e.Recover(context.Background()) if len(api.destroys) != 0 { t.Fatalf("a no-UPID Rollback entry must NEVER destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys) } if res.RolledBack != 1 || res.BringUpRolledBack != 0 { t.Fatalf("entry must be abandoned via the no-UPID fail-safe path, got %+v", res) } if len(j.InFlight()) != 0 { t.Errorf("abandoned entry must be terminal: %+v", j.InFlight()) } } // lockBackoffFast shrinks the F4 retry backoff for tests and restores it after. func lockBackoffFast(t *testing.T) { t.Helper() prev := configLockBackoff configLockBackoff = time.Millisecond t.Cleanup(func() { configLockBackoff = prev }) } // --- Part B (campaign-2 R2): pool re-assertion after bring-up ------------------------------------ // poolAddsFor returns the recorded PoolAddVMID calls for a vmid. func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall { var out []poolAddCall for _, c := range api.poolAdds { if c.vmid == vmid { out = append(out, c) } } return out } // A bring-up with a Pool set MUST re-assert pool membership after the restore (so a // restore-over-existing that dropped membership is healed for the NEXT restore-test/DR). // COMPANION RED-PROOF: on the pre-fix code (no PoolAddVMID call) poolAddsFor is empty → this FAILS. func TestRunBringUp_ReassertsPoolMembership(t *testing.T) { const vmid = 8100 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Pool: "felhom", }) if res.Err != nil || !res.Pass { t.Fatalf("bring-up must pass, got %+v", res) } adds := poolAddsFor(api, vmid) if len(adds) != 1 || adds[0].pool != "felhom" { t.Fatalf("pool membership not re-asserted: poolAdds=%+v", api.poolAdds) } } // No Pool → no PoolAddVMID call (a broad-token restore needs no pool). func TestRunBringUp_NoPool_NoPoolAdd(t *testing.T) { const vmid = 8101 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h", }) if res.Err != nil || !res.Pass { t.Fatalf("bring-up must pass, got %+v", res) } if len(api.poolAdds) != 0 { t.Fatalf("no Pool set → PoolAddVMID must not be called: %+v", api.poolAdds) } } // A pool-add FAILURE must NOT flip a healthy, running guest's verdict — it surfaces as a LOUD warning // (membership matters for the NEXT op, not this guest's boot). Liveness wins. func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) { const vmid = 8102 api := &fakeAPI{ cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"), } e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Pool: "felhom", }) if res.Err != nil || !res.Pass { t.Fatalf("a pool-add hiccup must NOT fail a healthy guest, got %+v", res) } // the guest is KEPT (not torn down) despite the pool-add failure. if len(api.destroys) != 0 { t.Fatalf("pool-add failure must not trigger teardown: %+v", api.destroys) } // the failure is surfaced as a warning. found := false for _, w := range res.StartWarnings { if strings.Contains(w, "pool re-assert failed") { found = true } } if !found { t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings) } } // ── GL-5: DR structural bind overrides + 4d swap ───────────────────────────────────────────────── // GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural // binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via // the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes. // COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts // (run→fail→revert, recorded in the REPORT). func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) { const vmid = 8200 cfg := scratchCfg() // after the swap PVE parks the two displaced throwaway volumes as unusedN cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`) cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`) api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} e, fr, sd, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: true, }) if res.Err != nil || !res.Pass { t.Fatalf("dr bring-up must pass, got %+v", res) } if len(api.restores) != 1 { t.Fatalf("expected one restore, got %+v", api.restores) } ov := api.restores[0].MountOverrides // rootfs rides along explicitly (PVE refuses mpN params without it — hit live in the GL-5 // validation), sized from the ARCHIVE's embedded config (the fake serves size=8G). want := map[string]string{ "rootfs": "local-lvm:8", "mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0", "mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0", } if len(ov) != 3 || ov["rootfs"] != want["rootfs"] || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] { t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want) } if len(api.extracts) != 1 || api.extracts[0] != "local:backup/customer.tar.zst" { t.Fatalf("the rootfs size must come from the archive's extracted config: %+v", api.extracts) } // 4d: mp9 host dir created under the engine state dir … bootDir := structuralBootHostDir(sd, vmid) if st, err := os.Stat(bootDir); err != nil || !st.IsDir() { t.Fatalf("mp9 bootstrap host dir not created: %v", err) } // … and the REAL binds set via the root runner, one slot per call, exact backhalf values. wantCmds := []string{ "mkdir -p /mnt/felhom-drives", "pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives", "pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1", } if len(fr.cmds) != len(wantCmds) { t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds) } for i := range wantCmds { if fr.cmds[i] != wantCmds[i] { t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i]) } } // displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue. del := "" for _, s := range api.sets { if s.vmid == vmid && s.params["delete"] != "" { del = s.params["delete"] } } if del != "unused0,unused1" { t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets) } } // GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never // touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the // override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT). func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) { const vmid = 8201 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h", }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } if api.restores[0].MountOverrides != nil { t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides) } if len(fr.cmds) != 0 { t.Fatalf("provision must not touch the host runner, got %v", fr.cmds) } } // GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state // named and compensating-rolls-back the guest — never a silent half-wired success. func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) { const vmid = 8202 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, fr, _, q := newDREngine(t, api) defer q.Close() fr.failOn = "-mp9" res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: true, }) if res.Err == nil || res.Pass { t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res) } if !strings.Contains(res.Err.Error(), "structural-bind swap") || !strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") { t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err) } if len(api.destroys) != 1 || api.destroys[0] != vmid { t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys) } if len(api.starts) != 0 { t.Fatalf("a half-wired guest must never be started, got %+v", api.starts) } } // GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived, // so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the // end state is identical, with no unusedN residue (here only ONE displaced volume shows up). func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) { const vmid = 8203 cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`) api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} e, fr, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: true, }) if res.Err != nil || !res.Pass { t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res) } ov := api.restores[0].MountOverrides if len(ov) != 3 || ov["mp8"] == "" || ov["mp9"] == "" || ov["rootfs"] == "" { t.Fatalf("overrides are constants (+explicit rootfs) — both mpN must be named regardless of the archive: %+v", ov) } if len(fr.cmds) != 3 { t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds) } del := "" for _, s := range api.sets { if s.vmid == vmid && s.params["delete"] != "" { del = s.params["delete"] } } if del != "unused0" { t.Fatalf("the one displaced volume must be deleted: delete=%q", del) } } // GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because // the structural-bind swap is a root pct op the API token cannot perform. func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) { const vmid = 8204 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", }) if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") { t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res) } if len(api.restores) != 0 { t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores) } } // GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired // guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently). func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) { const vmid = 8205 cfg := scratchCfg() cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`) api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} api.setFunc = func(_ int, params map[string]string) (string, error) { if params["delete"] != "" { return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed") } return "", nil } e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: true, }) if res.Err != nil || !res.Pass { t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res) } found := false for _, w := range res.StartWarnings { if strings.Contains(w, "displaced volumes not deleted") { found = true } } if !found { t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings) } } // GL-5: drRestoreOverrides builds the COMPLETE explicit-params restore set from the archive's // extracted config — the live validation proved PVE drops any mountpoint NOT named in the params, // so mp0/mp1 pass-through is what keeps a DR guest's data volumes. Snapshot sections never shadow; // unknown binds and unparseable sizes refuse. func TestDRRestoreOverrides(t *testing.T) { raw := `hostname: demo rootfs: local-lvm:vm-9201-disk-0,size=32G mp0: local-lvm:vm-9201-disk-1,mp=/var/lib/docker,backup=1,size=200G mp1: local-lvm:vm-9201-disk-2,mp=/mnt/sys_drive,backup=1,size=50G mp8: /mnt/felhom-drives,mp=/mnt/felhom-drives mp9: /var/lib/felhom-agent/guests/9201/bootstrap,mp=/etc/felhom-bootstrap,ro=1 [snap1] rootfs: local-lvm:vm-9201-disk-9,size=99G ` ov, err := drRestoreOverrides(raw, "local-lvm") if err != nil { t.Fatalf("drRestoreOverrides: %v", err) } want := map[string]string{ "rootfs": "local-lvm:32", "mp0": "local-lvm:200,mp=/var/lib/docker,backup=1", "mp1": "local-lvm:50,mp=/mnt/sys_drive,backup=1", "mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0", "mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0", } if len(ov) != len(want) { t.Fatalf("overrides = %+v, want %+v", ov, want) } for k, v := range want { if ov[k] != v { t.Errorf("override[%s] = %q, want %q", k, ov[k], v) } } // an UNKNOWN bind mpN = unknown topology → refuse (never restore a guest missing a mount) if _, err := drRestoreOverrides("rootfs: l:d,size=8G\nmp3: /srv/other,mp=/data\n", "local-lvm"); err == nil || !strings.Contains(err.Error(), "unknown bind mountpoint") { t.Errorf("unknown bind must refuse, got %v", err) } // no parseable rootfs → refuse if _, err := drRestoreOverrides("hostname: x\n", "local-lvm"); err == nil || !strings.Contains(err.Error(), "rootfs size") { t.Errorf("missing rootfs must refuse, got %v", err) } // a storage mpN without a size → refuse (cannot pass it through) if _, err := drRestoreOverrides("rootfs: l:d,size=8G\nmp0: l:d1,mp=/x\n", "local-lvm"); err == nil || !strings.Contains(err.Error(), "no parseable size") { t.Errorf("sizeless mpN must refuse, got %v", err) } } // GL-5: an unreadable archive config fails the DR bring-up BEFORE any restore (no rootfs size = // the restore would fail anyway; refuse cleanly, nothing to roll back). func TestRunBringUp_DRExtractConfigFailureRefuses(t *testing.T) { const vmid = 8206 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} api.extractErr = errors.New("proxmox: GET extractconfig -> HTTP 500: volume not found") e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/gone.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", }) if res.Err == nil || !strings.Contains(res.Err.Error(), "extract archive config") { t.Fatalf("extract failure must refuse naming the step, got %+v", res) } if len(api.restores) != 0 { t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores) } if len(api.destroys) != 0 { t.Fatalf("nothing was created — nothing to roll back, got %+v", api.destroys) } }