package reconcile import ( "context" "errors" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // setParamsFor returns the params of the (last) SetConfig call against vmid, or nil. func setParamsFor(api *fakeAPI, vmid int) map[string]string { var out map[string]string for _, s := range api.sets { if s.vmid == vmid { out = s.params } } return out } func TestRunBringUp_ProvisionHappyPath(t *testing.T) { const vmid = 8000 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} // empty lxc → vmid free; running default e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8000", Cores: 2, MemoryMB: 2048, RootfsGrowGB: 2, Mounts: []GuestMount{{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/data"}}, }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } if res.Verified != "boot+running" { t.Errorf("verified = %q", res.Verified) } // restore issued to the target vmid. if len(api.restores) != 1 || api.restores[0].VMID != vmid || api.restores[0].Archive != "local:backup/golden.tar.zst" { t.Fatalf("restore not issued correctly: %+v", api.restores) } // identity: a fresh MAC (net0 set WITHOUT hwaddr) + hostname, coalesced with sizing+mount. p := setParamsFor(api, vmid) if p == nil { t.Fatal("expected a coalesced config PUT") } if net0, ok := p["net0"]; !ok || strings.Contains(net0, "hwaddr=") { t.Errorf("provision must reset MAC: net0 must be set WITHOUT hwaddr, got %q", net0) } if p["hostname"] != "felhom-prov-8000" { t.Errorf("hostname not set: %q", p["hostname"]) } if p["cores"] != "2" || p["memory"] != "2048" { t.Errorf("sizing not coalesced: cores=%q memory=%q", p["cores"], p["memory"]) } if p["mp0"] != "local-lvm:1,mp=/mnt/data" { t.Errorf("mount not attached: mp0=%q", p["mp0"]) } // rootfs grow is a SEPARATE call (F4). if len(api.resizes) != 1 || api.resizes[0].vmid != vmid || api.resizes[0].size != "+2G" { t.Errorf("rootfs grow not issued separately: %+v", api.resizes) } // started link-up. if len(api.starts) != 1 || api.starts[0] != vmid { t.Errorf("guest not started: %+v", api.starts) } // THE key difference from the restore-test: the guest is KEPT (no teardown). if len(api.destroys) != 0 { t.Fatalf("provision success must NOT destroy the guest: %+v", api.destroys) } } // A data-bearing additive mount must carry backup=1 (so its DBs stay in PBS — storage-split B3); // a non-backup mount must NOT. Pure-function check on buildBringUpConfig. func TestBuildBringUpConfig_BackupFlagOnDataMount(t *testing.T) { params := buildBringUpConfig(BringUpSpec{ Mode: ModeProvision, Mounts: []GuestMount{ {Storage: "local-lvm", SizeGB: 2, MountPoint: "/mnt/data", Backup: true}, {Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/scratch"}, // no backup }, }, scratchCfg()) if params["mp0"] != "local-lvm:2,mp=/mnt/data,backup=1" { t.Errorf("data mount must carry backup=1: mp0=%q", params["mp0"]) } if params["mp1"] != "local-lvm:1,mp=/mnt/scratch" { t.Errorf("non-backup mount must NOT carry backup=1: mp1=%q", params["mp1"]) } } // The golden-carried Docker-data volume is grown via a SEPARATE resize on its mpN slot (B4), // alongside (but distinct from) the rootfs grow. func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) { const vmid = 8050 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8050", RootfsGrowGB: 8, DataVolGrowGB: 240, // grows mp0 (DefaultDataVolMount) }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } // TWO resizes: rootfs +8G and the Docker-data volume mp0 +240G. if len(api.resizes) != 2 { t.Fatalf("expected rootfs + data-volume resizes, got %+v", api.resizes) } var sawRootfs, sawData bool for _, r := range api.resizes { if r.disk == "rootfs" && r.size == "+8G" { sawRootfs = true } if r.disk == "mp0" && r.size == "+240G" { sawData = true } } if !sawRootfs || !sawData { t.Errorf("want rootfs +8G AND mp0 +240G, got %+v", api.resizes) } } func TestRunBringUp_CompensatingRollback(t *testing.T) { const vmid = 8000 lockBackoffFast(t) cases := []struct { name string setup func(*fakeAPI) }{ {"restore error", func(a *fakeAPI) { a.restoreErr = errors.New("restore boom") }}, {"config real error", func(a *fakeAPI) { a.setFunc = func(int, map[string]string) (string, error) { return "", &proxmox.APIError{StatusCode: 500, Body: "some non-lock internal error"} } }}, {"start-task real error", func(a *fakeAPI) { a.startUPID = "UPID:demo:start:8000:" a.waitFunc = func(upid string) (proxmox.TaskStatus, error) { if upid == "UPID:demo:start:8000:" { return proxmox.TaskStatus{}, errors.New("start task failed") } return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil } }}, {"waitRunning timeout", func(a *fakeAPI) { a.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}} }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} tc.setup(api) e, j, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h", BootTimeout: 40 * time.Millisecond, }) if res.Pass || res.Err == nil { t.Fatalf("must fail, got %+v", res) } // The adversarial point: the just-created guest was ACTUALLY destroyed. if len(api.destroys) != 1 || api.destroys[0] != vmid { t.Fatalf("compensating rollback must destroy the guest: destroys=%+v", api.destroys) } // And the owning entry is terminal (rollback complete) — not left in-flight. if len(j.InFlight()) != 0 { t.Errorf("owning entry must be terminal after rollback: %+v", j.InFlight()) } }) } } func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) { const vmid = 8001 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "ignored-in-dr", KeepMAC: true, Cores: 4, // a benign config PUT happens, so we can inspect it has NO identity resets }) if res.Err != nil || !res.Pass { t.Fatalf("dr bring-up must pass, got %+v", res) } p := setParamsFor(api, vmid) // DR continuity: MAC kept (no net0 reset) and hostname NOT force-reset. if p != nil { if _, ok := p["net0"]; ok { t.Errorf("dr+KeepMAC must NOT reset net0 (keep the archived MAC): %+v", p) } if _, ok := p["hostname"]; ok { t.Errorf("dr must NOT force-reset hostname (continuity): %+v", p) } if p["cores"] != "4" { t.Errorf("benign sizing should still apply: cores=%q", p["cores"]) } } // AssignedMAC reflects the kept archived MAC (from scratchCfg's net0). if res.AssignedMAC != "AA:BB:CC:DD:EE:FF" { t.Errorf("dr should keep the archived MAC, got %q", res.AssignedMAC) } if len(api.destroys) != 0 { t.Errorf("dr success must not destroy: %+v", api.destroys) } // The agent performs NO guest-internal host-key op — there is no such API call; host keys // are preserved (DR) or regenerated by the baked golden unit (provision). } func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) { const vmid = 8002 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeDRGuestLoss, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", KeepMAC: false, // a source guest may still be live → reset MAC even in DR }) if res.Err != nil || !res.Pass { t.Fatalf("got %+v", res) } p := setParamsFor(api, vmid) if p == nil || strings.Contains(p["net0"], "hwaddr=") { t.Errorf("dr with KeepMAC=false must reset MAC (net0 without hwaddr): %+v", p) } } func TestRunBringUp_LivenessIsTheVerdict(t *testing.T) { const vmid = 8000 const startUPID = "UPID:demo:start:8000:" mkAPI := func() *fakeAPI { return &fakeAPI{ cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, startUPID: startUPID, waitFunc: func(upid string) (proxmox.TaskStatus, error) { if upid == startUPID { return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil } return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil }, logTailFunc: func(string) ([]string, error) { return []string{"WARN: Systemd 257 detected. You may need to enable nesting."}, nil }, } } // start exits WARNINGS + guest reaches running → PASS, warnings surfaced + recognized. t.Run("warnings + running -> pass", func(t *testing.T) { api := mkAPI() e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if !res.Pass || res.Err != nil { t.Fatalf("warnings+running must pass: %+v", res) } if len(res.StartWarnings) != 1 || !res.WarningsRecognized { t.Errorf("warnings must be surfaced+recognized: %+v recognized=%v", res.StartWarnings, res.WarningsRecognized) } if len(api.destroys) != 0 { t.Errorf("a passed bring-up must not destroy: %+v", api.destroys) } }) // same warnings but guest NEVER reaches running → FAIL (verdict is liveness), guest destroyed. t.Run("warnings + not-running -> fail", func(t *testing.T) { api := mkAPI() api.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}} e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", BootTimeout: 40 * time.Millisecond, }) if res.Pass || res.Err == nil { t.Fatalf("not-running must fail regardless of warnings: %+v", res) } if len(api.destroys) != 1 { t.Errorf("a failed bring-up must roll back (destroy): %+v", api.destroys) } }) } func TestRunBringUp_F4_ConfigLockRetry(t *testing.T) { const vmid = 8000 lockBackoffFast(t) t.Run("transient lock-500 then 200 -> retries and proceeds", func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} var calls int api.setFunc = func(int, map[string]string) (string, error) { calls++ if calls == 1 { return "", &proxmox.APIError{StatusCode: 500, Body: "can't lock file '/run/lock/lxc/pve-config-8000.lock' - got timeout"} } return "", nil } e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Err != nil || !res.Pass { t.Fatalf("lock-500 then 200 must succeed: %+v", res) } if calls < 2 { t.Errorf("expected a retry on the transient lock-500, calls=%d", calls) } }) t.Run("non-lock 500 -> fails without retry", func(t *testing.T) { api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} var calls int api.setFunc = func(int, map[string]string) (string, error) { calls++ return "", &proxmox.APIError{StatusCode: 500, Body: "internal error: disk full"} } e, _, q := newEngine(t, api, EmptyProvider{}) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Pass || res.Err == nil { t.Fatalf("a non-lock 500 must fail: %+v", res) } if calls != 1 { t.Errorf("a real error must NOT be retried, calls=%d", calls) } }) } func TestRunBringUp_JournalsOwningEntryBeforeRestore(t *testing.T) { const vmid = 8000 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, j, q := newEngine(t, api, EmptyProvider{}) defer q.Close() // At the moment RestoreLXC is called, the owning Rollback entry must already be in-flight — // so a crash here is recoverable (Recover reaps the half-built guest). var ownedAtRestore bool api.restoreHook = func() { for _, en := range j.InFlight() { if en.VMID == vmid && en.Rollback { ownedAtRestore = true } } } res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h", }) if res.Err != nil { t.Fatalf("setup: %+v", res) } if !ownedAtRestore { t.Fatal("the owning Rollback entry MUST be journaled before the restore (crash-safety)") } } func TestRunBringUp_RejectsReservedAndExistingVMID(t *testing.T) { e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{}) defer q.Close() for _, id := range []int{9999, 990000, 990005} { res := e.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: id, RestoreStorage: "s"}) if res.Err == nil { t.Errorf("VMID %d is reserved and must be refused", id) } } // existing VMID → refuse (restore-over-existing is a signed op, not this benign path). api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 8000}}} e2, _, q2 := newEngine(t, api, EmptyProvider{}) defer q2.Close() res := e2.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: 8000, RestoreStorage: "s"}) if res.Err == nil { t.Error("bring-up over an existing guest must be refused") } if len(api.restores) != 0 || len(api.destroys) != 0 { t.Error("a refused bring-up must not restore or destroy anything") } } func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) { const vmid = 8000 // The guest still exists at startup (agent crashed mid-bring-up) → Recover destroys it. api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "running"}}} e, j, _ := newEngine(t, api, EmptyProvider{}) if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpTaskRunning, At: time.Now().UTC()}); err != nil { t.Fatal(err) } res := e.Recover(context.Background()) if res.BringUpRolledBack != 1 { t.Fatalf("half-built bring-up must be rolled back, got %+v", res) } if len(api.destroys) != 1 || api.destroys[0] != vmid { t.Fatalf("DestroyLXC not called for the half-built guest: %+v", api.destroys) } if len(j.InFlight()) != 0 { t.Errorf("resolved rollback entry must not be in-flight: %+v", j.InFlight()) } } func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) { const vmid = 8000 // Crash after the restore POST failed (no guest) → idempotent clean, no destroy. api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001}}} // 8000 absent e, j, _ := newEngine(t, api, EmptyProvider{}) j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()}) res := e.Recover(context.Background()) if res.BringUpClean != 1 || len(api.destroys) != 0 { t.Fatalf("already-gone bring-up must be clean with no destroy: res=%+v destroys=%+v", res, api.destroys) } if len(j.InFlight()) != 0 { t.Errorf("entry must be resolved: %+v", j.InFlight()) } } // lockBackoffFast shrinks the F4 retry backoff for tests and restores it after. func lockBackoffFast(t *testing.T) { t.Helper() prev := configLockBackoff configLockBackoff = time.Millisecond t.Cleanup(func() { configLockBackoff = prev }) }