felhom-agent/internal/reconcile/bringup_test.go

package reconcile

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a
// substring of the joined command — makes that one command fail (the C2 mid-swap failure driver).
type fakeRunner struct {
	mu     sync.Mutex
	cmds   []string
	failOn string
}

func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
	cmd := name + " " + strings.Join(args, " ")
	f.mu.Lock()
	f.cmds = append(f.cmds, cmd)
	f.mu.Unlock()
	if f.failOn != "" && strings.Contains(cmd, f.failOn) {
		return nil, []byte("boom"), errors.New("exit status 1")
	}
	return nil, nil, nil
}

func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
	return f.Run(ctx, name, args...)
}

// newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an
// isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it).
func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) {
	t.Helper()
	jp := filepath.Join(t.TempDir(), "journal.log")
	j, err := OpenJournal(jp)
	if err != nil {
		t.Fatalf("OpenJournal: %v", err)
	}
	t.Cleanup(func() { j.Close() })
	q := NewQueue()
	t.Cleanup(q.Close)
	fr := &fakeRunner{}
	sd := t.TempDir()
	e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd})
	return e, fr, sd, q
}

// setParamsFor returns the params of the (last) SetConfig call against vmid, or nil.
func setParamsFor(api *fakeAPI, vmid int) map[string]string {
	var out map[string]string
	for _, s := range api.sets {
		if s.vmid == vmid {
			out = s.params
		}
	}
	return out
}

func TestRunBringUp_ProvisionHappyPath(t *testing.T) {
	const vmid = 8000
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} // empty lxc → vmid free; running default
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "felhom-prov-8000",
		Cores: 2, MemoryMB: 2048, RootfsGrowGB: 2,
		Mounts: []GuestMount{{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/data"}},
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("provision must pass, got %+v", res)
	}
	if res.Verified != "boot+running" {
		t.Errorf("verified = %q", res.Verified)
	}
	// restore issued to the target vmid.
	if len(api.restores) != 1 || api.restores[0].VMID != vmid || api.restores[0].Archive != "local:backup/golden.tar.zst" {
		t.Fatalf("restore not issued correctly: %+v", api.restores)
	}
	// identity: a fresh MAC (net0 set WITHOUT hwaddr) + hostname, coalesced with sizing+mount.
	p := setParamsFor(api, vmid)
	if p == nil {
		t.Fatal("expected a coalesced config PUT")
	}
	if net0, ok := p["net0"]; !ok || strings.Contains(net0, "hwaddr=") {
		t.Errorf("provision must reset MAC: net0 must be set WITHOUT hwaddr, got %q", net0)
	}
	if p["hostname"] != "felhom-prov-8000" {
		t.Errorf("hostname not set: %q", p["hostname"])
	}
	if p["cores"] != "2" || p["memory"] != "2048" {
		t.Errorf("sizing not coalesced: cores=%q memory=%q", p["cores"], p["memory"])
	}
	if p["mp0"] != "local-lvm:1,mp=/mnt/data" {
		t.Errorf("mount not attached: mp0=%q", p["mp0"])
	}
	// rootfs grow is a SEPARATE call (F4).
	if len(api.resizes) != 1 || api.resizes[0].vmid != vmid || api.resizes[0].size != "+2G" {
		t.Errorf("rootfs grow not issued separately: %+v", api.resizes)
	}
	// started link-up.
	if len(api.starts) != 1 || api.starts[0] != vmid {
		t.Errorf("guest not started: %+v", api.starts)
	}
	// THE key difference from the restore-test: the guest is KEPT (no teardown).
	if len(api.destroys) != 0 {
		t.Fatalf("provision success must NOT destroy the guest: %+v", api.destroys)
	}
}

// A data-bearing additive mount must carry backup=1 (so its DBs stay in PBS — storage-split B3);
// a non-backup mount must NOT. Pure-function check on buildBringUpConfig.
func TestBuildBringUpConfig_BackupFlagOnDataMount(t *testing.T) {
	params := buildBringUpConfig(BringUpSpec{
		Mode: ModeProvision,
		Mounts: []GuestMount{
			{Storage: "local-lvm", SizeGB: 2, MountPoint: "/mnt/data", Backup: true},
			{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/scratch"}, // no backup
		},
	}, scratchCfg())
	if params["mp0"] != "local-lvm:2,mp=/mnt/data,backup=1" {
		t.Errorf("data mount must carry backup=1: mp0=%q", params["mp0"])
	}
	if params["mp1"] != "local-lvm:1,mp=/mnt/scratch" {
		t.Errorf("non-backup mount must NOT carry backup=1: mp1=%q", params["mp1"])
	}
}

// The operator-opt-in CPU/RAM cap is emitted into the SAME pre-start config PUT (so the guest never
// boots uncapped), and — critically — is OMITTED when unset (0), so an uncapped provision keeps the
// golden's baked sizes instead of being shrunk to 0. Pure-function check on buildBringUpConfig.
func TestBuildBringUpConfig_ResourceCaps(t *testing.T) {
	// caps set → both keys present, MiB integer for memory
	capped := buildBringUpConfig(BringUpSpec{
		Mode: ModeProvision, Cores: 2, MemoryMB: 4096,
	}, scratchCfg())
	if capped["cores"] != "2" {
		t.Errorf("cores cap must be emitted: cores=%q", capped["cores"])
	}
	if capped["memory"] != "4096" {
		t.Errorf("memory cap must be emitted (MiB): memory=%q", capped["memory"])
	}

	// caps unset (0) → NEITHER key present (omit-when-zero; else the guest would shrink to 0 cores)
	uncapped := buildBringUpConfig(BringUpSpec{
		Mode: ModeProvision, Cores: 0, MemoryMB: 0,
	}, scratchCfg())
	if _, ok := uncapped["cores"]; ok {
		t.Errorf("cores must be ABSENT when unset (golden default), got %q", uncapped["cores"])
	}
	if _, ok := uncapped["memory"]; ok {
		t.Errorf("memory must be ABSENT when unset (golden default), got %q", uncapped["memory"])
	}
}

// Both restore sites allocate the guest INTO the felhom pool (SPIKE 3b): the provision bring-up
// threads spec.Pool, and the restore-test hardcodes DefaultPool — else a pool-scoped token 403s on
// the created guest's config/start/destroy. Asserts via the fakeAPI's captured RestoreLXCOptions.
func TestRestoreSitesUsePool(t *testing.T) {
	// provision bring-up: spec.Pool must flow to the restore
	const vmid = 8060
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()
	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/g.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Pool: DefaultPool,
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("bring-up must pass: %+v", res)
	}
	if len(api.restores) != 1 || api.restores[0].Pool != DefaultPool {
		t.Fatalf("provision restore must set Pool=%q, got %+v", DefaultPool, api.restores)
	}

	// restore-test: the scratch restore hardcodes DefaultPool
	api2 := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}}
	e2, _, q2 := newEngine(t, api2, EmptyProvider{})
	defer q2.Close()
	rt := e2.RunRestoreTest(context.Background(), RestoreTestSpec{
		Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm",
		ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
	})
	if rt.Err != nil || !rt.Pass {
		t.Fatalf("restore-test must pass: %+v", rt)
	}
	if len(api2.restores) != 1 || api2.restores[0].Pool != DefaultPool {
		t.Fatalf("restore-test restore must set Pool=%q, got %+v", DefaultPool, api2.restores)
	}
}

// The golden-carried Docker-data volume is grown via a SEPARATE resize on its mpN slot (B4),
// alongside (but distinct from) the rootfs grow.
func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) {
	const vmid = 8050
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "felhom-prov-8050",
		RootfsGrowGB: 8, DataVolGrowGB: 240, // grows mp0 (DefaultDataVolMount)
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("provision must pass, got %+v", res)
	}
	// TWO resizes: rootfs +8G and the Docker-data volume mp0 +240G.
	if len(api.resizes) != 2 {
		t.Fatalf("expected rootfs + data-volume resizes, got %+v", api.resizes)
	}
	var sawRootfs, sawData bool
	for _, r := range api.resizes {
		if r.disk == "rootfs" && r.size == "+8G" {
			sawRootfs = true
		}
		if r.disk == "mp0" && r.size == "+240G" {
			sawData = true
		}
	}
	if !sawRootfs || !sawData {
		t.Errorf("want rootfs +8G AND mp0 +240G, got %+v", api.resizes)
	}
}

// The golden-carried SSD user-data volume (/mnt/sys_drive) is grown via a SEPARATE resize on its
// mpN slot (mp1), independent of the rootfs and Docker-data grows. With SysDataGrowGB=0 NO mp1
// resize is issued (the volume stays at the golden size, still a separate mount).
func TestRunBringUp_StorageSplit_SysDataGrow(t *testing.T) {
	const vmid = 8051
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "felhom-prov-8051",
		DataVolGrowGB: 240, SysDataGrowGB: 42, // grows mp0 AND mp1 (DefaultSysDataMount)
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("provision must pass, got %+v", res)
	}
	// TWO resizes here: Docker-data mp0 +240G and the user-data volume mp1 +42G (no rootfs grow).
	if len(api.resizes) != 2 {
		t.Fatalf("expected data-volume + sys-data resizes, got %+v", api.resizes)
	}
	var sawData, sawSys bool
	for _, r := range api.resizes {
		if r.disk == "mp0" && r.size == "+240G" {
			sawData = true
		}
		if r.disk == "mp1" && r.size == "+42G" {
			sawSys = true
		}
	}
	if !sawData || !sawSys {
		t.Errorf("want mp0 +240G AND mp1 +42G, got %+v", api.resizes)
	}
}

// SysDataGrowGB=0 must issue NO mp1 resize (grow is an orthogonal knob; separateness comes from the
// golden, not the grow).
func TestRunBringUp_StorageSplit_SysDataGrowZeroNoResize(t *testing.T) {
	const vmid = 8052
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "felhom-prov-8052",
		SysDataGrowGB: 0, // no sys-data grow
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("provision must pass, got %+v", res)
	}
	for _, r := range api.resizes {
		if r.disk == "mp1" {
			t.Errorf("SysDataGrowGB=0 must NOT resize mp1, got %+v", api.resizes)
		}
	}
}

func TestRunBringUp_CompensatingRollback(t *testing.T) {
	const vmid = 8000
	lockBackoffFast(t)

	cases := []struct {
		name  string
		setup func(*fakeAPI)
	}{
		{"restore-task failure after launch", func(a *fakeAPI) {
			// Scenario B (no-regression): the restore POST was ACCEPTED (UPID) but the task then
			// fails → we created/started it → the compensating destroy MUST still fire.
			a.restoreUPID = "UPID:demo:restore:8000:"
			a.waitFunc = func(upid string) (proxmox.TaskStatus, error) {
				if upid == "UPID:demo:restore:8000:" {
					return proxmox.TaskStatus{}, errors.New("restore task failed")
				}
				return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
			}
		}},
		{"config real error", func(a *fakeAPI) {
			a.setFunc = func(int, map[string]string) (string, error) {
				return "", &proxmox.APIError{StatusCode: 500, Body: "some non-lock internal error"}
			}
		}},
		{"start-task real error", func(a *fakeAPI) {
			a.startUPID = "UPID:demo:start:8000:"
			a.waitFunc = func(upid string) (proxmox.TaskStatus, error) {
				if upid == "UPID:demo:start:8000:" {
					return proxmox.TaskStatus{}, errors.New("start task failed")
				}
				return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
			}
		}},
		{"waitRunning timeout", func(a *fakeAPI) {
			a.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}}
		}},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
			tc.setup(api)
			e, j, q := newEngine(t, api, EmptyProvider{})
			defer q.Close()

			res := e.RunBringUp(context.Background(), BringUpSpec{
				Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm",
				Hostname: "h", BootTimeout: 40 * time.Millisecond,
			})
			if res.Pass || res.Err == nil {
				t.Fatalf("must fail, got %+v", res)
			}
			// The adversarial point: the just-created guest was ACTUALLY destroyed.
			if len(api.destroys) != 1 || api.destroys[0] != vmid {
				t.Fatalf("compensating rollback must destroy the guest: destroys=%+v", api.destroys)
			}
			// And the owning entry is terminal (rollback complete) — not left in-flight.
			if len(j.InFlight()) != 0 {
				t.Errorf("owning entry must be terminal after rollback: %+v", j.InFlight())
			}
		})
	}
}

// TestRunBringUp_NoLaunchNoDestroy is the F1a red-proof (campaign pool-effects): a restore that
// fails SYNCHRONOUSLY (no UPID — nothing created) must NOT arm the compensating destroy. The
// headline case is PVE refusing a vmid that already holds a guest the pool-blind duplicate guard
// couldn't see — the old defer would have destroyed that innocent pre-existing guest (only the
// pool ACL's 403 saved the non-pool subset; an in-pool one would have been destroyed).
// Red-proof: revert the `launched` gate in runBringUp and this test fails with destroys=[8000].
func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) {
	const vmid = 8000
	cases := []struct {
		name string
		err  error
	}{
		{"PVE refuses pre-existing vmid", &proxmox.APIError{
			StatusCode: 500, Method: "POST", Path: "/nodes/x/lxc",
			Body: `{"message":"CT 8000 already exists on node 'x'\n","data":null}`,
		}},
		{"plain synchronous restore error", errors.New("restore boom")},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, restoreErr: tc.err}
			e, j, q := newEngine(t, api, EmptyProvider{})
			defer q.Close()

			res := e.RunBringUp(context.Background(), BringUpSpec{
				Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h",
			})
			if res.Pass || res.Err == nil {
				t.Fatalf("must fail, got %+v", res)
			}
			// THE point: no destroy is even attempted — the txn created nothing.
			if len(api.destroys) != 0 {
				t.Fatalf("no-launch failure must NOT destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
			}
			// And the owning entry is closed terminal in-process (not left for Recover).
			if len(j.InFlight()) != 0 {
				t.Errorf("owning entry must be terminal after a no-launch failure: %+v", j.InFlight())
			}
		})
	}
}

func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
	const vmid = 8001
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "ignored-in-dr", KeepMAC: true,
		Cores: 4, // a benign config PUT happens, so we can inspect it has NO identity resets
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("dr bring-up must pass, got %+v", res)
	}
	p := setParamsFor(api, vmid)
	// DR continuity: MAC kept (no net0 reset) and hostname NOT force-reset.
	if p != nil {
		if _, ok := p["net0"]; ok {
			t.Errorf("dr+KeepMAC must NOT reset net0 (keep the archived MAC): %+v", p)
		}
		if _, ok := p["hostname"]; ok {
			t.Errorf("dr must NOT force-reset hostname (continuity): %+v", p)
		}
		if p["cores"] != "4" {
			t.Errorf("benign sizing should still apply: cores=%q", p["cores"])
		}
	}
	// AssignedMAC reflects the kept archived MAC (from scratchCfg's net0).
	if res.AssignedMAC != "AA:BB:CC:DD:EE:FF" {
		t.Errorf("dr should keep the archived MAC, got %q", res.AssignedMAC)
	}
	if len(api.destroys) != 0 {
		t.Errorf("dr success must not destroy: %+v", api.destroys)
	}
	// The agent performs NO guest-internal host-key op — there is no such API call; host keys
	// are preserved (DR) or regenerated by the baked golden unit (provision).
}

func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) {
	const vmid = 8002
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm",
		KeepMAC: false, // a source guest may still be live → reset MAC even in DR
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("got %+v", res)
	}
	p := setParamsFor(api, vmid)
	if p == nil || strings.Contains(p["net0"], "hwaddr=") {
		t.Errorf("dr with KeepMAC=false must reset MAC (net0 without hwaddr): %+v", p)
	}
}

func TestRunBringUp_LivenessIsTheVerdict(t *testing.T) {
	const vmid = 8000
	const startUPID = "UPID:demo:start:8000:"
	mkAPI := func() *fakeAPI {
		return &fakeAPI{
			cfg:       map[int]proxmox.GuestConfig{vmid: scratchCfg()},
			startUPID: startUPID,
			waitFunc: func(upid string) (proxmox.TaskStatus, error) {
				if upid == startUPID {
					return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil
				}
				return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
			},
			logTailFunc: func(string) ([]string, error) {
				return []string{"WARN: Systemd 257 detected. You may need to enable nesting."}, nil
			},
		}
	}

	// start exits WARNINGS + guest reaches running → PASS, warnings surfaced + recognized.
	t.Run("warnings + running -> pass", func(t *testing.T) {
		api := mkAPI()
		e, _, q := newEngine(t, api, EmptyProvider{})
		defer q.Close()
		res := e.RunBringUp(context.Background(), BringUpSpec{
			Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
		})
		if !res.Pass || res.Err != nil {
			t.Fatalf("warnings+running must pass: %+v", res)
		}
		if len(res.StartWarnings) != 1 || !res.WarningsRecognized {
			t.Errorf("warnings must be surfaced+recognized: %+v recognized=%v", res.StartWarnings, res.WarningsRecognized)
		}
		if len(api.destroys) != 0 {
			t.Errorf("a passed bring-up must not destroy: %+v", api.destroys)
		}
	})

	// same warnings but guest NEVER reaches running → FAIL (verdict is liveness), guest destroyed.
	t.Run("warnings + not-running -> fail", func(t *testing.T) {
		api := mkAPI()
		api.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}}
		e, _, q := newEngine(t, api, EmptyProvider{})
		defer q.Close()
		res := e.RunBringUp(context.Background(), BringUpSpec{
			Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
			BootTimeout: 40 * time.Millisecond,
		})
		if res.Pass || res.Err == nil {
			t.Fatalf("not-running must fail regardless of warnings: %+v", res)
		}
		if len(api.destroys) != 1 {
			t.Errorf("a failed bring-up must roll back (destroy): %+v", api.destroys)
		}
	})
}

func TestRunBringUp_F4_ConfigLockRetry(t *testing.T) {
	const vmid = 8000
	lockBackoffFast(t)

	t.Run("transient lock-500 then 200 -> retries and proceeds", func(t *testing.T) {
		api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
		var calls int
		api.setFunc = func(int, map[string]string) (string, error) {
			calls++
			if calls == 1 {
				return "", &proxmox.APIError{StatusCode: 500, Body: "can't lock file '/run/lock/lxc/pve-config-8000.lock' - got timeout"}
			}
			return "", nil
		}
		e, _, q := newEngine(t, api, EmptyProvider{})
		defer q.Close()
		res := e.RunBringUp(context.Background(), BringUpSpec{
			Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
		})
		if res.Err != nil || !res.Pass {
			t.Fatalf("lock-500 then 200 must succeed: %+v", res)
		}
		if calls < 2 {
			t.Errorf("expected a retry on the transient lock-500, calls=%d", calls)
		}
	})

	t.Run("non-lock 500 -> fails without retry", func(t *testing.T) {
		api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
		var calls int
		api.setFunc = func(int, map[string]string) (string, error) {
			calls++
			return "", &proxmox.APIError{StatusCode: 500, Body: "internal error: disk full"}
		}
		e, _, q := newEngine(t, api, EmptyProvider{})
		defer q.Close()
		res := e.RunBringUp(context.Background(), BringUpSpec{
			Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
		})
		if res.Pass || res.Err == nil {
			t.Fatalf("a non-lock 500 must fail: %+v", res)
		}
		if calls != 1 {
			t.Errorf("a real error must NOT be retried, calls=%d", calls)
		}
	})
}

func TestRunBringUp_JournalsOwningEntryBeforeRestore(t *testing.T) {
	const vmid = 8000
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, j, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	// At the moment RestoreLXC is called, the owning Rollback entry must already be in-flight —
	// so a crash here is recoverable (Recover reaps the half-built guest).
	var ownedAtRestore bool
	api.restoreHook = func() {
		for _, en := range j.InFlight() {
			if en.VMID == vmid && en.Rollback {
				ownedAtRestore = true
			}
		}
	}
	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
	})
	if res.Err != nil {
		t.Fatalf("setup: %+v", res)
	}
	if !ownedAtRestore {
		t.Fatal("the owning Rollback entry MUST be journaled before the restore (crash-safety)")
	}
}

func TestRunBringUp_RejectsReservedAndExistingVMID(t *testing.T) {
	e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{})
	defer q.Close()
	for _, id := range []int{9999, 990000, 990005} {
		res := e.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: id, RestoreStorage: "s"})
		if res.Err == nil {
			t.Errorf("VMID %d is reserved and must be refused", id)
		}
	}
	// existing VMID → refuse (restore-over-existing is a signed op, not this benign path).
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 8000}}}
	e2, _, q2 := newEngine(t, api, EmptyProvider{})
	defer q2.Close()
	res := e2.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: 8000, RestoreStorage: "s"})
	if res.Err == nil {
		t.Error("bring-up over an existing guest must be refused")
	}
	if len(api.restores) != 0 || len(api.destroys) != 0 {
		t.Error("a refused bring-up must not restore or destroy anything")
	}
}

func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) {
	const vmid = 8000
	// The guest still exists at startup (agent crashed mid-bring-up AFTER the restore launched —
	// the entry carries the UPID = the launch proof) → Recover destroys it.
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "running"}}}
	e, j, _ := newEngine(t, api, EmptyProvider{})
	if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	res := e.Recover(context.Background())
	if res.BringUpRolledBack != 1 {
		t.Fatalf("half-built bring-up must be rolled back, got %+v", res)
	}
	if len(api.destroys) != 1 || api.destroys[0] != vmid {
		t.Fatalf("DestroyLXC not called for the half-built guest: %+v", api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("resolved rollback entry must not be in-flight: %+v", j.InFlight())
	}
}

func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) {
	const vmid = 8000
	// Crash after a LAUNCHED restore whose guest is already gone → idempotent clean, no destroy.
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001}}} // 8000 absent
	e, j, _ := newEngine(t, api, EmptyProvider{})
	j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()})
	res := e.Recover(context.Background())
	if res.BringUpClean != 1 || len(api.destroys) != 0 {
		t.Fatalf("already-gone bring-up must be clean with no destroy: res=%+v destroys=%+v", res, api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("entry must be resolved: %+v", j.InFlight())
	}
}

// TestRecover_BringUpNoUPIDAbandoned is the F1c red-proof (campaign pool-effects, Scenario C): a
// crash left a Rollback entry with NO UPID (the restore was never confirmed) while a guest sits
// at that vmid — e.g. a PRE-EXISTING guest the pool-blind list-existence check would have called
// "ours". Recover must ABANDON (fail-safe), never destroy: no journaled UPID ⇒ this transaction
// created nothing. Red-proof: route the entry to recoverBringUp (pre-fix order) and this fails
// with destroys=[8000].
func TestRecover_BringUpNoUPIDAbandoned(t *testing.T) {
	const vmid = 8000
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "stopped"}}} // a guest IS at the vmid
	e, j, _ := newEngine(t, api, EmptyProvider{})
	j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()})
	res := e.Recover(context.Background())
	if len(api.destroys) != 0 {
		t.Fatalf("a no-UPID Rollback entry must NEVER destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
	}
	if res.RolledBack != 1 || res.BringUpRolledBack != 0 {
		t.Fatalf("entry must be abandoned via the no-UPID fail-safe path, got %+v", res)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("abandoned entry must be terminal: %+v", j.InFlight())
	}
}

// lockBackoffFast shrinks the F4 retry backoff for tests and restores it after.
func lockBackoffFast(t *testing.T) {
	t.Helper()
	prev := configLockBackoff
	configLockBackoff = time.Millisecond
	t.Cleanup(func() { configLockBackoff = prev })
}

// --- Part B (campaign-2 R2): pool re-assertion after bring-up ------------------------------------

// poolAddsFor returns the recorded PoolAddVMID calls for a vmid.
func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall {
	var out []poolAddCall
	for _, c := range api.poolAdds {
		if c.vmid == vmid {
			out = append(out, c)
		}
	}
	return out
}

// A bring-up with a Pool set MUST re-assert pool membership after the restore (so a
// restore-over-existing that dropped membership is healed for the NEXT restore-test/DR).
// COMPANION RED-PROOF: on the pre-fix code (no PoolAddVMID call) poolAddsFor is empty → this FAILS.
func TestRunBringUp_ReassertsPoolMembership(t *testing.T) {
	const vmid = 8100
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Pool: "felhom",
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("bring-up must pass, got %+v", res)
	}
	adds := poolAddsFor(api, vmid)
	if len(adds) != 1 || adds[0].pool != "felhom" {
		t.Fatalf("pool membership not re-asserted: poolAdds=%+v", api.poolAdds)
	}
}

// No Pool → no PoolAddVMID call (a broad-token restore needs no pool).
func TestRunBringUp_NoPool_NoPoolAdd(t *testing.T) {
	const vmid = 8101
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "h",
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("bring-up must pass, got %+v", res)
	}
	if len(api.poolAdds) != 0 {
		t.Fatalf("no Pool set → PoolAddVMID must not be called: %+v", api.poolAdds)
	}
}

// A pool-add FAILURE must NOT flip a healthy, running guest's verdict — it surfaces as a LOUD warning
// (membership matters for the NEXT op, not this guest's boot). Liveness wins.
func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
	const vmid = 8102
	api := &fakeAPI{
		cfg:        map[int]proxmox.GuestConfig{vmid: scratchCfg()},
		poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"),
	}
	e, _, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Pool: "felhom",
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("a pool-add hiccup must NOT fail a healthy guest, got %+v", res)
	}
	// the guest is KEPT (not torn down) despite the pool-add failure.
	if len(api.destroys) != 0 {
		t.Fatalf("pool-add failure must not trigger teardown: %+v", api.destroys)
	}
	// the failure is surfaced as a warning.
	found := false
	for _, w := range res.StartWarnings {
		if strings.Contains(w, "pool re-assert failed") {
			found = true
		}
	}
	if !found {
		t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings)
	}
}

// ── GL-5: DR structural bind overrides + 4d swap ─────────────────────────────────────────────────

// GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural
// binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via
// the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes.
// COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts
// (run→fail→revert, recorded in the REPORT).
func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) {
	const vmid = 8200
	cfg := scratchCfg()
	// after the swap PVE parks the two displaced throwaway volumes as unusedN
	cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`)
	cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`)
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
	e, fr, sd, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", KeepMAC: true,
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("dr bring-up must pass, got %+v", res)
	}
	if len(api.restores) != 1 {
		t.Fatalf("expected one restore, got %+v", api.restores)
	}
	ov := api.restores[0].MountOverrides
	want := map[string]string{
		"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
		"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
	}
	if len(ov) != 2 || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] {
		t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want)
	}
	// 4d: mp9 host dir created under the engine state dir …
	bootDir := structuralBootHostDir(sd, vmid)
	if st, err := os.Stat(bootDir); err != nil || !st.IsDir() {
		t.Fatalf("mp9 bootstrap host dir not created: %v", err)
	}
	// … and the REAL binds set via the root runner, one slot per call, exact backhalf values.
	wantCmds := []string{
		"mkdir -p /mnt/felhom-drives",
		"pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives",
		"pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1",
	}
	if len(fr.cmds) != len(wantCmds) {
		t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds)
	}
	for i := range wantCmds {
		if fr.cmds[i] != wantCmds[i] {
			t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i])
		}
	}
	// displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue.
	del := ""
	for _, s := range api.sets {
		if s.vmid == vmid && s.params["delete"] != "" {
			del = s.params["delete"]
		}
	}
	if del != "unused0,unused1" {
		t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets)
	}
}

// GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never
// touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the
// override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT).
func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) {
	const vmid = 8201
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", Hostname: "h",
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("provision must pass, got %+v", res)
	}
	if api.restores[0].MountOverrides != nil {
		t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides)
	}
	if len(fr.cmds) != 0 {
		t.Fatalf("provision must not touch the host runner, got %v", fr.cmds)
	}
}

// GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state
// named and compensating-rolls-back the guest — never a silent half-wired success.
func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) {
	const vmid = 8202
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, fr, _, q := newDREngine(t, api)
	defer q.Close()
	fr.failOn = "-mp9"

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", KeepMAC: true,
	})
	if res.Err == nil || res.Pass {
		t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res)
	}
	if !strings.Contains(res.Err.Error(), "structural-bind swap") ||
		!strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") {
		t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err)
	}
	if len(api.destroys) != 1 || api.destroys[0] != vmid {
		t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys)
	}
	if len(api.starts) != 0 {
		t.Fatalf("a half-wired guest must never be started, got %+v", api.starts)
	}
}

// GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived,
// so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the
// end state is identical, with no unusedN residue (here only ONE displaced volume shows up).
func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) {
	const vmid = 8203
	cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape
	cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`)
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
	e, fr, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", KeepMAC: true,
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res)
	}
	ov := api.restores[0].MountOverrides
	if len(ov) != 2 || ov["mp8"] == "" || ov["mp9"] == "" {
		t.Fatalf("overrides are constants — both mpN must be named regardless of the archive: %+v", ov)
	}
	if len(fr.cmds) != 3 {
		t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds)
	}
	del := ""
	for _, s := range api.sets {
		if s.vmid == vmid && s.params["delete"] != "" {
			del = s.params["delete"]
		}
	}
	if del != "unused0" {
		t.Fatalf("the one displaced volume must be deleted: delete=%q", del)
	}
}

// GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because
// the structural-bind swap is a root pct op the API token cannot perform.
func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) {
	const vmid = 8204
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
	e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm",
	})
	if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") {
		t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res)
	}
	if len(api.restores) != 0 {
		t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores)
	}
}

// GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired
// guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently).
func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) {
	const vmid = 8205
	cfg := scratchCfg()
	cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`)
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
	api.setFunc = func(_ int, params map[string]string) (string, error) {
		if params["delete"] != "" {
			return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed")
		}
		return "", nil
	}
	e, _, _, q := newDREngine(t, api)
	defer q.Close()

	res := e.RunBringUp(context.Background(), BringUpSpec{
		Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
		RestoreStorage: "local-lvm", KeepMAC: true,
	})
	if res.Err != nil || !res.Pass {
		t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res)
	}
	found := false
	for _, w := range res.StartWarnings {
		if strings.Contains(w, "displaced volumes not deleted") {
			found = true
		}
	}
	if !found {
		t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings)
	}
}