felhom-agent/internal/reconcile/restoretest_test.go

package reconcile

import (
	"context"
	"encoding/json"
	"errors"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// scratchCfg builds a fake GuestConfig with one net interface (so the link-down SetConfig
// step runs).
func scratchCfg() proxmox.GuestConfig {
	return proxmox.GuestConfig{Extra: map[string]json.RawMessage{
		"net0": json.RawMessage(`"name=eth0,bridge=vmbr0,hwaddr=AA:BB:CC:DD:EE:FF,ip=dhcp"`),
	}}
}

func TestRunRestoreTest_PassAndTeardown(t *testing.T) {
	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}} // empty lxc → 990000 free; running default
	e, j, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
		Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm",
		ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
	})
	if res.Skipped || !res.Pass || res.Err != nil {
		t.Fatalf("expected pass, got %+v", res)
	}
	if res.ScratchVMID != 990000 || res.Verified != "boot+running" {
		t.Fatalf("result = %+v", res)
	}
	if len(api.restores) != 1 || api.restores[0].VMID != 990000 || api.restores[0].Archive != "local:backup/x.tar.zst" {
		t.Fatalf("restore not issued correctly: %+v", api.restores)
	}
	// net link-down applied before boot.
	foundLinkDown := false
	for _, s := range api.sets {
		if s.vmid == 990000 && s.params["net0"] != "" && contains2(s.params["net0"], "link_down=1") {
			foundLinkDown = true
		}
	}
	if !foundLinkDown {
		t.Errorf("expected a net link-down SetConfig, got %+v", api.sets)
	}
	// teardown destroyed the scratch guest, and the journal entry is terminal (not in-flight).
	if len(api.destroys) != 1 || api.destroys[0] != 990000 {
		t.Fatalf("scratch not torn down: %+v", api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("scratch entry must be terminal after teardown: %+v", j.InFlight())
	}
}

func TestRunRestoreTest_TeardownOnFailedVerify(t *testing.T) {
	// Guest never reaches running → verify fails, but teardown MUST still run.
	api := &fakeAPI{
		cfg:    map[int]proxmox.GuestConfig{990000: scratchCfg()},
		status: map[int]proxmox.Guest{990000: {VMID: 990000, Status: "stopped"}},
	}
	e, j, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
		Archive: "vol", RestoreStorage: "local-lvm",
		ScratchMin: 990000, ScratchMax: 990009, BootTimeout: 40 * time.Millisecond,
	})
	if res.Pass || res.Err == nil {
		t.Fatalf("expected a failed verify, got %+v", res)
	}
	if len(api.destroys) != 1 || api.destroys[0] != 990000 {
		t.Fatalf("teardown MUST run even on a failed verify: destroys=%+v", api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("scratch entry must be terminal after teardown: %+v", j.InFlight())
	}
}

func TestRunRestoreTest_RestoreFailureStillTearsDown(t *testing.T) {
	api := &fakeAPI{restoreErr: errors.New("restore boom")}
	e, j, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
	})
	if res.Pass || res.Err == nil {
		t.Fatalf("expected restore failure, got %+v", res)
	}
	// Even though restore failed, the scratch entry was journaled BEFORE the restore, so
	// teardown runs (idempotent — destroys the maybe-partial guest).
	if len(api.destroys) != 1 {
		t.Fatalf("teardown must run after a restore failure: %+v", api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("scratch entry must be terminal: %+v", j.InFlight())
	}
}

func TestRunRestoreTest_FullBandSkips(t *testing.T) {
	// Whole band occupied → skipped, never run / out-of-band.
	var guests []proxmox.Guest
	for id := 990000; id <= 990001; id++ {
		guests = append(guests, proxmox.Guest{VMID: id})
	}
	api := &fakeAPI{lxc: guests}
	e, _, q := newEngine(t, api, EmptyProvider{})
	defer q.Close()

	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990001,
	})
	if !res.Skipped {
		t.Fatalf("full band must skip, got %+v", res)
	}
	if len(api.restores) != 0 || len(api.destroys) != 0 {
		t.Errorf("a skipped test must not restore or destroy anything")
	}
}

func TestRunRestoreTest_InvalidBandErrors(t *testing.T) {
	e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{})
	defer q.Close()
	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{Archive: "v", RestoreStorage: "s", ScratchMin: 0})
	if res.Err == nil {
		t.Fatal("an invalid scratch band must error")
	}
}

func TestPickScratchVMID(t *testing.T) {
	// excludes 9999 and in-use; lowest free.
	got, ok := pickScratchVMID([]proxmox.Guest{{VMID: 990000}}, 990000, 990009)
	if !ok || got != 990001 {
		t.Errorf("pick = %d,%v want 990001,true", got, ok)
	}
	// full band.
	full := []proxmox.Guest{{VMID: 990000}, {VMID: 990001}}
	if _, ok := pickScratchVMID(full, 990000, 990001); ok {
		t.Error("full band must return ok=false")
	}
}

func TestWithLinkDown(t *testing.T) {
	got := withLinkDown("name=eth0,bridge=vmbr0,ip=dhcp")
	if !contains2(got, "link_down=1") || !contains2(got, "name=eth0") {
		t.Errorf("withLinkDown lost fields or didn't set link_down: %q", got)
	}
	// idempotent: an existing link_down is replaced, not duplicated.
	got = withLinkDown("name=eth0,link_down=0,bridge=vmbr0")
	if count(got, "link_down=") != 1 || !contains2(got, "link_down=1") {
		t.Errorf("withLinkDown must replace an existing link_down (got %q)", got)
	}
}

// --- recover the leaked scratch guest (the headline crash-safety test) ---

func TestRecover_LeakedScratchDestroyed(t *testing.T) {
	// The scratch guest still exists at startup (agent crashed mid-test) → Recover destroys it.
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 990000, Status: "running"}}}
	e, j, _ := newEngine(t, api, EmptyProvider{})
	if err := j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	res := e.Recover(context.Background())
	if res.ScratchDestroyed != 1 {
		t.Fatalf("leaked scratch must be destroyed, got %+v", res)
	}
	if len(api.destroys) != 1 || api.destroys[0] != 990000 {
		t.Fatalf("DestroyLXC not called for the leaked scratch: %+v", api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("resolved scratch entry must not be in-flight: %+v", j.InFlight())
	}
}

func TestRecover_LeakedScratchAlreadyGone(t *testing.T) {
	// Crash AFTER the destroy task but BEFORE the terminal record → guest already gone →
	// idempotent clean (no destroy issued).
	api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001, Status: "stopped"}}} // 990000 absent
	e, j, _ := newEngine(t, api, EmptyProvider{})
	j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()})
	res := e.Recover(context.Background())
	if res.ScratchClean != 1 || len(api.destroys) != 0 {
		t.Fatalf("already-gone scratch must be clean with no destroy, got res=%+v destroys=%+v", res, api.destroys)
	}
	if len(j.InFlight()) != 0 {
		t.Errorf("entry must be resolved: %+v", j.InFlight())
	}
}

func TestRecover_LeakedScratchListUnreadable(t *testing.T) {
	api := &fakeAPI{listErr: errors.New("api down")}
	e, j, _ := newEngine(t, api, EmptyProvider{})
	j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()})
	res := e.Recover(context.Background())
	if res.Unresolved != 1 || len(j.InFlight()) != 1 {
		t.Fatalf("unreadable list must leave the scratch in-flight for a later Recover, got res=%+v inflight=%d", res, len(j.InFlight()))
	}
	if len(api.destroys) != 0 {
		t.Error("must not destroy when it can't confirm the guest exists")
	}
}

// small string helpers (avoid importing strings in the test for one call).
func contains2(s, sub string) bool { return indexOf(s, sub) >= 0 }
func count(s, sub string) int {
	n, i := 0, 0
	for {
		j := indexOf(s[i:], sub)
		if j < 0 {
			return n
		}
		n++
		i += j + len(sub)
	}
}
func indexOf(s, sub string) int {
	for i := 0; i+len(sub) <= len(s); i++ {
		if s[i:i+len(sub)] == sub {
			return i
		}
	}
	return -1
}