admin
cd6e26785a
gates / gates (push) Successful in 5s
build-golden.sh 2.1.0 -> 3.0.0: a layout change is a major. The golden ships ONE data volume at a NEUTRAL path (/var/lib/felhom); both /var/lib/docker and /mnt/sys_drive are binds of subdirectories of it. mp1 is gone. The variant was chosen on measurement. Three candidates were built and rebooted (SPIKE-r165-phase0-2026-08-03.md); all three boot, reboot 3/3, give ONE df figure and keep a container's statfs(/) on the merged volume — the ordering worry that motivated the probe did not materialise. They differ only in which guarantee they break: volume-at-docker puts customer backups inside Docker's data-root; volume-at-sys_drive puts Docker's ENTIRE data-root under /mnt, which the controller container mounts wholesale (measured: it then sees /mnt/sys_drive/docker). V-c breaks neither. The four assertions were RETARGETED, never deleted, and each was RUN against a deliberately wrong shape — a real split guest and a real archive of it: 8 checks, 8 passed. A new 2b asserts both paths are ONE filesystem, which catches the S2 shape the spike ranked worse than the split. Assertion 5 replaces the old "was mp1 excluded?" guard, whose pattern could no longer match — a guard that cannot match has silently stopped guarding. Provisioning: one volume, one grow. SysDataGrowGB is FOLDED IN rather than dropped, because a census established that felhom-host-install.sh passes -sysdata-grow and the two do not upgrade in the same instant; dropping it would silently shrink every appliance by 42 of 250 GiB. The flags stay accepted for the same reason. The existing test was retargeted to pin the fold, and it caught the change before I did.
1094 lines
43 KiB
Go
1094 lines
43 KiB
Go
package reconcile
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a
|
|
// substring of the joined command — makes that one command fail (the C2 mid-swap failure driver).
|
|
type fakeRunner struct {
|
|
mu sync.Mutex
|
|
cmds []string
|
|
failOn string
|
|
}
|
|
|
|
func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
|
cmd := name + " " + strings.Join(args, " ")
|
|
f.mu.Lock()
|
|
f.cmds = append(f.cmds, cmd)
|
|
f.mu.Unlock()
|
|
if f.failOn != "" && strings.Contains(cmd, f.failOn) {
|
|
return nil, []byte("boom"), errors.New("exit status 1")
|
|
}
|
|
return nil, nil, nil
|
|
}
|
|
|
|
func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
|
return f.Run(ctx, name, args...)
|
|
}
|
|
|
|
// newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an
|
|
// isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it).
|
|
func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) {
|
|
t.Helper()
|
|
jp := filepath.Join(t.TempDir(), "journal.log")
|
|
j, err := OpenJournal(jp)
|
|
if err != nil {
|
|
t.Fatalf("OpenJournal: %v", err)
|
|
}
|
|
t.Cleanup(func() { j.Close() })
|
|
q := NewQueue()
|
|
t.Cleanup(q.Close)
|
|
fr := &fakeRunner{}
|
|
sd := t.TempDir()
|
|
e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd})
|
|
return e, fr, sd, q
|
|
}
|
|
|
|
// setParamsFor returns the params of the (last) SetConfig call against vmid, or nil.
|
|
func setParamsFor(api *fakeAPI, vmid int) map[string]string {
|
|
var out map[string]string
|
|
for _, s := range api.sets {
|
|
if s.vmid == vmid {
|
|
out = s.params
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestRunBringUp_ProvisionHappyPath(t *testing.T) {
|
|
const vmid = 8000
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} // empty lxc → vmid free; running default
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "felhom-prov-8000",
|
|
Cores: 2, MemoryMB: 2048, RootfsGrowGB: 2,
|
|
Mounts: []GuestMount{{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/data"}},
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("provision must pass, got %+v", res)
|
|
}
|
|
if res.Verified != "boot+running" {
|
|
t.Errorf("verified = %q", res.Verified)
|
|
}
|
|
// restore issued to the target vmid.
|
|
if len(api.restores) != 1 || api.restores[0].VMID != vmid || api.restores[0].Archive != "local:backup/golden.tar.zst" {
|
|
t.Fatalf("restore not issued correctly: %+v", api.restores)
|
|
}
|
|
// identity: a fresh MAC (net0 set WITHOUT hwaddr) + hostname, coalesced with sizing+mount.
|
|
p := setParamsFor(api, vmid)
|
|
if p == nil {
|
|
t.Fatal("expected a coalesced config PUT")
|
|
}
|
|
if net0, ok := p["net0"]; !ok || strings.Contains(net0, "hwaddr=") {
|
|
t.Errorf("provision must reset MAC: net0 must be set WITHOUT hwaddr, got %q", net0)
|
|
}
|
|
if p["hostname"] != "felhom-prov-8000" {
|
|
t.Errorf("hostname not set: %q", p["hostname"])
|
|
}
|
|
if p["cores"] != "2" || p["memory"] != "2048" {
|
|
t.Errorf("sizing not coalesced: cores=%q memory=%q", p["cores"], p["memory"])
|
|
}
|
|
if p["mp0"] != "local-lvm:1,mp=/mnt/data" {
|
|
t.Errorf("mount not attached: mp0=%q", p["mp0"])
|
|
}
|
|
// rootfs grow is a SEPARATE call (F4).
|
|
if len(api.resizes) != 1 || api.resizes[0].vmid != vmid || api.resizes[0].size != "+2G" {
|
|
t.Errorf("rootfs grow not issued separately: %+v", api.resizes)
|
|
}
|
|
// started link-up.
|
|
if len(api.starts) != 1 || api.starts[0] != vmid {
|
|
t.Errorf("guest not started: %+v", api.starts)
|
|
}
|
|
// THE key difference from the restore-test: the guest is KEPT (no teardown).
|
|
if len(api.destroys) != 0 {
|
|
t.Fatalf("provision success must NOT destroy the guest: %+v", api.destroys)
|
|
}
|
|
}
|
|
|
|
// A data-bearing additive mount must carry backup=1 (so its DBs stay in PBS — storage-split B3);
|
|
// a non-backup mount must NOT. Pure-function check on buildBringUpConfig.
|
|
func TestBuildBringUpConfig_BackupFlagOnDataMount(t *testing.T) {
|
|
params := buildBringUpConfig(BringUpSpec{
|
|
Mode: ModeProvision,
|
|
Mounts: []GuestMount{
|
|
{Storage: "local-lvm", SizeGB: 2, MountPoint: "/mnt/data", Backup: true},
|
|
{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/scratch"}, // no backup
|
|
},
|
|
}, scratchCfg())
|
|
if params["mp0"] != "local-lvm:2,mp=/mnt/data,backup=1" {
|
|
t.Errorf("data mount must carry backup=1: mp0=%q", params["mp0"])
|
|
}
|
|
if params["mp1"] != "local-lvm:1,mp=/mnt/scratch" {
|
|
t.Errorf("non-backup mount must NOT carry backup=1: mp1=%q", params["mp1"])
|
|
}
|
|
}
|
|
|
|
// The operator-opt-in CPU/RAM cap is emitted into the SAME pre-start config PUT (so the guest never
|
|
// boots uncapped), and — critically — is OMITTED when unset (0), so an uncapped provision keeps the
|
|
// golden's baked sizes instead of being shrunk to 0. Pure-function check on buildBringUpConfig.
|
|
func TestBuildBringUpConfig_ResourceCaps(t *testing.T) {
|
|
// caps set → both keys present, MiB integer for memory
|
|
capped := buildBringUpConfig(BringUpSpec{
|
|
Mode: ModeProvision, Cores: 2, MemoryMB: 4096,
|
|
}, scratchCfg())
|
|
if capped["cores"] != "2" {
|
|
t.Errorf("cores cap must be emitted: cores=%q", capped["cores"])
|
|
}
|
|
if capped["memory"] != "4096" {
|
|
t.Errorf("memory cap must be emitted (MiB): memory=%q", capped["memory"])
|
|
}
|
|
|
|
// caps unset (0) → NEITHER key present (omit-when-zero; else the guest would shrink to 0 cores)
|
|
uncapped := buildBringUpConfig(BringUpSpec{
|
|
Mode: ModeProvision, Cores: 0, MemoryMB: 0,
|
|
}, scratchCfg())
|
|
if _, ok := uncapped["cores"]; ok {
|
|
t.Errorf("cores must be ABSENT when unset (golden default), got %q", uncapped["cores"])
|
|
}
|
|
if _, ok := uncapped["memory"]; ok {
|
|
t.Errorf("memory must be ABSENT when unset (golden default), got %q", uncapped["memory"])
|
|
}
|
|
}
|
|
|
|
// R-50: with the island configured, bring-up attaches a static net1 on the island bridge; with it
|
|
// unset (or half-set), NO net1 is emitted — byte-for-byte the pre-R-50 config on non-island hosts.
|
|
// Pure-function check on buildBringUpConfig (the derivation that makes fresh installs F1-immune).
|
|
func TestBuildBringUpConfig_IslandNIC(t *testing.T) {
|
|
// island set → net1 present, exact shape, no hwaddr (PVE mints a fresh per-guest MAC)
|
|
island := buildBringUpConfig(BringUpSpec{
|
|
Mode: ModeProvision, IslandBridge: "vmbr9", IslandGuestAddr: "169.254.253.2/30",
|
|
}, scratchCfg())
|
|
if got, want := island["net1"], "name=eth1,bridge=vmbr9,ip=169.254.253.2/30"; got != want {
|
|
t.Errorf("island net1 mismatch:\n got %q\nwant %q", got, want)
|
|
}
|
|
// DR mode too — a restored customer guest must also reach the island-bound agent on the host.
|
|
dr := buildBringUpConfig(BringUpSpec{
|
|
Mode: ModeDRGuestLoss, KeepMAC: true, IslandBridge: "vmbr9", IslandGuestAddr: "169.254.253.2/30",
|
|
}, scratchCfg())
|
|
if _, ok := dr["net1"]; !ok {
|
|
t.Errorf("DR bring-up must also attach the island net1, got none")
|
|
}
|
|
// island unset → NO net1 key (non-island hosts unchanged; the pre-R-50 default)
|
|
none := buildBringUpConfig(BringUpSpec{Mode: ModeProvision}, scratchCfg())
|
|
if v, ok := none["net1"]; ok {
|
|
t.Errorf("net1 must be ABSENT when the island is not configured, got %q", v)
|
|
}
|
|
// half-configured (bridge only) → still no net1 (all-or-nothing; config.Validate rejects the config too)
|
|
half := buildBringUpConfig(BringUpSpec{Mode: ModeProvision, IslandBridge: "vmbr9"}, scratchCfg())
|
|
if v, ok := half["net1"]; ok {
|
|
t.Errorf("net1 must be ABSENT when only the bridge is set, got %q", v)
|
|
}
|
|
}
|
|
|
|
// Both restore sites allocate the guest INTO the felhom pool (SPIKE 3b): the provision bring-up
|
|
// threads spec.Pool, and the restore-test hardcodes DefaultPool — else a pool-scoped token 403s on
|
|
// the created guest's config/start/destroy. Asserts via the fakeAPI's captured RestoreLXCOptions.
|
|
func TestRestoreSitesUsePool(t *testing.T) {
|
|
// provision bring-up: spec.Pool must flow to the restore
|
|
const vmid = 8060
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/g.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Pool: DefaultPool,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("bring-up must pass: %+v", res)
|
|
}
|
|
if len(api.restores) != 1 || api.restores[0].Pool != DefaultPool {
|
|
t.Fatalf("provision restore must set Pool=%q, got %+v", DefaultPool, api.restores)
|
|
}
|
|
|
|
// restore-test: the scratch restore hardcodes DefaultPool
|
|
api2 := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}}
|
|
e2, _, q2 := newEngine(t, api2, EmptyProvider{})
|
|
defer q2.Close()
|
|
rt := e2.RunRestoreTest(context.Background(), RestoreTestSpec{
|
|
Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm",
|
|
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
|
})
|
|
if rt.Err != nil || !rt.Pass {
|
|
t.Fatalf("restore-test must pass: %+v", rt)
|
|
}
|
|
if len(api2.restores) != 1 || api2.restores[0].Pool != DefaultPool {
|
|
t.Fatalf("restore-test restore must set Pool=%q, got %+v", DefaultPool, api2.restores)
|
|
}
|
|
}
|
|
|
|
// The golden-carried Docker-data volume is grown via a SEPARATE resize on its mpN slot (B4),
|
|
// alongside (but distinct from) the rootfs grow.
|
|
func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) {
|
|
const vmid = 8050
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "felhom-prov-8050",
|
|
RootfsGrowGB: 8, DataVolGrowGB: 240, // grows mp0 (DefaultDataVolMount)
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("provision must pass, got %+v", res)
|
|
}
|
|
// TWO resizes: rootfs +8G and the Docker-data volume mp0 +240G.
|
|
if len(api.resizes) != 2 {
|
|
t.Fatalf("expected rootfs + data-volume resizes, got %+v", api.resizes)
|
|
}
|
|
var sawRootfs, sawData bool
|
|
for _, r := range api.resizes {
|
|
if r.disk == "rootfs" && r.size == "+8G" {
|
|
sawRootfs = true
|
|
}
|
|
if r.disk == "mp0" && r.size == "+240G" {
|
|
sawData = true
|
|
}
|
|
}
|
|
if !sawRootfs || !sawData {
|
|
t.Errorf("want rootfs +8G AND mp0 +240G, got %+v", api.resizes)
|
|
}
|
|
}
|
|
|
|
// R-165 RETARGETED THIS TEST, and the retarget IS the contract change. There is no longer a second
|
|
// volume, so `SysDataGrowGB` no longer drives its own resize on mp1 — its GiB are FOLDED INTO the
|
|
// single volume's grow.
|
|
//
|
|
// FOLDED, NOT DROPPED, and that is the whole point. `felhom-host-install.sh` computes and passes
|
|
// `-sysdata-grow` from the thin pool's free space, and an installer and an agent do not upgrade in
|
|
// the same instant. Dropping the value would silently shrink every appliance built by an older
|
|
// installer by the user-data share — 42 of 250 GiB on the standard branch — which is precisely the
|
|
// "a knob that silently does nothing" outcome this work was told to avoid.
|
|
func TestRunBringUp_StorageSplit_SysDataGrowIsFoldedIn(t *testing.T) {
|
|
const vmid = 8051
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "felhom-prov-8051",
|
|
DataVolGrowGB: 240, SysDataGrowGB: 42, // ONE volume: 240 + 42 = 282
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("provision must pass, got %+v", res)
|
|
}
|
|
// EXACTLY ONE resize. A second one would mean an mp1 the golden no longer ships.
|
|
if len(api.resizes) != 1 {
|
|
t.Fatalf("expected exactly ONE data-volume resize (there is no mp1 since R-165), got %+v", api.resizes)
|
|
}
|
|
r := api.resizes[0]
|
|
if r.disk != "mp0" {
|
|
t.Fatalf("resized %q, want mp0 — the single data volume", r.disk)
|
|
}
|
|
if r.size != "+282G" {
|
|
t.Fatalf("resized %s, want +282G (240 data + 42 folded sys-data). Anything less means the "+
|
|
"retired knob's GiB were DROPPED, silently shrinking every appliance an older "+
|
|
"felhom-host-install.sh provisions", r.size)
|
|
}
|
|
for _, rr := range api.resizes {
|
|
if rr.disk == "mp1" {
|
|
t.Fatalf("an mp1 resize was issued (%+v) — the golden ships no second volume, so this "+
|
|
"would fail on a real box", rr)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SysDataGrowGB=0 must issue NO mp1 resize (grow is an orthogonal knob; separateness comes from the
|
|
// golden, not the grow).
|
|
func TestRunBringUp_StorageSplit_SysDataGrowZeroNoResize(t *testing.T) {
|
|
const vmid = 8052
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "felhom-prov-8052",
|
|
SysDataGrowGB: 0, // no sys-data grow
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("provision must pass, got %+v", res)
|
|
}
|
|
for _, r := range api.resizes {
|
|
if r.disk == "mp1" {
|
|
t.Errorf("SysDataGrowGB=0 must NOT resize mp1, got %+v", api.resizes)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunBringUp_CompensatingRollback(t *testing.T) {
|
|
const vmid = 8000
|
|
lockBackoffFast(t)
|
|
|
|
cases := []struct {
|
|
name string
|
|
setup func(*fakeAPI)
|
|
}{
|
|
{"restore-task failure after launch", func(a *fakeAPI) {
|
|
// Scenario B (no-regression): the restore POST was ACCEPTED (UPID) but the task then
|
|
// fails → we created/started it → the compensating destroy MUST still fire.
|
|
a.restoreUPID = "UPID:demo:restore:8000:"
|
|
a.waitFunc = func(upid string) (proxmox.TaskStatus, error) {
|
|
if upid == "UPID:demo:restore:8000:" {
|
|
return proxmox.TaskStatus{}, errors.New("restore task failed")
|
|
}
|
|
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
|
|
}
|
|
}},
|
|
{"config real error", func(a *fakeAPI) {
|
|
a.setFunc = func(int, map[string]string) (string, error) {
|
|
return "", &proxmox.APIError{StatusCode: 500, Body: "some non-lock internal error"}
|
|
}
|
|
}},
|
|
{"start-task real error", func(a *fakeAPI) {
|
|
a.startUPID = "UPID:demo:start:8000:"
|
|
a.waitFunc = func(upid string) (proxmox.TaskStatus, error) {
|
|
if upid == "UPID:demo:start:8000:" {
|
|
return proxmox.TaskStatus{}, errors.New("start task failed")
|
|
}
|
|
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
|
|
}
|
|
}},
|
|
{"waitRunning timeout", func(a *fakeAPI) {
|
|
a.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}}
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
tc.setup(api)
|
|
e, j, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm",
|
|
Hostname: "h", BootTimeout: 40 * time.Millisecond,
|
|
})
|
|
if res.Pass || res.Err == nil {
|
|
t.Fatalf("must fail, got %+v", res)
|
|
}
|
|
// The adversarial point: the just-created guest was ACTUALLY destroyed.
|
|
if len(api.destroys) != 1 || api.destroys[0] != vmid {
|
|
t.Fatalf("compensating rollback must destroy the guest: destroys=%+v", api.destroys)
|
|
}
|
|
// And the owning entry is terminal (rollback complete) — not left in-flight.
|
|
if len(j.InFlight()) != 0 {
|
|
t.Errorf("owning entry must be terminal after rollback: %+v", j.InFlight())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestRunBringUp_NoLaunchNoDestroy is the F1a red-proof (campaign pool-effects): a restore that
|
|
// fails SYNCHRONOUSLY (no UPID — nothing created) must NOT arm the compensating destroy. The
|
|
// headline case is PVE refusing a vmid that already holds a guest the pool-blind duplicate guard
|
|
// couldn't see — the old defer would have destroyed that innocent pre-existing guest (only the
|
|
// pool ACL's 403 saved the non-pool subset; an in-pool one would have been destroyed).
|
|
// Red-proof: revert the `launched` gate in runBringUp and this test fails with destroys=[8000].
|
|
func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) {
|
|
const vmid = 8000
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
}{
|
|
{"PVE refuses pre-existing vmid", &proxmox.APIError{
|
|
StatusCode: 500, Method: "POST", Path: "/nodes/x/lxc",
|
|
Body: `{"message":"CT 8000 already exists on node 'x'\n","data":null}`,
|
|
}},
|
|
{"plain synchronous restore error", errors.New("restore boom")},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, restoreErr: tc.err}
|
|
e, j, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h",
|
|
})
|
|
if res.Pass || res.Err == nil {
|
|
t.Fatalf("must fail, got %+v", res)
|
|
}
|
|
// THE point: no destroy is even attempted — the txn created nothing.
|
|
if len(api.destroys) != 0 {
|
|
t.Fatalf("no-launch failure must NOT destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
|
|
}
|
|
// And the owning entry is closed terminal in-process (not left for Recover).
|
|
if len(j.InFlight()) != 0 {
|
|
t.Errorf("owning entry must be terminal after a no-launch failure: %+v", j.InFlight())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
|
|
const vmid = 8001
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "ignored-in-dr", KeepMAC: true,
|
|
Cores: 4, // a benign config PUT happens, so we can inspect it has NO identity resets
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("dr bring-up must pass, got %+v", res)
|
|
}
|
|
p := setParamsFor(api, vmid)
|
|
// DR continuity: MAC kept (no net0 reset) and hostname NOT force-reset.
|
|
if p != nil {
|
|
if _, ok := p["net0"]; ok {
|
|
t.Errorf("dr+KeepMAC must NOT reset net0 (keep the archived MAC): %+v", p)
|
|
}
|
|
if _, ok := p["hostname"]; ok {
|
|
t.Errorf("dr must NOT force-reset hostname (continuity): %+v", p)
|
|
}
|
|
if p["cores"] != "4" {
|
|
t.Errorf("benign sizing should still apply: cores=%q", p["cores"])
|
|
}
|
|
}
|
|
// AssignedMAC reflects the kept archived MAC (from scratchCfg's net0).
|
|
if res.AssignedMAC != "AA:BB:CC:DD:EE:FF" {
|
|
t.Errorf("dr should keep the archived MAC, got %q", res.AssignedMAC)
|
|
}
|
|
if len(api.destroys) != 0 {
|
|
t.Errorf("dr success must not destroy: %+v", api.destroys)
|
|
}
|
|
// The agent performs NO guest-internal host-key op — there is no such API call; host keys
|
|
// are preserved (DR) or regenerated by the baked golden unit (provision).
|
|
}
|
|
|
|
func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) {
|
|
const vmid = 8002
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm",
|
|
KeepMAC: false, // a source guest may still be live → reset MAC even in DR
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("got %+v", res)
|
|
}
|
|
p := setParamsFor(api, vmid)
|
|
if p == nil || strings.Contains(p["net0"], "hwaddr=") {
|
|
t.Errorf("dr with KeepMAC=false must reset MAC (net0 without hwaddr): %+v", p)
|
|
}
|
|
}
|
|
|
|
func TestRunBringUp_LivenessIsTheVerdict(t *testing.T) {
|
|
const vmid = 8000
|
|
const startUPID = "UPID:demo:start:8000:"
|
|
mkAPI := func() *fakeAPI {
|
|
return &fakeAPI{
|
|
cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()},
|
|
startUPID: startUPID,
|
|
waitFunc: func(upid string) (proxmox.TaskStatus, error) {
|
|
if upid == startUPID {
|
|
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil
|
|
}
|
|
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
|
|
},
|
|
logTailFunc: func(string) ([]string, error) {
|
|
return []string{"WARN: Systemd 257 detected. You may need to enable nesting."}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// start exits WARNINGS + guest reaches running → PASS, warnings surfaced + recognized.
|
|
t.Run("warnings + running -> pass", func(t *testing.T) {
|
|
api := mkAPI()
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
|
|
})
|
|
if !res.Pass || res.Err != nil {
|
|
t.Fatalf("warnings+running must pass: %+v", res)
|
|
}
|
|
if len(res.StartWarnings) != 1 || !res.WarningsRecognized {
|
|
t.Errorf("warnings must be surfaced+recognized: %+v recognized=%v", res.StartWarnings, res.WarningsRecognized)
|
|
}
|
|
if len(api.destroys) != 0 {
|
|
t.Errorf("a passed bring-up must not destroy: %+v", api.destroys)
|
|
}
|
|
})
|
|
|
|
// same warnings but guest NEVER reaches running → FAIL (verdict is liveness), guest destroyed.
|
|
t.Run("warnings + not-running -> fail", func(t *testing.T) {
|
|
api := mkAPI()
|
|
api.status = map[int]proxmox.Guest{vmid: {VMID: vmid, Status: "stopped"}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
|
|
BootTimeout: 40 * time.Millisecond,
|
|
})
|
|
if res.Pass || res.Err == nil {
|
|
t.Fatalf("not-running must fail regardless of warnings: %+v", res)
|
|
}
|
|
if len(api.destroys) != 1 {
|
|
t.Errorf("a failed bring-up must roll back (destroy): %+v", api.destroys)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRunBringUp_F4_ConfigLockRetry(t *testing.T) {
|
|
const vmid = 8000
|
|
lockBackoffFast(t)
|
|
|
|
t.Run("transient lock-500 then 200 -> retries and proceeds", func(t *testing.T) {
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
var calls int
|
|
api.setFunc = func(int, map[string]string) (string, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
return "", &proxmox.APIError{StatusCode: 500, Body: "can't lock file '/run/lock/lxc/pve-config-8000.lock' - got timeout"}
|
|
}
|
|
return "", nil
|
|
}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("lock-500 then 200 must succeed: %+v", res)
|
|
}
|
|
if calls < 2 {
|
|
t.Errorf("expected a retry on the transient lock-500, calls=%d", calls)
|
|
}
|
|
})
|
|
|
|
t.Run("non-lock 500 -> fails without retry", func(t *testing.T) {
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
var calls int
|
|
api.setFunc = func(int, map[string]string) (string, error) {
|
|
calls++
|
|
return "", &proxmox.APIError{StatusCode: 500, Body: "internal error: disk full"}
|
|
}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
|
|
})
|
|
if res.Pass || res.Err == nil {
|
|
t.Fatalf("a non-lock 500 must fail: %+v", res)
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("a real error must NOT be retried, calls=%d", calls)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRunBringUp_JournalsOwningEntryBeforeRestore(t *testing.T) {
|
|
const vmid = 8000
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, j, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
// At the moment RestoreLXC is called, the owning Rollback entry must already be in-flight —
|
|
// so a crash here is recoverable (Recover reaps the half-built guest).
|
|
var ownedAtRestore bool
|
|
api.restoreHook = func() {
|
|
for _, en := range j.InFlight() {
|
|
if en.VMID == vmid && en.Rollback {
|
|
ownedAtRestore = true
|
|
}
|
|
}
|
|
}
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "v", VMID: vmid, RestoreStorage: "s", Hostname: "h",
|
|
})
|
|
if res.Err != nil {
|
|
t.Fatalf("setup: %+v", res)
|
|
}
|
|
if !ownedAtRestore {
|
|
t.Fatal("the owning Rollback entry MUST be journaled before the restore (crash-safety)")
|
|
}
|
|
}
|
|
|
|
func TestRunBringUp_RejectsReservedAndExistingVMID(t *testing.T) {
|
|
e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{})
|
|
defer q.Close()
|
|
for _, id := range []int{9999, 990000, 990005} {
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: id, RestoreStorage: "s"})
|
|
if res.Err == nil {
|
|
t.Errorf("VMID %d is reserved and must be refused", id)
|
|
}
|
|
}
|
|
// existing VMID → refuse (restore-over-existing is a signed op, not this benign path).
|
|
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 8000}}}
|
|
e2, _, q2 := newEngine(t, api, EmptyProvider{})
|
|
defer q2.Close()
|
|
res := e2.RunBringUp(context.Background(), BringUpSpec{Mode: ModeProvision, Archive: "v", VMID: 8000, RestoreStorage: "s"})
|
|
if res.Err == nil {
|
|
t.Error("bring-up over an existing guest must be refused")
|
|
}
|
|
if len(api.restores) != 0 || len(api.destroys) != 0 {
|
|
t.Error("a refused bring-up must not restore or destroy anything")
|
|
}
|
|
}
|
|
|
|
func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) {
|
|
const vmid = 8000
|
|
// The guest still exists at startup (agent crashed mid-bring-up AFTER the restore launched —
|
|
// the entry carries the UPID = the launch proof) → Recover destroys it.
|
|
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "running"}}}
|
|
e, j, _ := newEngine(t, api, EmptyProvider{})
|
|
if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res := e.Recover(context.Background())
|
|
if res.BringUpRolledBack != 1 {
|
|
t.Fatalf("half-built bring-up must be rolled back, got %+v", res)
|
|
}
|
|
if len(api.destroys) != 1 || api.destroys[0] != vmid {
|
|
t.Fatalf("DestroyLXC not called for the half-built guest: %+v", api.destroys)
|
|
}
|
|
if len(j.InFlight()) != 0 {
|
|
t.Errorf("resolved rollback entry must not be in-flight: %+v", j.InFlight())
|
|
}
|
|
}
|
|
|
|
func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) {
|
|
const vmid = 8000
|
|
// Crash after a LAUNCHED restore whose guest is already gone → idempotent clean, no destroy.
|
|
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001}}} // 8000 absent
|
|
e, j, _ := newEngine(t, api, EmptyProvider{})
|
|
j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()})
|
|
res := e.Recover(context.Background())
|
|
if res.BringUpClean != 1 || len(api.destroys) != 0 {
|
|
t.Fatalf("already-gone bring-up must be clean with no destroy: res=%+v destroys=%+v", res, api.destroys)
|
|
}
|
|
if len(j.InFlight()) != 0 {
|
|
t.Errorf("entry must be resolved: %+v", j.InFlight())
|
|
}
|
|
}
|
|
|
|
// TestRecover_BringUpNoUPIDAbandoned is the F1c red-proof (campaign pool-effects, Scenario C): a
|
|
// crash left a Rollback entry with NO UPID (the restore was never confirmed) while a guest sits
|
|
// at that vmid — e.g. a PRE-EXISTING guest the pool-blind list-existence check would have called
|
|
// "ours". Recover must ABANDON (fail-safe), never destroy: no journaled UPID ⇒ this transaction
|
|
// created nothing. Red-proof: route the entry to recoverBringUp (pre-fix order) and this fails
|
|
// with destroys=[8000].
|
|
func TestRecover_BringUpNoUPIDAbandoned(t *testing.T) {
|
|
const vmid = 8000
|
|
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "stopped"}}} // a guest IS at the vmid
|
|
e, j, _ := newEngine(t, api, EmptyProvider{})
|
|
j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()})
|
|
res := e.Recover(context.Background())
|
|
if len(api.destroys) != 0 {
|
|
t.Fatalf("a no-UPID Rollback entry must NEVER destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
|
|
}
|
|
if res.RolledBack != 1 || res.BringUpRolledBack != 0 {
|
|
t.Fatalf("entry must be abandoned via the no-UPID fail-safe path, got %+v", res)
|
|
}
|
|
if len(j.InFlight()) != 0 {
|
|
t.Errorf("abandoned entry must be terminal: %+v", j.InFlight())
|
|
}
|
|
}
|
|
|
|
// lockBackoffFast shrinks the F4 retry backoff for tests and restores it after.
|
|
func lockBackoffFast(t *testing.T) {
|
|
t.Helper()
|
|
prev := configLockBackoff
|
|
configLockBackoff = time.Millisecond
|
|
t.Cleanup(func() { configLockBackoff = prev })
|
|
}
|
|
|
|
// --- Part B (campaign-2 R2): pool re-assertion after bring-up ------------------------------------
|
|
|
|
// poolAddsFor returns the recorded PoolAddVMID calls for a vmid.
|
|
func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall {
|
|
var out []poolAddCall
|
|
for _, c := range api.poolAdds {
|
|
if c.vmid == vmid {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// A bring-up with a Pool set MUST re-assert pool membership after the restore (so a
|
|
// restore-over-existing that dropped membership is healed for the NEXT restore-test/DR).
|
|
// COMPANION RED-PROOF: on the pre-fix code (no PoolAddVMID call) poolAddsFor is empty → this FAILS.
|
|
func TestRunBringUp_ReassertsPoolMembership(t *testing.T) {
|
|
const vmid = 8100
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Pool: "felhom",
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("bring-up must pass, got %+v", res)
|
|
}
|
|
adds := poolAddsFor(api, vmid)
|
|
if len(adds) != 1 || adds[0].pool != "felhom" {
|
|
t.Fatalf("pool membership not re-asserted: poolAdds=%+v", api.poolAdds)
|
|
}
|
|
}
|
|
|
|
// No Pool → no PoolAddVMID call (a broad-token restore needs no pool).
|
|
func TestRunBringUp_NoPool_NoPoolAdd(t *testing.T) {
|
|
const vmid = 8101
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{})
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "h",
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("bring-up must pass, got %+v", res)
|
|
}
|
|
if len(api.poolAdds) != 0 {
|
|
t.Fatalf("no Pool set → PoolAddVMID must not be called: %+v", api.poolAdds)
|
|
}
|
|
}
|
|
|
|
// A pool-add FAILURE must NOT flip a healthy, running guest's verdict — it surfaces as a LOUD warning
|
|
// (membership matters for the NEXT op, not this guest's boot). Liveness wins.
|
|
func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
|
|
const vmid = 8102
|
|
api := &fakeAPI{
|
|
cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()},
|
|
poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"),
|
|
}
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Pool: "felhom",
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("a pool-add hiccup must NOT fail a healthy guest, got %+v", res)
|
|
}
|
|
// the guest is KEPT (not torn down) despite the pool-add failure.
|
|
if len(api.destroys) != 0 {
|
|
t.Fatalf("pool-add failure must not trigger teardown: %+v", api.destroys)
|
|
}
|
|
// the failure is surfaced as a warning.
|
|
found := false
|
|
for _, w := range res.StartWarnings {
|
|
if strings.Contains(w, "pool re-assert failed") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings)
|
|
}
|
|
}
|
|
|
|
// ── GL-5: DR structural bind overrides + 4d swap ─────────────────────────────────────────────────
|
|
|
|
// GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural
|
|
// binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via
|
|
// the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes.
|
|
// COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts
|
|
// (run→fail→revert, recorded in the REPORT).
|
|
func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) {
|
|
const vmid = 8200
|
|
cfg := scratchCfg()
|
|
// after the swap PVE parks the two displaced throwaway volumes as unusedN
|
|
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`)
|
|
cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`)
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
|
e, fr, sd, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", KeepMAC: true,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("dr bring-up must pass, got %+v", res)
|
|
}
|
|
if len(api.restores) != 1 {
|
|
t.Fatalf("expected one restore, got %+v", api.restores)
|
|
}
|
|
ov := api.restores[0].MountOverrides
|
|
// rootfs rides along explicitly (PVE refuses mpN params without it — hit live in the GL-5
|
|
// validation), sized from the ARCHIVE's embedded config (the fake serves size=8G).
|
|
want := map[string]string{
|
|
"rootfs": "local-lvm:8",
|
|
"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
|
|
"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
|
|
}
|
|
if len(ov) != 3 || ov["rootfs"] != want["rootfs"] || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] {
|
|
t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want)
|
|
}
|
|
if len(api.extracts) != 1 || api.extracts[0] != "local:backup/customer.tar.zst" {
|
|
t.Fatalf("the rootfs size must come from the archive's extracted config: %+v", api.extracts)
|
|
}
|
|
// 4d: mp9 host dir created under the engine state dir …
|
|
bootDir := structuralBootHostDir(sd, vmid)
|
|
if st, err := os.Stat(bootDir); err != nil || !st.IsDir() {
|
|
t.Fatalf("mp9 bootstrap host dir not created: %v", err)
|
|
}
|
|
// … and the REAL binds set via the root runner, one slot per call, exact backhalf values.
|
|
wantCmds := []string{
|
|
"mkdir -p /mnt/felhom-drives",
|
|
"pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives",
|
|
"pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1",
|
|
}
|
|
if len(fr.cmds) != len(wantCmds) {
|
|
t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds)
|
|
}
|
|
for i := range wantCmds {
|
|
if fr.cmds[i] != wantCmds[i] {
|
|
t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i])
|
|
}
|
|
}
|
|
// displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue.
|
|
del := ""
|
|
for _, s := range api.sets {
|
|
if s.vmid == vmid && s.params["delete"] != "" {
|
|
del = s.params["delete"]
|
|
}
|
|
}
|
|
if del != "unused0,unused1" {
|
|
t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets)
|
|
}
|
|
}
|
|
|
|
// GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never
|
|
// touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the
|
|
// override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT).
|
|
func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) {
|
|
const vmid = 8201
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", Hostname: "h",
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("provision must pass, got %+v", res)
|
|
}
|
|
if api.restores[0].MountOverrides != nil {
|
|
t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides)
|
|
}
|
|
if len(fr.cmds) != 0 {
|
|
t.Fatalf("provision must not touch the host runner, got %v", fr.cmds)
|
|
}
|
|
}
|
|
|
|
// GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state
|
|
// named and compensating-rolls-back the guest — never a silent half-wired success.
|
|
func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) {
|
|
const vmid = 8202
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, fr, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
fr.failOn = "-mp9"
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", KeepMAC: true,
|
|
})
|
|
if res.Err == nil || res.Pass {
|
|
t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res)
|
|
}
|
|
if !strings.Contains(res.Err.Error(), "structural-bind swap") ||
|
|
!strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") {
|
|
t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err)
|
|
}
|
|
if len(api.destroys) != 1 || api.destroys[0] != vmid {
|
|
t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys)
|
|
}
|
|
if len(api.starts) != 0 {
|
|
t.Fatalf("a half-wired guest must never be started, got %+v", api.starts)
|
|
}
|
|
}
|
|
|
|
// GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived,
|
|
// so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the
|
|
// end state is identical, with no unusedN residue (here only ONE displaced volume shows up).
|
|
func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) {
|
|
const vmid = 8203
|
|
cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape
|
|
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`)
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
|
e, fr, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", KeepMAC: true,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res)
|
|
}
|
|
ov := api.restores[0].MountOverrides
|
|
if len(ov) != 3 || ov["mp8"] == "" || ov["mp9"] == "" || ov["rootfs"] == "" {
|
|
t.Fatalf("overrides are constants (+explicit rootfs) — both mpN must be named regardless of the archive: %+v", ov)
|
|
}
|
|
if len(fr.cmds) != 3 {
|
|
t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds)
|
|
}
|
|
del := ""
|
|
for _, s := range api.sets {
|
|
if s.vmid == vmid && s.params["delete"] != "" {
|
|
del = s.params["delete"]
|
|
}
|
|
}
|
|
if del != "unused0" {
|
|
t.Fatalf("the one displaced volume must be deleted: delete=%q", del)
|
|
}
|
|
}
|
|
|
|
// GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because
|
|
// the structural-bind swap is a root pct op the API token cannot perform.
|
|
func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) {
|
|
const vmid = 8204
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm",
|
|
})
|
|
if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") {
|
|
t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res)
|
|
}
|
|
if len(api.restores) != 0 {
|
|
t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores)
|
|
}
|
|
}
|
|
|
|
// GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired
|
|
// guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently).
|
|
func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) {
|
|
const vmid = 8205
|
|
cfg := scratchCfg()
|
|
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`)
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
|
api.setFunc = func(_ int, params map[string]string) (string, error) {
|
|
if params["delete"] != "" {
|
|
return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed")
|
|
}
|
|
return "", nil
|
|
}
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm", KeepMAC: true,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res)
|
|
}
|
|
found := false
|
|
for _, w := range res.StartWarnings {
|
|
if strings.Contains(w, "displaced volumes not deleted") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings)
|
|
}
|
|
}
|
|
|
|
// GL-5: drRestoreOverrides builds the COMPLETE explicit-params restore set from the archive's
|
|
// extracted config — the live validation proved PVE drops any mountpoint NOT named in the params,
|
|
// so mp0/mp1 pass-through is what keeps a DR guest's data volumes. Snapshot sections never shadow;
|
|
// unknown binds and unparseable sizes refuse.
|
|
func TestDRRestoreOverrides(t *testing.T) {
|
|
raw := `hostname: demo
|
|
rootfs: local-lvm:vm-9201-disk-0,size=32G
|
|
mp0: local-lvm:vm-9201-disk-1,mp=/var/lib/docker,backup=1,size=200G
|
|
mp1: local-lvm:vm-9201-disk-2,mp=/mnt/sys_drive,backup=1,size=50G
|
|
mp8: /mnt/felhom-drives,mp=/mnt/felhom-drives
|
|
mp9: /var/lib/felhom-agent/guests/9201/bootstrap,mp=/etc/felhom-bootstrap,ro=1
|
|
[snap1]
|
|
rootfs: local-lvm:vm-9201-disk-9,size=99G
|
|
`
|
|
ov, err := drRestoreOverrides(raw, "local-lvm")
|
|
if err != nil {
|
|
t.Fatalf("drRestoreOverrides: %v", err)
|
|
}
|
|
want := map[string]string{
|
|
"rootfs": "local-lvm:32",
|
|
"mp0": "local-lvm:200,mp=/var/lib/docker,backup=1",
|
|
"mp1": "local-lvm:50,mp=/mnt/sys_drive,backup=1",
|
|
"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
|
|
"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
|
|
}
|
|
if len(ov) != len(want) {
|
|
t.Fatalf("overrides = %+v, want %+v", ov, want)
|
|
}
|
|
for k, v := range want {
|
|
if ov[k] != v {
|
|
t.Errorf("override[%s] = %q, want %q", k, ov[k], v)
|
|
}
|
|
}
|
|
|
|
// an UNKNOWN bind mpN = unknown topology → refuse (never restore a guest missing a mount)
|
|
if _, err := drRestoreOverrides("rootfs: l:d,size=8G\nmp3: /srv/other,mp=/data\n", "local-lvm"); err == nil ||
|
|
!strings.Contains(err.Error(), "unknown bind mountpoint") {
|
|
t.Errorf("unknown bind must refuse, got %v", err)
|
|
}
|
|
// no parseable rootfs → refuse
|
|
if _, err := drRestoreOverrides("hostname: x\n", "local-lvm"); err == nil ||
|
|
!strings.Contains(err.Error(), "rootfs size") {
|
|
t.Errorf("missing rootfs must refuse, got %v", err)
|
|
}
|
|
// a storage mpN without a size → refuse (cannot pass it through)
|
|
if _, err := drRestoreOverrides("rootfs: l:d,size=8G\nmp0: l:d1,mp=/x\n", "local-lvm"); err == nil ||
|
|
!strings.Contains(err.Error(), "no parseable size") {
|
|
t.Errorf("sizeless mpN must refuse, got %v", err)
|
|
}
|
|
}
|
|
|
|
// GL-5: an unreadable archive config fails the DR bring-up BEFORE any restore (no rootfs size =
|
|
// the restore would fail anyway; refuse cleanly, nothing to roll back).
|
|
func TestRunBringUp_DRExtractConfigFailureRefuses(t *testing.T) {
|
|
const vmid = 8206
|
|
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
|
api.extractErr = errors.New("proxmox: GET extractconfig -> HTTP 500: volume not found")
|
|
e, _, _, q := newDREngine(t, api)
|
|
defer q.Close()
|
|
|
|
res := e.RunBringUp(context.Background(), BringUpSpec{
|
|
Mode: ModeDRGuestLoss, Archive: "local:backup/gone.tar.zst", VMID: vmid,
|
|
RestoreStorage: "local-lvm",
|
|
})
|
|
if res.Err == nil || !strings.Contains(res.Err.Error(), "extract archive config") {
|
|
t.Fatalf("extract failure must refuse naming the step, got %+v", res)
|
|
}
|
|
if len(api.restores) != 0 {
|
|
t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores)
|
|
}
|
|
if len(api.destroys) != 0 {
|
|
t.Fatalf("nothing was created — nothing to roll back, got %+v", api.destroys)
|
|
}
|
|
}
|