diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 6ddbfb7..46526d3 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -20,6 +20,7 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -1426,8 +1427,16 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log } } gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger) + // GL-5: DR bring-up swaps the structural binds (mp8/mp9) via root pct ops — wire the same + // Runner shape the provision back-half uses. Provision mode never touches it. + rMode := proxmox.RunnerMode(cfg.Privileged.Mode) + if rMode == "" { + rMode = proxmox.RunnerSudo + } + hostRunner := &proxmox.ExecRunner{Mode: rMode, SudoPath: cfg.Privileged.SudoPath} engine := reconcile.NewEngine(reconcile.EngineOptions{ API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger, + HostRunner: hostRunner, }) fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid) @@ -1475,6 +1484,14 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log return 1 } } + // GL-5: the DR bind swap created /guests//bootstrap for this SCRATCH vmid — + // remove the agent-owned per-guest dir with the guest (never a real drive's bind source; a + // scratch vmid has no other state). Best-effort. + if bmode == reconcile.ModeDRGuestLoss { + if err := os.RemoveAll(filepath.Join("/var/lib/felhom-agent/guests", strconv.Itoa(vmid))); err != nil { + fmt.Fprintf(os.Stderr, " [WARN] scratch mp9 host dir cleanup: %v\n", err) + } + } fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid) return 0 } diff --git a/internal/proxmox/types.go b/internal/proxmox/types.go index 75bc26b..9ec7d2a 100644 --- a/internal/proxmox/types.go +++ b/internal/proxmox/types.go @@ -133,6 +133,13 @@ func (g *GuestConfig) Nets() map[string]string { return g.prefixed("net") } +// Unused returns the unusedN entries from Extra — the displaced volumes PVE parks in the config +// when a mountpoint is overridden/replaced (e.g. the DR bring-up's structural-bind swap replacing +// its restore-time throwaway volumes). The caller deletes these so a kept guest carries no residue. +func (g *GuestConfig) Unused() map[string]string { + return g.prefixed("unused") +} + // Lock returns the guest's current lock ("backup", "snapshot-delete", "migrate", …) from the config, // or "" when unlocked. An interrupted vzdump leaves a "backup" or "snapshot-delete" lock — the signal // the startup stale-lock recovery (F2-b) keys on. diff --git a/internal/reconcile/bringup.go b/internal/reconcile/bringup.go index 92d2224..cba529d 100644 --- a/internal/reconcile/bringup.go +++ b/internal/reconcile/bringup.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "os" + "sort" "strconv" "strings" "time" @@ -54,6 +56,32 @@ const DefaultPool = "felhom" // volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half). const DefaultSysDataMount = "mp1" +// Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of +// SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8, +// /mnt/felhom-drives, same in-guest path) and the bootstrap config bind (mp9, +// /guests//bootstrap -> /etc/felhom-bootstrap, read-only). These are PLATFORM +// CONSTANTS - backhalf.go names mp8 "the single permanent parent bind" and mp9's host path is +// templated only by vmid - which is exactly why DR can synthesize them without reading the lost +// guest's config. Mirrors provision/backhalf.go's stableParentDir/parentBindSlot/DefaultGuestPath/ +// DefaultMountIndex as literals (the same avoid-the-import-edge rationale backhalf itself uses for +// its localapi mirror). A customer archive CARRIES these mpN entries, and a pct restore of a +// bind mount is root@pam-only ("restoring 'mpN' to bind mount is only possible for root") - so a +// DR restore under the privsep token MUST override them (throwaway volumes) and swap the real +// binds back post-restore (step 4d). +const ( + structuralParentSlot = "mp8" + structuralParentDir = "/mnt/felhom-drives" + structuralBootSlot = "mp9" + structuralBootGuestPath = "/etc/felhom-bootstrap" +) + +// structuralBootHostDir is the mp9 bind's host source for vmid (mirrors the back-half's +// /guests//bootstrap layout). Joined with "/" explicitly: this is a HOST (Linux) +// path that flows into pct arguments - filepath.Join would mangle it on a non-Linux test runner. +func structuralBootHostDir(stateDir string, vmid int) string { + return strings.TrimRight(stateDir, "/") + "/guests/" + strconv.Itoa(vmid) + "/bootstrap" +} + // configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so // tests can shrink it (the production value gives PVE time to release its async config lock). const configLockMaxAttempts = 5 @@ -77,15 +105,15 @@ type GuestMount struct { // BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10 // hub desired-state); this job does not decide WHAT to provision. type BringUpSpec struct { - Mode BringUpMode // provision | dr_guest_loss - Archive string // source volid (golden for provision; customer backup for DR) - VMID int // caller-provided target VMID (NOT the restore-test band / 9999) - RestoreStorage string // rootfs target storage - Hostname string // hostname to set (provision); ignored for DR (continuity) - Pool string // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token - Cores int // 0 = leave as restored - MemoryMB int // 0 = leave as restored - RootfsGrowGB int // optional grow-only rootfs resize (0 = skip) + Mode BringUpMode // provision | dr_guest_loss + Archive string // source volid (golden for provision; customer backup for DR) + VMID int // caller-provided target VMID (NOT the restore-test band / 9999) + RestoreStorage string // rootfs target storage + Hostname string // hostname to set (provision); ignored for DR (continuity) + Pool string // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token + Cores int // 0 = leave as restored + MemoryMB int // 0 = leave as restored + RootfsGrowGB int // optional grow-only rootfs resize (0 = skip) // DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the // per-customer target. The golden ships a small data volume with the baked images; provision // grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that @@ -101,8 +129,8 @@ type BringUpSpec struct { // SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1"). SysDataMount string Mounts []GuestMount // additive mpN mounts (slice 7 may pass empty/test) - KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live - BootTimeout time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait + KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live + BootTimeout time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait } // BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface @@ -163,6 +191,12 @@ func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode) return res } + // DR needs the host runner: the structural-bind swap (4d) is a root pct op the API token + // cannot perform. Refuse up front rather than fail after a restore (GL-5). + if spec.Mode == ModeDRGuestLoss && e.hostRun == nil { + res.Err = fmt.Errorf("reconcile: dr bring-up needs a host runner (the mp8/mp9 structural-bind swap is a root pct op) — wire EngineOptions.HostRunner") + return res + } ch := e.queue.Submit(spec.VMID, func() error { e.runBringUp(ctx, spec, &res) @@ -219,8 +253,26 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR }() // 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike). + // DR (GL-5): the customer archive carries the two STRUCTURAL host-bind mountpoints (mp8 + // parent bind + mp9 bootstrap bind) that a restore under the privsep token cannot recreate + // — without overrides the whole restore FAILS ("restoring 'mp8' to bind mount is only + // possible for root"). Synthesize throwaway-volume overrides for the two known-constant mpN + // via the shared format helper (bindMountOverrides' is-a-bind filter is for reading real + // configs, which DR by definition cannot do — the guest is gone); step 4d swaps the real + // binds back post-restore. An archive that LACKS one of them (older backup) is fine: the + // override simply creates that mpN at restore and 4d normalizes it — the end state is + // identical (C3). Provision stays override-free (nil): the golden has no mp8/mp9 (the + // back-half adds them post-bring-up) — that asymmetry is the whole GL-5 bug. + var overrides map[string]string + if spec.Mode == ModeDRGuestLoss { + overrides = map[string]string{ + structuralParentSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralParentDir), + structuralBootSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralBootGuestPath), + } + } upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{ VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool, + MountOverrides: overrides, }) if err != nil { // No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy. @@ -307,6 +359,20 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR } } + // 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the + // restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR + // guest carries no unusedN residue. Bind-mount pct sets are root@pam-only — the swap goes + // through the host runner, never the API. Runs BEFORE start so the first boot already sees + // the real binds (the golden's baked bootstrap unit + the drives parent). A failure here is + // surfaced with the exact mpN state and rolls back per the envelope (committed is still + // false) — never a silent half-wired success (C2). + if spec.Mode == ModeDRGuestLoss { + if err := e.swapStructuralBinds(ctx, spec, res); err != nil { + res.Err = fmt.Errorf("reconcile: bring-up structural-bind swap: %w", err) + return + } + } + // Capture the post-reset MAC for the result (fresh for provision; archived for DR keep). if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil { res.AssignedMAC = net0MAC(cfg2) @@ -367,6 +433,58 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR e.append(withState(base, OpSucceeded)) } +// swapStructuralBinds is bring-up step 4d (DR only): set mp8/mp9 to the real host binds via the +// root runner (one slot per call so an error names the exact mpN that failed — C2), then delete +// the displaced throwaway volumes PVE parked as unusedN. The mp9 host dir is created first (pct +// validates the bind source; agent-owned path, plain MkdirAll like the back-half's own bootstrap +// dir) — idempotent: on a same-host guest-loss the dir usually still exists WITH bootstrap.json, +// which the swap must not touch. The unusedN delete goes through the API config PUT +// (VM.Config.Disk + Datastore.Allocate cover it); if the scoped token refuses, the residue is +// logged LOUDLY + surfaced as a result warning and the bring-up continues — a correctly-wired +// guest with a stray volume beats a rollback, and privileges are never widened silently. +func (e *Engine) swapStructuralBinds(ctx context.Context, spec BringUpSpec, res *BringUpResult) error { + bootDir := structuralBootHostDir(e.stateDir, spec.VMID) + if err := os.MkdirAll(bootDir, 0o700); err != nil { + return fmt.Errorf("mp9 bootstrap host dir %s: %w", bootDir, err) + } + if _, stderr, err := e.hostRun.Run(ctx, "mkdir", "-p", structuralParentDir); err != nil { + return fmt.Errorf("parent dir %s: %w: %s", structuralParentDir, err, stderr) + } + parentSpec := structuralParentDir + ",mp=" + structuralParentDir + if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralParentSlot, parentSpec); err != nil { + return fmt.Errorf("set %s (parent bind; neither bind landed): %w: %s", structuralParentSlot, err, stderr) + } + bootSpec := bootDir + ",mp=" + structuralBootGuestPath + ",ro=1" + if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralBootSlot, bootSpec); err != nil { + return fmt.Errorf("set %s (bootstrap bind; %s already landed): %w: %s", structuralBootSlot, structuralParentSlot, err, stderr) + } + e.logger.Info("bring-up: structural binds swapped in", "vmid", spec.VMID, + structuralParentSlot, parentSpec, structuralBootSlot, bootSpec) + + // The displaced throwaway volumes now sit as unusedN — read the config and delete them. + cfg, err := e.api.GuestConfig(ctx, spec.VMID) + if err != nil { + return fmt.Errorf("read config after bind swap: %w", err) + } + var unused []string + for k := range cfg.Unused() { + unused = append(unused, k) + } + if len(unused) == 0 { + return nil + } + sort.Strings(unused) + if err := e.setConfigWithLockRetry(ctx, spec.VMID, map[string]string{"delete": strings.Join(unused, ",")}); err != nil { + e.logger.Error("bring-up: could not delete displaced throwaway volumes (guest is correctly wired; residue remains)", + "vmid", spec.VMID, "unused", unused, "err", err) + res.StartWarnings = append(res.StartWarnings, + fmt.Sprintf("structural-bind swap: displaced volumes not deleted (%s): %v", strings.Join(unused, ","), err)) + return nil + } + e.logger.Info("bring-up: displaced throwaway volumes deleted", "vmid", spec.VMID, "unused", unused) + return nil +} + // rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated // provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST // was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not diff --git a/internal/reconcile/bringup_test.go b/internal/reconcile/bringup_test.go index a0decdc..3812f53 100644 --- a/internal/reconcile/bringup_test.go +++ b/internal/reconcile/bringup_test.go @@ -2,14 +2,60 @@ package reconcile import ( "context" + "encoding/json" "errors" + "io" + "os" + "path/filepath" "strings" + "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) +// fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a +// substring of the joined command — makes that one command fail (the C2 mid-swap failure driver). +type fakeRunner struct { + mu sync.Mutex + cmds []string + failOn string +} + +func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { + cmd := name + " " + strings.Join(args, " ") + f.mu.Lock() + f.cmds = append(f.cmds, cmd) + f.mu.Unlock() + if f.failOn != "" && strings.Contains(cmd, f.failOn) { + return nil, []byte("boom"), errors.New("exit status 1") + } + return nil, nil, nil +} + +func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) { + return f.Run(ctx, name, args...) +} + +// newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an +// isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it). +func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) { + t.Helper() + jp := filepath.Join(t.TempDir(), "journal.log") + j, err := OpenJournal(jp) + if err != nil { + t.Fatalf("OpenJournal: %v", err) + } + t.Cleanup(func() { j.Close() }) + q := NewQueue() + t.Cleanup(q.Close) + fr := &fakeRunner{} + sd := t.TempDir() + e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd}) + return e, fr, sd, q +} + // setParamsFor returns the params of the (last) SetConfig call against vmid, or nil. func setParamsFor(api *fakeAPI, vmid int) map[string]string { var out map[string]string @@ -355,7 +401,7 @@ func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) { func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) { const vmid = 8001 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} - e, _, q := newEngine(t, api, EmptyProvider{}) + e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ @@ -393,7 +439,7 @@ func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) { func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) { const vmid = 8002 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} - e, _, q := newEngine(t, api, EmptyProvider{}) + e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ @@ -649,7 +695,7 @@ func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall { func TestRunBringUp_ReassertsPoolMembership(t *testing.T) { const vmid = 8100 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} - e, _, q := newEngine(t, api, EmptyProvider{}) + e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ @@ -692,7 +738,7 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) { cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"), } - e, _, q := newEngine(t, api, EmptyProvider{}) + e, _, _, q := newDREngine(t, api) defer q.Close() res := e.RunBringUp(context.Background(), BringUpSpec{ @@ -717,3 +763,211 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) { t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings) } } + +// ── GL-5: DR structural bind overrides + 4d swap ───────────────────────────────────────────────── + +// GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural +// binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via +// the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes. +// COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts +// (run→fail→revert, recorded in the REPORT). +func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) { + const vmid = 8200 + cfg := scratchCfg() + // after the swap PVE parks the two displaced throwaway volumes as unusedN + cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`) + cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`) + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} + e, fr, sd, q := newDREngine(t, api) + defer q.Close() + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", KeepMAC: true, + }) + if res.Err != nil || !res.Pass { + t.Fatalf("dr bring-up must pass, got %+v", res) + } + if len(api.restores) != 1 { + t.Fatalf("expected one restore, got %+v", api.restores) + } + ov := api.restores[0].MountOverrides + want := map[string]string{ + "mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0", + "mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0", + } + if len(ov) != 2 || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] { + t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want) + } + // 4d: mp9 host dir created under the engine state dir … + bootDir := structuralBootHostDir(sd, vmid) + if st, err := os.Stat(bootDir); err != nil || !st.IsDir() { + t.Fatalf("mp9 bootstrap host dir not created: %v", err) + } + // … and the REAL binds set via the root runner, one slot per call, exact backhalf values. + wantCmds := []string{ + "mkdir -p /mnt/felhom-drives", + "pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives", + "pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1", + } + if len(fr.cmds) != len(wantCmds) { + t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds) + } + for i := range wantCmds { + if fr.cmds[i] != wantCmds[i] { + t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i]) + } + } + // displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue. + del := "" + for _, s := range api.sets { + if s.vmid == vmid && s.params["delete"] != "" { + del = s.params["delete"] + } + } + if del != "unused0,unused1" { + t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets) + } +} + +// GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never +// touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the +// override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT). +func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) { + const vmid = 8201 + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} + e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED + defer q.Close() + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", Hostname: "h", + }) + if res.Err != nil || !res.Pass { + t.Fatalf("provision must pass, got %+v", res) + } + if api.restores[0].MountOverrides != nil { + t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides) + } + if len(fr.cmds) != 0 { + t.Fatalf("provision must not touch the host runner, got %v", fr.cmds) + } +} + +// GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state +// named and compensating-rolls-back the guest — never a silent half-wired success. +func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) { + const vmid = 8202 + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} + e, fr, _, q := newDREngine(t, api) + defer q.Close() + fr.failOn = "-mp9" + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", KeepMAC: true, + }) + if res.Err == nil || res.Pass { + t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res) + } + if !strings.Contains(res.Err.Error(), "structural-bind swap") || + !strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") { + t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err) + } + if len(api.destroys) != 1 || api.destroys[0] != vmid { + t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys) + } + if len(api.starts) != 0 { + t.Fatalf("a half-wired guest must never be started, got %+v", api.starts) + } +} + +// GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived, +// so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the +// end state is identical, with no unusedN residue (here only ONE displaced volume shows up). +func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) { + const vmid = 8203 + cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape + cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`) + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} + e, fr, _, q := newDREngine(t, api) + defer q.Close() + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", KeepMAC: true, + }) + if res.Err != nil || !res.Pass { + t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res) + } + ov := api.restores[0].MountOverrides + if len(ov) != 2 || ov["mp8"] == "" || ov["mp9"] == "" { + t.Fatalf("overrides are constants — both mpN must be named regardless of the archive: %+v", ov) + } + if len(fr.cmds) != 3 { + t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds) + } + del := "" + for _, s := range api.sets { + if s.vmid == vmid && s.params["delete"] != "" { + del = s.params["delete"] + } + } + if del != "unused0" { + t.Fatalf("the one displaced volume must be deleted: delete=%q", del) + } +} + +// GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because +// the structural-bind swap is a root pct op the API token cannot perform. +func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) { + const vmid = 8204 + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} + e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner + defer q.Close() + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", + }) + if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") { + t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res) + } + if len(api.restores) != 0 { + t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores) + } +} + +// GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired +// guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently). +func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) { + const vmid = 8205 + cfg := scratchCfg() + cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`) + api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}} + api.setFunc = func(_ int, params map[string]string) (string, error) { + if params["delete"] != "" { + return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed") + } + return "", nil + } + e, _, _, q := newDREngine(t, api) + defer q.Close() + + res := e.RunBringUp(context.Background(), BringUpSpec{ + Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid, + RestoreStorage: "local-lvm", KeepMAC: true, + }) + if res.Err != nil || !res.Pass { + t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res) + } + found := false + for _, w := range res.StartWarnings { + if strings.Contains(w, "displaced volumes not deleted") { + found = true + } + } + if !found { + t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings) + } +} diff --git a/internal/reconcile/engine.go b/internal/reconcile/engine.go index 9885c45..811acfd 100644 --- a/internal/reconcile/engine.go +++ b/internal/reconcile/engine.go @@ -32,6 +32,15 @@ type Engine struct { hostID string logger *slog.Logger + // hostRun executes host-root commands for the DR structural-bind swap (bring-up 4d): a + // bind-mount `pct set` is root@pam-only, so it cannot go through the API token (the same + // constraint the provision back-half documents). nil on API-only engines — RunBringUp + // refuses ModeDRGuestLoss then (GL-5). + hostRun proxmox.Runner + // stateDir is the agent state dir the mp9 bootstrap host dir lives under + // (/guests//bootstrap); "" → /var/lib/felhom-agent (mirrors provision.NewBackHalf). + stateDir string + opSeq uint64 // atomic; makes each op id unique per attempt } @@ -46,6 +55,11 @@ type EngineOptions struct { Gate *Gate HostID string Logger *slog.Logger + // HostRunner enables the DR structural-bind swap (bring-up 4d, root pct ops). Optional — + // engines that never run ModeDRGuestLoss may leave it nil. + HostRunner proxmox.Runner + // StateDir is the agent state dir ("" → /var/lib/felhom-agent); only the 4d swap reads it. + StateDir string } // NewEngine builds an Engine. The Queue is shared (the single §10 choke point); the @@ -69,6 +83,10 @@ func NewEngine(opts EngineOptions) *Engine { // the common slice-4 daemon state (no signers pinned, no desired state). gate = NewGate(nil, opts.HostID, nil, logger) } + stateDir := opts.StateDir + if stateDir == "" { + stateDir = "/var/lib/felhom-agent" + } return &Engine{ api: opts.API, queue: opts.Queue, @@ -78,6 +96,8 @@ func NewEngine(opts EngineOptions) *Engine { gate: gate, hostID: opts.HostID, logger: logger, + hostRun: opts.HostRunner, + stateDir: stateDir, } } diff --git a/internal/reconcile/restoretest.go b/internal/reconcile/restoretest.go index dc5e930..5ca724d 100644 --- a/internal/reconcile/restoretest.go +++ b/internal/reconcile/restoretest.go @@ -342,7 +342,7 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string if mp == "" { mp = volPart // fall back to the host path if no explicit in-guest mp= } - out[key] = fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, mp) + out[key] = throwawayVolumeOverride(restoreStorage, mp) } if len(out) == 0 { return nil @@ -350,6 +350,15 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string return out } +// throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G +// volume on restoreStorage at the given in-guest path, excluded from backup. Used by the +// restore-test's source-config-derived overrides (above) and by the DR bring-up's synthesized +// structural-bind overrides (GL-5, bringup.go — its mpN are platform constants, so it skips this +// file's is-a-bind filtering and calls the format directly). +func throwawayVolumeOverride(restoreStorage, guestPath string) string { + return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath) +} + // mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent). func mountPathOf(opts string) string { for _, kv := range strings.Split(opts, ",") {