GL-5: DR bring-up structural bind overrides + 4d real-bind swap

ModeDRGuestLoss now passes restore-time MountOverrides for the two platform-constant structural binds (mp8 parent, mp9 bootstrap) via the shared throwaway-volume format helper - without them a customer-archive restore under the privsep token fails outright ("restoring 'mp8' to bind mount is only possible for root"). New post-restore step 4d swaps the real binds in via the host runner (root pct set, one slot per call), deletes the displaced unusedN volumes (API config PUT; a scoped-token refusal logs the residue loudly instead of widening privileges), and respects the committed/launched rollback envelope. Provision passes nil overrides - byte-identical behavior (regression contract test). Engine grows an optional HostRunner + StateDir seam (DR refuses up front without a runner); selftest bring-up wires the ExecRunner + cleans the scratch mp9 host dir on teardown; proxmox.GuestConfig.Unused() added. 6 new tests incl. C2 mid-swap rollback + C3 older-archive + 403-warn paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-08 08:48:11 +02:00
parent 4c40846769
commit c12b512316
6 changed files with 441 additions and 16 deletions
@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -54,6 +56,32 @@ const DefaultPool = "felhom"
 // volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half).
 const DefaultSysDataMount = "mp1"

+// Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of
+// SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8,
+// /mnt/felhom-drives, same in-guest path) and the bootstrap config bind (mp9,
+// <stateDir>/guests/<vmid>/bootstrap -> /etc/felhom-bootstrap, read-only). These are PLATFORM
+// CONSTANTS - backhalf.go names mp8 "the single permanent parent bind" and mp9's host path is
+// templated only by vmid - which is exactly why DR can synthesize them without reading the lost
+// guest's config. Mirrors provision/backhalf.go's stableParentDir/parentBindSlot/DefaultGuestPath/
+// DefaultMountIndex as literals (the same avoid-the-import-edge rationale backhalf itself uses for
+// its localapi mirror). A customer archive CARRIES these mpN entries, and a pct restore of a
+// bind mount is root@pam-only ("restoring 'mpN' to bind mount is only possible for root") - so a
+// DR restore under the privsep token MUST override them (throwaway volumes) and swap the real
+// binds back post-restore (step 4d).
+const (
+	structuralParentSlot    = "mp8"
+	structuralParentDir     = "/mnt/felhom-drives"
+	structuralBootSlot      = "mp9"
+	structuralBootGuestPath = "/etc/felhom-bootstrap"
+)
+
+// structuralBootHostDir is the mp9 bind's host source for vmid (mirrors the back-half's
+// <stateDir>/guests/<vmid>/bootstrap layout). Joined with "/" explicitly: this is a HOST (Linux)
+// path that flows into pct arguments - filepath.Join would mangle it on a non-Linux test runner.
+func structuralBootHostDir(stateDir string, vmid int) string {
+	return strings.TrimRight(stateDir, "/") + "/guests/" + strconv.Itoa(vmid) + "/bootstrap"
+}
+
 // configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
 // tests can shrink it (the production value gives PVE time to release its async config lock).
 const configLockMaxAttempts = 5
@@ -77,15 +105,15 @@ type GuestMount struct {
 // BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10
 // hub desired-state); this job does not decide WHAT to provision.
 type BringUpSpec struct {
-	Mode           BringUpMode   // provision | dr_guest_loss
-	Archive        string        // source volid (golden for provision; customer backup for DR)
-	VMID           int           // caller-provided target VMID (NOT the restore-test band / 9999)
-	RestoreStorage string        // rootfs target storage
-	Hostname       string        // hostname to set (provision); ignored for DR (continuity)
-	Pool           string        // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token
-	Cores          int           // 0 = leave as restored
-	MemoryMB       int           // 0 = leave as restored
-	RootfsGrowGB   int           // optional grow-only rootfs resize (0 = skip)
+	Mode           BringUpMode // provision | dr_guest_loss
+	Archive        string      // source volid (golden for provision; customer backup for DR)
+	VMID           int         // caller-provided target VMID (NOT the restore-test band / 9999)
+	RestoreStorage string      // rootfs target storage
+	Hostname       string      // hostname to set (provision); ignored for DR (continuity)
+	Pool           string      // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token
+	Cores          int         // 0 = leave as restored
+	MemoryMB       int         // 0 = leave as restored
+	RootfsGrowGB   int         // optional grow-only rootfs resize (0 = skip)
 	// DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the
 	// per-customer target. The golden ships a small data volume with the baked images; provision
 	// grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that
@@ -101,8 +129,8 @@ type BringUpSpec struct {
 	// SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1").
 	SysDataMount string
 	Mounts       []GuestMount  // additive mpN mounts (slice 7 may pass empty/test)
-	KeepMAC        bool          // DR knob: keep the archived MAC (true) unless a source may be live
-	BootTimeout    time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
+	KeepMAC      bool          // DR knob: keep the archived MAC (true) unless a source may be live
+	BootTimeout  time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
 }

 // BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface
@@ -163,6 +191,12 @@ func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult
 		res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode)
 		return res
 	}
+	// DR needs the host runner: the structural-bind swap (4d) is a root pct op the API token
+	// cannot perform. Refuse up front rather than fail after a restore (GL-5).
+	if spec.Mode == ModeDRGuestLoss && e.hostRun == nil {
+		res.Err = fmt.Errorf("reconcile: dr bring-up needs a host runner (the mp8/mp9 structural-bind swap is a root pct op) — wire EngineOptions.HostRunner")
+		return res
+	}

 	ch := e.queue.Submit(spec.VMID, func() error {
 		e.runBringUp(ctx, spec, &res)
@@ -219,8 +253,26 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
 	}()

 	// 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike).
+	//    DR (GL-5): the customer archive carries the two STRUCTURAL host-bind mountpoints (mp8
+	//    parent bind + mp9 bootstrap bind) that a restore under the privsep token cannot recreate
+	//    — without overrides the whole restore FAILS ("restoring 'mp8' to bind mount is only
+	//    possible for root"). Synthesize throwaway-volume overrides for the two known-constant mpN
+	//    via the shared format helper (bindMountOverrides' is-a-bind filter is for reading real
+	//    configs, which DR by definition cannot do — the guest is gone); step 4d swaps the real
+	//    binds back post-restore. An archive that LACKS one of them (older backup) is fine: the
+	//    override simply creates that mpN at restore and 4d normalizes it — the end state is
+	//    identical (C3). Provision stays override-free (nil): the golden has no mp8/mp9 (the
+	//    back-half adds them post-bring-up) — that asymmetry is the whole GL-5 bug.
+	var overrides map[string]string
+	if spec.Mode == ModeDRGuestLoss {
+		overrides = map[string]string{
+			structuralParentSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralParentDir),
+			structuralBootSlot:   throwawayVolumeOverride(spec.RestoreStorage, structuralBootGuestPath),
+		}
+	}
 	upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
 		VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
+		MountOverrides: overrides,
 	})
 	if err != nil {
 		// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
@@ -307,6 +359,20 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
 		}
 	}

+	// 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the
+	//     restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR
+	//     guest carries no unusedN residue. Bind-mount pct sets are root@pam-only — the swap goes
+	//     through the host runner, never the API. Runs BEFORE start so the first boot already sees
+	//     the real binds (the golden's baked bootstrap unit + the drives parent). A failure here is
+	//     surfaced with the exact mpN state and rolls back per the envelope (committed is still
+	//     false) — never a silent half-wired success (C2).
+	if spec.Mode == ModeDRGuestLoss {
+		if err := e.swapStructuralBinds(ctx, spec, res); err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up structural-bind swap: %w", err)
+			return
+		}
+	}
+
 	// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
 	if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
 		res.AssignedMAC = net0MAC(cfg2)
@@ -367,6 +433,58 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
 	e.append(withState(base, OpSucceeded))
 }

+// swapStructuralBinds is bring-up step 4d (DR only): set mp8/mp9 to the real host binds via the
+// root runner (one slot per call so an error names the exact mpN that failed — C2), then delete
+// the displaced throwaway volumes PVE parked as unusedN. The mp9 host dir is created first (pct
+// validates the bind source; agent-owned path, plain MkdirAll like the back-half's own bootstrap
+// dir) — idempotent: on a same-host guest-loss the dir usually still exists WITH bootstrap.json,
+// which the swap must not touch. The unusedN delete goes through the API config PUT
+// (VM.Config.Disk + Datastore.Allocate cover it); if the scoped token refuses, the residue is
+// logged LOUDLY + surfaced as a result warning and the bring-up continues — a correctly-wired
+// guest with a stray volume beats a rollback, and privileges are never widened silently.
+func (e *Engine) swapStructuralBinds(ctx context.Context, spec BringUpSpec, res *BringUpResult) error {
+	bootDir := structuralBootHostDir(e.stateDir, spec.VMID)
+	if err := os.MkdirAll(bootDir, 0o700); err != nil {
+		return fmt.Errorf("mp9 bootstrap host dir %s: %w", bootDir, err)
+	}
+	if _, stderr, err := e.hostRun.Run(ctx, "mkdir", "-p", structuralParentDir); err != nil {
+		return fmt.Errorf("parent dir %s: %w: %s", structuralParentDir, err, stderr)
+	}
+	parentSpec := structuralParentDir + ",mp=" + structuralParentDir
+	if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralParentSlot, parentSpec); err != nil {
+		return fmt.Errorf("set %s (parent bind; neither bind landed): %w: %s", structuralParentSlot, err, stderr)
+	}
+	bootSpec := bootDir + ",mp=" + structuralBootGuestPath + ",ro=1"
+	if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralBootSlot, bootSpec); err != nil {
+		return fmt.Errorf("set %s (bootstrap bind; %s already landed): %w: %s", structuralBootSlot, structuralParentSlot, err, stderr)
+	}
+	e.logger.Info("bring-up: structural binds swapped in", "vmid", spec.VMID,
+		structuralParentSlot, parentSpec, structuralBootSlot, bootSpec)
+
+	// The displaced throwaway volumes now sit as unusedN — read the config and delete them.
+	cfg, err := e.api.GuestConfig(ctx, spec.VMID)
+	if err != nil {
+		return fmt.Errorf("read config after bind swap: %w", err)
+	}
+	var unused []string
+	for k := range cfg.Unused() {
+		unused = append(unused, k)
+	}
+	if len(unused) == 0 {
+		return nil
+	}
+	sort.Strings(unused)
+	if err := e.setConfigWithLockRetry(ctx, spec.VMID, map[string]string{"delete": strings.Join(unused, ",")}); err != nil {
+		e.logger.Error("bring-up: could not delete displaced throwaway volumes (guest is correctly wired; residue remains)",
+			"vmid", spec.VMID, "unused", unused, "err", err)
+		res.StartWarnings = append(res.StartWarnings,
+			fmt.Sprintf("structural-bind swap: displaced volumes not deleted (%s): %v", strings.Join(unused, ","), err))
+		return nil
+	}
+	e.logger.Info("bring-up: displaced throwaway volumes deleted", "vmid", spec.VMID, "unused", unused)
+	return nil
+}
+
 // rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
 // provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
 // was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not