GL-5: DR bring-up structural bind overrides + 4d real-bind swap

ModeDRGuestLoss now passes restore-time MountOverrides for the two
platform-constant structural binds (mp8 parent, mp9 bootstrap) via the
shared throwaway-volume format helper - without them a customer-archive
restore under the privsep token fails outright ("restoring 'mp8' to bind
mount is only possible for root"). New post-restore step 4d swaps the real
binds in via the host runner (root pct set, one slot per call), deletes the
displaced unusedN volumes (API config PUT; a scoped-token refusal logs the
residue loudly instead of widening privileges), and respects the
committed/launched rollback envelope. Provision passes nil overrides -
byte-identical behavior (regression contract test).

Engine grows an optional HostRunner + StateDir seam (DR refuses up front
without a runner); selftest bring-up wires the ExecRunner + cleans the
scratch mp9 host dir on teardown; proxmox.GuestConfig.Unused() added.
6 new tests incl. C2 mid-swap rollback + C3 older-archive + 403-warn paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-08 08:48:11 +02:00
parent 4c40846769
commit c12b512316
6 changed files with 441 additions and 16 deletions
+17
View File
@@ -20,6 +20,7 @@ import (
"os" "os"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"syscall" "syscall"
"time" "time"
@@ -1426,8 +1427,16 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
} }
} }
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger) gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
// GL-5: DR bring-up swaps the structural binds (mp8/mp9) via root pct ops — wire the same
// Runner shape the provision back-half uses. Provision mode never touches it.
rMode := proxmox.RunnerMode(cfg.Privileged.Mode)
if rMode == "" {
rMode = proxmox.RunnerSudo
}
hostRunner := &proxmox.ExecRunner{Mode: rMode, SudoPath: cfg.Privileged.SudoPath}
engine := reconcile.NewEngine(reconcile.EngineOptions{ engine := reconcile.NewEngine(reconcile.EngineOptions{
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger, API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
HostRunner: hostRunner,
}) })
fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid) fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
@@ -1475,6 +1484,14 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
return 1 return 1
} }
} }
// GL-5: the DR bind swap created <stateDir>/guests/<vmid>/bootstrap for this SCRATCH vmid —
// remove the agent-owned per-guest dir with the guest (never a real drive's bind source; a
// scratch vmid has no other state). Best-effort.
if bmode == reconcile.ModeDRGuestLoss {
if err := os.RemoveAll(filepath.Join("/var/lib/felhom-agent/guests", strconv.Itoa(vmid))); err != nil {
fmt.Fprintf(os.Stderr, " [WARN] scratch mp9 host dir cleanup: %v\n", err)
}
}
fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid) fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
return 0 return 0
} }
+7
View File
@@ -133,6 +133,13 @@ func (g *GuestConfig) Nets() map[string]string {
return g.prefixed("net") return g.prefixed("net")
} }
// Unused returns the unusedN entries from Extra — the displaced volumes PVE parks in the config
// when a mountpoint is overridden/replaced (e.g. the DR bring-up's structural-bind swap replacing
// its restore-time throwaway volumes). The caller deletes these so a kept guest carries no residue.
func (g *GuestConfig) Unused() map[string]string {
return g.prefixed("unused")
}
// Lock returns the guest's current lock ("backup", "snapshot-delete", "migrate", …) from the config, // Lock returns the guest's current lock ("backup", "snapshot-delete", "migrate", …) from the config,
// or "" when unlocked. An interrupted vzdump leaves a "backup" or "snapshot-delete" lock — the signal // or "" when unlocked. An interrupted vzdump leaves a "backup" or "snapshot-delete" lock — the signal
// the startup stale-lock recovery (F2-b) keys on. // the startup stale-lock recovery (F2-b) keys on.
+129 -11
View File
@@ -4,6 +4,8 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"os"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -54,6 +56,32 @@ const DefaultPool = "felhom"
// volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half). // volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half).
const DefaultSysDataMount = "mp1" const DefaultSysDataMount = "mp1"
// Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of
// SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8,
// /mnt/felhom-drives, same in-guest path) and the bootstrap config bind (mp9,
// <stateDir>/guests/<vmid>/bootstrap -> /etc/felhom-bootstrap, read-only). These are PLATFORM
// CONSTANTS - backhalf.go names mp8 "the single permanent parent bind" and mp9's host path is
// templated only by vmid - which is exactly why DR can synthesize them without reading the lost
// guest's config. Mirrors provision/backhalf.go's stableParentDir/parentBindSlot/DefaultGuestPath/
// DefaultMountIndex as literals (the same avoid-the-import-edge rationale backhalf itself uses for
// its localapi mirror). A customer archive CARRIES these mpN entries, and a pct restore of a
// bind mount is root@pam-only ("restoring 'mpN' to bind mount is only possible for root") - so a
// DR restore under the privsep token MUST override them (throwaway volumes) and swap the real
// binds back post-restore (step 4d).
const (
structuralParentSlot = "mp8"
structuralParentDir = "/mnt/felhom-drives"
structuralBootSlot = "mp9"
structuralBootGuestPath = "/etc/felhom-bootstrap"
)
// structuralBootHostDir is the mp9 bind's host source for vmid (mirrors the back-half's
// <stateDir>/guests/<vmid>/bootstrap layout). Joined with "/" explicitly: this is a HOST (Linux)
// path that flows into pct arguments - filepath.Join would mangle it on a non-Linux test runner.
func structuralBootHostDir(stateDir string, vmid int) string {
return strings.TrimRight(stateDir, "/") + "/guests/" + strconv.Itoa(vmid) + "/bootstrap"
}
// configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so // configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
// tests can shrink it (the production value gives PVE time to release its async config lock). // tests can shrink it (the production value gives PVE time to release its async config lock).
const configLockMaxAttempts = 5 const configLockMaxAttempts = 5
@@ -77,15 +105,15 @@ type GuestMount struct {
// BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10 // BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10
// hub desired-state); this job does not decide WHAT to provision. // hub desired-state); this job does not decide WHAT to provision.
type BringUpSpec struct { type BringUpSpec struct {
Mode BringUpMode // provision | dr_guest_loss Mode BringUpMode // provision | dr_guest_loss
Archive string // source volid (golden for provision; customer backup for DR) Archive string // source volid (golden for provision; customer backup for DR)
VMID int // caller-provided target VMID (NOT the restore-test band / 9999) VMID int // caller-provided target VMID (NOT the restore-test band / 9999)
RestoreStorage string // rootfs target storage RestoreStorage string // rootfs target storage
Hostname string // hostname to set (provision); ignored for DR (continuity) Hostname string // hostname to set (provision); ignored for DR (continuity)
Pool string // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token Pool string // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token
Cores int // 0 = leave as restored Cores int // 0 = leave as restored
MemoryMB int // 0 = leave as restored MemoryMB int // 0 = leave as restored
RootfsGrowGB int // optional grow-only rootfs resize (0 = skip) RootfsGrowGB int // optional grow-only rootfs resize (0 = skip)
// DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the // DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the
// per-customer target. The golden ships a small data volume with the baked images; provision // per-customer target. The golden ships a small data volume with the baked images; provision
// grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that // grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that
@@ -101,8 +129,8 @@ type BringUpSpec struct {
// SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1"). // SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1").
SysDataMount string SysDataMount string
Mounts []GuestMount // additive mpN mounts (slice 7 may pass empty/test) Mounts []GuestMount // additive mpN mounts (slice 7 may pass empty/test)
KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live
BootTimeout time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait BootTimeout time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
} }
// BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface // BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface
@@ -163,6 +191,12 @@ func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult
res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode) res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode)
return res return res
} }
// DR needs the host runner: the structural-bind swap (4d) is a root pct op the API token
// cannot perform. Refuse up front rather than fail after a restore (GL-5).
if spec.Mode == ModeDRGuestLoss && e.hostRun == nil {
res.Err = fmt.Errorf("reconcile: dr bring-up needs a host runner (the mp8/mp9 structural-bind swap is a root pct op) — wire EngineOptions.HostRunner")
return res
}
ch := e.queue.Submit(spec.VMID, func() error { ch := e.queue.Submit(spec.VMID, func() error {
e.runBringUp(ctx, spec, &res) e.runBringUp(ctx, spec, &res)
@@ -219,8 +253,26 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
}() }()
// 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike). // 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike).
// DR (GL-5): the customer archive carries the two STRUCTURAL host-bind mountpoints (mp8
// parent bind + mp9 bootstrap bind) that a restore under the privsep token cannot recreate
// — without overrides the whole restore FAILS ("restoring 'mp8' to bind mount is only
// possible for root"). Synthesize throwaway-volume overrides for the two known-constant mpN
// via the shared format helper (bindMountOverrides' is-a-bind filter is for reading real
// configs, which DR by definition cannot do — the guest is gone); step 4d swaps the real
// binds back post-restore. An archive that LACKS one of them (older backup) is fine: the
// override simply creates that mpN at restore and 4d normalizes it — the end state is
// identical (C3). Provision stays override-free (nil): the golden has no mp8/mp9 (the
// back-half adds them post-bring-up) — that asymmetry is the whole GL-5 bug.
var overrides map[string]string
if spec.Mode == ModeDRGuestLoss {
overrides = map[string]string{
structuralParentSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralParentDir),
structuralBootSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralBootGuestPath),
}
}
upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{ upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool, VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
MountOverrides: overrides,
}) })
if err != nil { if err != nil {
// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy. // No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
@@ -307,6 +359,20 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
} }
} }
// 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the
// restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR
// guest carries no unusedN residue. Bind-mount pct sets are root@pam-only — the swap goes
// through the host runner, never the API. Runs BEFORE start so the first boot already sees
// the real binds (the golden's baked bootstrap unit + the drives parent). A failure here is
// surfaced with the exact mpN state and rolls back per the envelope (committed is still
// false) — never a silent half-wired success (C2).
if spec.Mode == ModeDRGuestLoss {
if err := e.swapStructuralBinds(ctx, spec, res); err != nil {
res.Err = fmt.Errorf("reconcile: bring-up structural-bind swap: %w", err)
return
}
}
// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep). // Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil { if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
res.AssignedMAC = net0MAC(cfg2) res.AssignedMAC = net0MAC(cfg2)
@@ -367,6 +433,58 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
e.append(withState(base, OpSucceeded)) e.append(withState(base, OpSucceeded))
} }
// swapStructuralBinds is bring-up step 4d (DR only): set mp8/mp9 to the real host binds via the
// root runner (one slot per call so an error names the exact mpN that failed — C2), then delete
// the displaced throwaway volumes PVE parked as unusedN. The mp9 host dir is created first (pct
// validates the bind source; agent-owned path, plain MkdirAll like the back-half's own bootstrap
// dir) — idempotent: on a same-host guest-loss the dir usually still exists WITH bootstrap.json,
// which the swap must not touch. The unusedN delete goes through the API config PUT
// (VM.Config.Disk + Datastore.Allocate cover it); if the scoped token refuses, the residue is
// logged LOUDLY + surfaced as a result warning and the bring-up continues — a correctly-wired
// guest with a stray volume beats a rollback, and privileges are never widened silently.
func (e *Engine) swapStructuralBinds(ctx context.Context, spec BringUpSpec, res *BringUpResult) error {
bootDir := structuralBootHostDir(e.stateDir, spec.VMID)
if err := os.MkdirAll(bootDir, 0o700); err != nil {
return fmt.Errorf("mp9 bootstrap host dir %s: %w", bootDir, err)
}
if _, stderr, err := e.hostRun.Run(ctx, "mkdir", "-p", structuralParentDir); err != nil {
return fmt.Errorf("parent dir %s: %w: %s", structuralParentDir, err, stderr)
}
parentSpec := structuralParentDir + ",mp=" + structuralParentDir
if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralParentSlot, parentSpec); err != nil {
return fmt.Errorf("set %s (parent bind; neither bind landed): %w: %s", structuralParentSlot, err, stderr)
}
bootSpec := bootDir + ",mp=" + structuralBootGuestPath + ",ro=1"
if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralBootSlot, bootSpec); err != nil {
return fmt.Errorf("set %s (bootstrap bind; %s already landed): %w: %s", structuralBootSlot, structuralParentSlot, err, stderr)
}
e.logger.Info("bring-up: structural binds swapped in", "vmid", spec.VMID,
structuralParentSlot, parentSpec, structuralBootSlot, bootSpec)
// The displaced throwaway volumes now sit as unusedN — read the config and delete them.
cfg, err := e.api.GuestConfig(ctx, spec.VMID)
if err != nil {
return fmt.Errorf("read config after bind swap: %w", err)
}
var unused []string
for k := range cfg.Unused() {
unused = append(unused, k)
}
if len(unused) == 0 {
return nil
}
sort.Strings(unused)
if err := e.setConfigWithLockRetry(ctx, spec.VMID, map[string]string{"delete": strings.Join(unused, ",")}); err != nil {
e.logger.Error("bring-up: could not delete displaced throwaway volumes (guest is correctly wired; residue remains)",
"vmid", spec.VMID, "unused", unused, "err", err)
res.StartWarnings = append(res.StartWarnings,
fmt.Sprintf("structural-bind swap: displaced volumes not deleted (%s): %v", strings.Join(unused, ","), err))
return nil
}
e.logger.Info("bring-up: displaced throwaway volumes deleted", "vmid", spec.VMID, "unused", unused)
return nil
}
// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated // rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
// provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST // provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
// was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not // was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not
+258 -4
View File
@@ -2,14 +2,60 @@ package reconcile
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"io"
"os"
"path/filepath"
"strings" "strings"
"sync"
"testing" "testing"
"time" "time"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
) )
// fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a
// substring of the joined command — makes that one command fail (the C2 mid-swap failure driver).
type fakeRunner struct {
mu sync.Mutex
cmds []string
failOn string
}
func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
cmd := name + " " + strings.Join(args, " ")
f.mu.Lock()
f.cmds = append(f.cmds, cmd)
f.mu.Unlock()
if f.failOn != "" && strings.Contains(cmd, f.failOn) {
return nil, []byte("boom"), errors.New("exit status 1")
}
return nil, nil, nil
}
func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
return f.Run(ctx, name, args...)
}
// newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an
// isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it).
func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) {
t.Helper()
jp := filepath.Join(t.TempDir(), "journal.log")
j, err := OpenJournal(jp)
if err != nil {
t.Fatalf("OpenJournal: %v", err)
}
t.Cleanup(func() { j.Close() })
q := NewQueue()
t.Cleanup(q.Close)
fr := &fakeRunner{}
sd := t.TempDir()
e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd})
return e, fr, sd, q
}
// setParamsFor returns the params of the (last) SetConfig call against vmid, or nil. // setParamsFor returns the params of the (last) SetConfig call against vmid, or nil.
func setParamsFor(api *fakeAPI, vmid int) map[string]string { func setParamsFor(api *fakeAPI, vmid int) map[string]string {
var out map[string]string var out map[string]string
@@ -355,7 +401,7 @@ func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) {
func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) { func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
const vmid = 8001 const vmid = 8001
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, _, q := newEngine(t, api, EmptyProvider{}) e, _, _, q := newDREngine(t, api)
defer q.Close() defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{ res := e.RunBringUp(context.Background(), BringUpSpec{
@@ -393,7 +439,7 @@ func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) { func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) {
const vmid = 8002 const vmid = 8002
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, _, q := newEngine(t, api, EmptyProvider{}) e, _, _, q := newDREngine(t, api)
defer q.Close() defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{ res := e.RunBringUp(context.Background(), BringUpSpec{
@@ -649,7 +695,7 @@ func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall {
func TestRunBringUp_ReassertsPoolMembership(t *testing.T) { func TestRunBringUp_ReassertsPoolMembership(t *testing.T) {
const vmid = 8100 const vmid = 8100
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, _, q := newEngine(t, api, EmptyProvider{}) e, _, _, q := newDREngine(t, api)
defer q.Close() defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{ res := e.RunBringUp(context.Background(), BringUpSpec{
@@ -692,7 +738,7 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()},
poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"), poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"),
} }
e, _, q := newEngine(t, api, EmptyProvider{}) e, _, _, q := newDREngine(t, api)
defer q.Close() defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{ res := e.RunBringUp(context.Background(), BringUpSpec{
@@ -717,3 +763,211 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings) t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings)
} }
} }
// ── GL-5: DR structural bind overrides + 4d swap ─────────────────────────────────────────────────
// GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural
// binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via
// the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes.
// COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts
// (run→fail→revert, recorded in the REPORT).
func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) {
const vmid = 8200
cfg := scratchCfg()
// after the swap PVE parks the two displaced throwaway volumes as unusedN
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`)
cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`)
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
e, fr, sd, q := newDREngine(t, api)
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm", KeepMAC: true,
})
if res.Err != nil || !res.Pass {
t.Fatalf("dr bring-up must pass, got %+v", res)
}
if len(api.restores) != 1 {
t.Fatalf("expected one restore, got %+v", api.restores)
}
ov := api.restores[0].MountOverrides
want := map[string]string{
"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
}
if len(ov) != 2 || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] {
t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want)
}
// 4d: mp9 host dir created under the engine state dir …
bootDir := structuralBootHostDir(sd, vmid)
if st, err := os.Stat(bootDir); err != nil || !st.IsDir() {
t.Fatalf("mp9 bootstrap host dir not created: %v", err)
}
// … and the REAL binds set via the root runner, one slot per call, exact backhalf values.
wantCmds := []string{
"mkdir -p /mnt/felhom-drives",
"pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives",
"pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1",
}
if len(fr.cmds) != len(wantCmds) {
t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds)
}
for i := range wantCmds {
if fr.cmds[i] != wantCmds[i] {
t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i])
}
}
// displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue.
del := ""
for _, s := range api.sets {
if s.vmid == vmid && s.params["delete"] != "" {
del = s.params["delete"]
}
}
if del != "unused0,unused1" {
t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets)
}
}
// GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never
// touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the
// override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT).
func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) {
const vmid = 8201
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm", Hostname: "h",
})
if res.Err != nil || !res.Pass {
t.Fatalf("provision must pass, got %+v", res)
}
if api.restores[0].MountOverrides != nil {
t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides)
}
if len(fr.cmds) != 0 {
t.Fatalf("provision must not touch the host runner, got %v", fr.cmds)
}
}
// GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state
// named and compensating-rolls-back the guest — never a silent half-wired success.
func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) {
const vmid = 8202
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, fr, _, q := newDREngine(t, api)
defer q.Close()
fr.failOn = "-mp9"
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm", KeepMAC: true,
})
if res.Err == nil || res.Pass {
t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res)
}
if !strings.Contains(res.Err.Error(), "structural-bind swap") ||
!strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") {
t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err)
}
if len(api.destroys) != 1 || api.destroys[0] != vmid {
t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys)
}
if len(api.starts) != 0 {
t.Fatalf("a half-wired guest must never be started, got %+v", api.starts)
}
}
// GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived,
// so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the
// end state is identical, with no unusedN residue (here only ONE displaced volume shows up).
func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) {
const vmid = 8203
cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`)
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
e, fr, _, q := newDREngine(t, api)
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm", KeepMAC: true,
})
if res.Err != nil || !res.Pass {
t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res)
}
ov := api.restores[0].MountOverrides
if len(ov) != 2 || ov["mp8"] == "" || ov["mp9"] == "" {
t.Fatalf("overrides are constants — both mpN must be named regardless of the archive: %+v", ov)
}
if len(fr.cmds) != 3 {
t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds)
}
del := ""
for _, s := range api.sets {
if s.vmid == vmid && s.params["delete"] != "" {
del = s.params["delete"]
}
}
if del != "unused0" {
t.Fatalf("the one displaced volume must be deleted: delete=%q", del)
}
}
// GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because
// the structural-bind swap is a root pct op the API token cannot perform.
func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) {
const vmid = 8204
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm",
})
if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") {
t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res)
}
if len(api.restores) != 0 {
t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores)
}
}
// GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired
// guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently).
func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) {
const vmid = 8205
cfg := scratchCfg()
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`)
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
api.setFunc = func(_ int, params map[string]string) (string, error) {
if params["delete"] != "" {
return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed")
}
return "", nil
}
e, _, _, q := newDREngine(t, api)
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
RestoreStorage: "local-lvm", KeepMAC: true,
})
if res.Err != nil || !res.Pass {
t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res)
}
found := false
for _, w := range res.StartWarnings {
if strings.Contains(w, "displaced volumes not deleted") {
found = true
}
}
if !found {
t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings)
}
}
+20
View File
@@ -32,6 +32,15 @@ type Engine struct {
hostID string hostID string
logger *slog.Logger logger *slog.Logger
// hostRun executes host-root commands for the DR structural-bind swap (bring-up 4d): a
// bind-mount `pct set` is root@pam-only, so it cannot go through the API token (the same
// constraint the provision back-half documents). nil on API-only engines — RunBringUp
// refuses ModeDRGuestLoss then (GL-5).
hostRun proxmox.Runner
// stateDir is the agent state dir the mp9 bootstrap host dir lives under
// (<stateDir>/guests/<vmid>/bootstrap); "" → /var/lib/felhom-agent (mirrors provision.NewBackHalf).
stateDir string
opSeq uint64 // atomic; makes each op id unique per attempt opSeq uint64 // atomic; makes each op id unique per attempt
} }
@@ -46,6 +55,11 @@ type EngineOptions struct {
Gate *Gate Gate *Gate
HostID string HostID string
Logger *slog.Logger Logger *slog.Logger
// HostRunner enables the DR structural-bind swap (bring-up 4d, root pct ops). Optional —
// engines that never run ModeDRGuestLoss may leave it nil.
HostRunner proxmox.Runner
// StateDir is the agent state dir ("" → /var/lib/felhom-agent); only the 4d swap reads it.
StateDir string
} }
// NewEngine builds an Engine. The Queue is shared (the single §10 choke point); the // NewEngine builds an Engine. The Queue is shared (the single §10 choke point); the
@@ -69,6 +83,10 @@ func NewEngine(opts EngineOptions) *Engine {
// the common slice-4 daemon state (no signers pinned, no desired state). // the common slice-4 daemon state (no signers pinned, no desired state).
gate = NewGate(nil, opts.HostID, nil, logger) gate = NewGate(nil, opts.HostID, nil, logger)
} }
stateDir := opts.StateDir
if stateDir == "" {
stateDir = "/var/lib/felhom-agent"
}
return &Engine{ return &Engine{
api: opts.API, api: opts.API,
queue: opts.Queue, queue: opts.Queue,
@@ -78,6 +96,8 @@ func NewEngine(opts EngineOptions) *Engine {
gate: gate, gate: gate,
hostID: opts.HostID, hostID: opts.HostID,
logger: logger, logger: logger,
hostRun: opts.HostRunner,
stateDir: stateDir,
} }
} }
+10 -1
View File
@@ -342,7 +342,7 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string
if mp == "" { if mp == "" {
mp = volPart // fall back to the host path if no explicit in-guest mp= mp = volPart // fall back to the host path if no explicit in-guest mp=
} }
out[key] = fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, mp) out[key] = throwawayVolumeOverride(restoreStorage, mp)
} }
if len(out) == 0 { if len(out) == 0 {
return nil return nil
@@ -350,6 +350,15 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string
return out return out
} }
// throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G
// volume on restoreStorage at the given in-guest path, excluded from backup. Used by the
// restore-test's source-config-derived overrides (above) and by the DR bring-up's synthesized
// structural-bind overrides (GL-5, bringup.go — its mpN are platform constants, so it skips this
// file's is-a-bind filtering and calls the format directly).
func throwawayVolumeOverride(restoreStorage, guestPath string) string {
return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath)
}
// mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent). // mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent).
func mountPathOf(opts string) string { func mountPathOf(opts string) string {
for _, kv := range strings.Split(opts, ",") { for _, kv := range strings.Split(opts, ",") {