GL-5: DR bring-up structural bind overrides + 4d real-bind swap
ModeDRGuestLoss now passes restore-time MountOverrides for the two
platform-constant structural binds (mp8 parent, mp9 bootstrap) via the
shared throwaway-volume format helper - without them a customer-archive
restore under the privsep token fails outright ("restoring 'mp8' to bind
mount is only possible for root"). New post-restore step 4d swaps the real
binds in via the host runner (root pct set, one slot per call), deletes the
displaced unusedN volumes (API config PUT; a scoped-token refusal logs the
residue loudly instead of widening privileges), and respects the
committed/launched rollback envelope. Provision passes nil overrides -
byte-identical behavior (regression contract test).
Engine grows an optional HostRunner + StateDir seam (DR refuses up front
without a runner); selftest bring-up wires the ExecRunner + cleans the
scratch mp9 host dir on teardown; proxmox.GuestConfig.Unused() added.
6 new tests incl. C2 mid-swap rollback + C3 older-archive + 403-warn paths.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -20,6 +20,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -1426,8 +1427,16 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
|
||||
}
|
||||
}
|
||||
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
||||
// GL-5: DR bring-up swaps the structural binds (mp8/mp9) via root pct ops — wire the same
|
||||
// Runner shape the provision back-half uses. Provision mode never touches it.
|
||||
rMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||
if rMode == "" {
|
||||
rMode = proxmox.RunnerSudo
|
||||
}
|
||||
hostRunner := &proxmox.ExecRunner{Mode: rMode, SudoPath: cfg.Privileged.SudoPath}
|
||||
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
||||
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
||||
HostRunner: hostRunner,
|
||||
})
|
||||
|
||||
fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
|
||||
@@ -1475,6 +1484,14 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
|
||||
return 1
|
||||
}
|
||||
}
|
||||
// GL-5: the DR bind swap created <stateDir>/guests/<vmid>/bootstrap for this SCRATCH vmid —
|
||||
// remove the agent-owned per-guest dir with the guest (never a real drive's bind source; a
|
||||
// scratch vmid has no other state). Best-effort.
|
||||
if bmode == reconcile.ModeDRGuestLoss {
|
||||
if err := os.RemoveAll(filepath.Join("/var/lib/felhom-agent/guests", strconv.Itoa(vmid))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, " [WARN] scratch mp9 host dir cleanup: %v\n", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -133,6 +133,13 @@ func (g *GuestConfig) Nets() map[string]string {
|
||||
return g.prefixed("net")
|
||||
}
|
||||
|
||||
// Unused returns the unusedN entries from Extra — the displaced volumes PVE parks in the config
|
||||
// when a mountpoint is overridden/replaced (e.g. the DR bring-up's structural-bind swap replacing
|
||||
// its restore-time throwaway volumes). The caller deletes these so a kept guest carries no residue.
|
||||
func (g *GuestConfig) Unused() map[string]string {
|
||||
return g.prefixed("unused")
|
||||
}
|
||||
|
||||
// Lock returns the guest's current lock ("backup", "snapshot-delete", "migrate", …) from the config,
|
||||
// or "" when unlocked. An interrupted vzdump leaves a "backup" or "snapshot-delete" lock — the signal
|
||||
// the startup stale-lock recovery (F2-b) keys on.
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -54,6 +56,32 @@ const DefaultPool = "felhom"
|
||||
// volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half).
|
||||
const DefaultSysDataMount = "mp1"
|
||||
|
||||
// Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of
|
||||
// SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8,
|
||||
// /mnt/felhom-drives, same in-guest path) and the bootstrap config bind (mp9,
|
||||
// <stateDir>/guests/<vmid>/bootstrap -> /etc/felhom-bootstrap, read-only). These are PLATFORM
|
||||
// CONSTANTS - backhalf.go names mp8 "the single permanent parent bind" and mp9's host path is
|
||||
// templated only by vmid - which is exactly why DR can synthesize them without reading the lost
|
||||
// guest's config. Mirrors provision/backhalf.go's stableParentDir/parentBindSlot/DefaultGuestPath/
|
||||
// DefaultMountIndex as literals (the same avoid-the-import-edge rationale backhalf itself uses for
|
||||
// its localapi mirror). A customer archive CARRIES these mpN entries, and a pct restore of a
|
||||
// bind mount is root@pam-only ("restoring 'mpN' to bind mount is only possible for root") - so a
|
||||
// DR restore under the privsep token MUST override them (throwaway volumes) and swap the real
|
||||
// binds back post-restore (step 4d).
|
||||
const (
|
||||
structuralParentSlot = "mp8"
|
||||
structuralParentDir = "/mnt/felhom-drives"
|
||||
structuralBootSlot = "mp9"
|
||||
structuralBootGuestPath = "/etc/felhom-bootstrap"
|
||||
)
|
||||
|
||||
// structuralBootHostDir is the mp9 bind's host source for vmid (mirrors the back-half's
|
||||
// <stateDir>/guests/<vmid>/bootstrap layout). Joined with "/" explicitly: this is a HOST (Linux)
|
||||
// path that flows into pct arguments - filepath.Join would mangle it on a non-Linux test runner.
|
||||
func structuralBootHostDir(stateDir string, vmid int) string {
|
||||
return strings.TrimRight(stateDir, "/") + "/guests/" + strconv.Itoa(vmid) + "/bootstrap"
|
||||
}
|
||||
|
||||
// configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
|
||||
// tests can shrink it (the production value gives PVE time to release its async config lock).
|
||||
const configLockMaxAttempts = 5
|
||||
@@ -163,6 +191,12 @@ func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult
|
||||
res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode)
|
||||
return res
|
||||
}
|
||||
// DR needs the host runner: the structural-bind swap (4d) is a root pct op the API token
|
||||
// cannot perform. Refuse up front rather than fail after a restore (GL-5).
|
||||
if spec.Mode == ModeDRGuestLoss && e.hostRun == nil {
|
||||
res.Err = fmt.Errorf("reconcile: dr bring-up needs a host runner (the mp8/mp9 structural-bind swap is a root pct op) — wire EngineOptions.HostRunner")
|
||||
return res
|
||||
}
|
||||
|
||||
ch := e.queue.Submit(spec.VMID, func() error {
|
||||
e.runBringUp(ctx, spec, &res)
|
||||
@@ -219,8 +253,26 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
}()
|
||||
|
||||
// 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike).
|
||||
// DR (GL-5): the customer archive carries the two STRUCTURAL host-bind mountpoints (mp8
|
||||
// parent bind + mp9 bootstrap bind) that a restore under the privsep token cannot recreate
|
||||
// — without overrides the whole restore FAILS ("restoring 'mp8' to bind mount is only
|
||||
// possible for root"). Synthesize throwaway-volume overrides for the two known-constant mpN
|
||||
// via the shared format helper (bindMountOverrides' is-a-bind filter is for reading real
|
||||
// configs, which DR by definition cannot do — the guest is gone); step 4d swaps the real
|
||||
// binds back post-restore. An archive that LACKS one of them (older backup) is fine: the
|
||||
// override simply creates that mpN at restore and 4d normalizes it — the end state is
|
||||
// identical (C3). Provision stays override-free (nil): the golden has no mp8/mp9 (the
|
||||
// back-half adds them post-bring-up) — that asymmetry is the whole GL-5 bug.
|
||||
var overrides map[string]string
|
||||
if spec.Mode == ModeDRGuestLoss {
|
||||
overrides = map[string]string{
|
||||
structuralParentSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralParentDir),
|
||||
structuralBootSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralBootGuestPath),
|
||||
}
|
||||
}
|
||||
upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
|
||||
VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
|
||||
MountOverrides: overrides,
|
||||
})
|
||||
if err != nil {
|
||||
// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
|
||||
@@ -307,6 +359,20 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
}
|
||||
}
|
||||
|
||||
// 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the
|
||||
// restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR
|
||||
// guest carries no unusedN residue. Bind-mount pct sets are root@pam-only — the swap goes
|
||||
// through the host runner, never the API. Runs BEFORE start so the first boot already sees
|
||||
// the real binds (the golden's baked bootstrap unit + the drives parent). A failure here is
|
||||
// surfaced with the exact mpN state and rolls back per the envelope (committed is still
|
||||
// false) — never a silent half-wired success (C2).
|
||||
if spec.Mode == ModeDRGuestLoss {
|
||||
if err := e.swapStructuralBinds(ctx, spec, res); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: bring-up structural-bind swap: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
|
||||
if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
|
||||
res.AssignedMAC = net0MAC(cfg2)
|
||||
@@ -367,6 +433,58 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
e.append(withState(base, OpSucceeded))
|
||||
}
|
||||
|
||||
// swapStructuralBinds is bring-up step 4d (DR only): set mp8/mp9 to the real host binds via the
|
||||
// root runner (one slot per call so an error names the exact mpN that failed — C2), then delete
|
||||
// the displaced throwaway volumes PVE parked as unusedN. The mp9 host dir is created first (pct
|
||||
// validates the bind source; agent-owned path, plain MkdirAll like the back-half's own bootstrap
|
||||
// dir) — idempotent: on a same-host guest-loss the dir usually still exists WITH bootstrap.json,
|
||||
// which the swap must not touch. The unusedN delete goes through the API config PUT
|
||||
// (VM.Config.Disk + Datastore.Allocate cover it); if the scoped token refuses, the residue is
|
||||
// logged LOUDLY + surfaced as a result warning and the bring-up continues — a correctly-wired
|
||||
// guest with a stray volume beats a rollback, and privileges are never widened silently.
|
||||
func (e *Engine) swapStructuralBinds(ctx context.Context, spec BringUpSpec, res *BringUpResult) error {
|
||||
bootDir := structuralBootHostDir(e.stateDir, spec.VMID)
|
||||
if err := os.MkdirAll(bootDir, 0o700); err != nil {
|
||||
return fmt.Errorf("mp9 bootstrap host dir %s: %w", bootDir, err)
|
||||
}
|
||||
if _, stderr, err := e.hostRun.Run(ctx, "mkdir", "-p", structuralParentDir); err != nil {
|
||||
return fmt.Errorf("parent dir %s: %w: %s", structuralParentDir, err, stderr)
|
||||
}
|
||||
parentSpec := structuralParentDir + ",mp=" + structuralParentDir
|
||||
if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralParentSlot, parentSpec); err != nil {
|
||||
return fmt.Errorf("set %s (parent bind; neither bind landed): %w: %s", structuralParentSlot, err, stderr)
|
||||
}
|
||||
bootSpec := bootDir + ",mp=" + structuralBootGuestPath + ",ro=1"
|
||||
if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralBootSlot, bootSpec); err != nil {
|
||||
return fmt.Errorf("set %s (bootstrap bind; %s already landed): %w: %s", structuralBootSlot, structuralParentSlot, err, stderr)
|
||||
}
|
||||
e.logger.Info("bring-up: structural binds swapped in", "vmid", spec.VMID,
|
||||
structuralParentSlot, parentSpec, structuralBootSlot, bootSpec)
|
||||
|
||||
// The displaced throwaway volumes now sit as unusedN — read the config and delete them.
|
||||
cfg, err := e.api.GuestConfig(ctx, spec.VMID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read config after bind swap: %w", err)
|
||||
}
|
||||
var unused []string
|
||||
for k := range cfg.Unused() {
|
||||
unused = append(unused, k)
|
||||
}
|
||||
if len(unused) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(unused)
|
||||
if err := e.setConfigWithLockRetry(ctx, spec.VMID, map[string]string{"delete": strings.Join(unused, ",")}); err != nil {
|
||||
e.logger.Error("bring-up: could not delete displaced throwaway volumes (guest is correctly wired; residue remains)",
|
||||
"vmid", spec.VMID, "unused", unused, "err", err)
|
||||
res.StartWarnings = append(res.StartWarnings,
|
||||
fmt.Sprintf("structural-bind swap: displaced volumes not deleted (%s): %v", strings.Join(unused, ","), err))
|
||||
return nil
|
||||
}
|
||||
e.logger.Info("bring-up: displaced throwaway volumes deleted", "vmid", spec.VMID, "unused", unused)
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
|
||||
// provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
|
||||
// was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not
|
||||
|
||||
@@ -2,14 +2,60 @@ package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// fakeRunner records the host-root commands the DR structural-bind swap issues (GL-5). failOn — a
|
||||
// substring of the joined command — makes that one command fail (the C2 mid-swap failure driver).
|
||||
type fakeRunner struct {
|
||||
mu sync.Mutex
|
||||
cmds []string
|
||||
failOn string
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
cmd := name + " " + strings.Join(args, " ")
|
||||
f.mu.Lock()
|
||||
f.cmds = append(f.cmds, cmd)
|
||||
f.mu.Unlock()
|
||||
if f.failOn != "" && strings.Contains(cmd, f.failOn) {
|
||||
return nil, []byte("boom"), errors.New("exit status 1")
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||
return f.Run(ctx, name, args...)
|
||||
}
|
||||
|
||||
// newDREngine builds an engine wired for ModeDRGuestLoss (GL-5): a recording host runner + an
|
||||
// isolated state dir (the 4d swap MkdirAlls the mp9 host dir under it).
|
||||
func newDREngine(t *testing.T, api GuestAPI) (*Engine, *fakeRunner, string, *Queue) {
|
||||
t.Helper()
|
||||
jp := filepath.Join(t.TempDir(), "journal.log")
|
||||
j, err := OpenJournal(jp)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenJournal: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { j.Close() })
|
||||
q := NewQueue()
|
||||
t.Cleanup(q.Close)
|
||||
fr := &fakeRunner{}
|
||||
sd := t.TempDir()
|
||||
e := NewEngine(EngineOptions{API: api, Queue: q, Journal: j, Provider: EmptyProvider{}, HostRunner: fr, StateDir: sd})
|
||||
return e, fr, sd, q
|
||||
}
|
||||
|
||||
// setParamsFor returns the params of the (last) SetConfig call against vmid, or nil.
|
||||
func setParamsFor(api *fakeAPI, vmid int) map[string]string {
|
||||
var out map[string]string
|
||||
@@ -355,7 +401,7 @@ func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) {
|
||||
func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
|
||||
const vmid = 8001
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
e, _, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
@@ -393,7 +439,7 @@ func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
|
||||
func TestRunBringUp_DRResetMACWhenSourceMayBeLive(t *testing.T) {
|
||||
const vmid = 8002
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
e, _, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
@@ -649,7 +695,7 @@ func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall {
|
||||
func TestRunBringUp_ReassertsPoolMembership(t *testing.T) {
|
||||
const vmid = 8100
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
e, _, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
@@ -692,7 +738,7 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
|
||||
cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()},
|
||||
poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"),
|
||||
}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
e, _, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
@@ -717,3 +763,211 @@ func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
|
||||
t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// ── GL-5: DR structural bind overrides + 4d swap ─────────────────────────────────────────────────
|
||||
|
||||
// GL-5 Scenario A: a DR bring-up passes restore-time MountOverrides for EXACTLY the two structural
|
||||
// binds (throwaway volumes in bindMountOverrides format), then step 4d swaps the REAL binds in via
|
||||
// the root runner (mp9 host dir created first) and deletes the displaced unusedN volumes.
|
||||
// COMPANION RED-PROOF: reverting the Part-1 override synthesis fails the MountOverrides asserts
|
||||
// (run→fail→revert, recorded in the REPORT).
|
||||
func TestRunBringUp_DRStructuralBindOverridesAndSwap(t *testing.T) {
|
||||
const vmid = 8200
|
||||
cfg := scratchCfg()
|
||||
// after the swap PVE parks the two displaced throwaway volumes as unusedN
|
||||
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8200-disk-2"`)
|
||||
cfg.Extra["unused1"] = json.RawMessage(`"local-lvm:vm-8200-disk-3"`)
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
||||
e, fr, sd, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm", KeepMAC: true,
|
||||
})
|
||||
if res.Err != nil || !res.Pass {
|
||||
t.Fatalf("dr bring-up must pass, got %+v", res)
|
||||
}
|
||||
if len(api.restores) != 1 {
|
||||
t.Fatalf("expected one restore, got %+v", api.restores)
|
||||
}
|
||||
ov := api.restores[0].MountOverrides
|
||||
want := map[string]string{
|
||||
"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
|
||||
"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
|
||||
}
|
||||
if len(ov) != 2 || ov["mp8"] != want["mp8"] || ov["mp9"] != want["mp9"] {
|
||||
t.Fatalf("MountOverrides = %+v, want exactly %+v", ov, want)
|
||||
}
|
||||
// 4d: mp9 host dir created under the engine state dir …
|
||||
bootDir := structuralBootHostDir(sd, vmid)
|
||||
if st, err := os.Stat(bootDir); err != nil || !st.IsDir() {
|
||||
t.Fatalf("mp9 bootstrap host dir not created: %v", err)
|
||||
}
|
||||
// … and the REAL binds set via the root runner, one slot per call, exact backhalf values.
|
||||
wantCmds := []string{
|
||||
"mkdir -p /mnt/felhom-drives",
|
||||
"pct set 8200 -mp8 /mnt/felhom-drives,mp=/mnt/felhom-drives",
|
||||
"pct set 8200 -mp9 " + bootDir + ",mp=/etc/felhom-bootstrap,ro=1",
|
||||
}
|
||||
if len(fr.cmds) != len(wantCmds) {
|
||||
t.Fatalf("runner cmds = %v, want %v", fr.cmds, wantCmds)
|
||||
}
|
||||
for i := range wantCmds {
|
||||
if fr.cmds[i] != wantCmds[i] {
|
||||
t.Fatalf("runner cmd[%d] = %q, want %q", i, fr.cmds[i], wantCmds[i])
|
||||
}
|
||||
}
|
||||
// displaced throwaways deleted in ONE config PUT (deterministic order) — no unusedN residue.
|
||||
del := ""
|
||||
for _, s := range api.sets {
|
||||
if s.vmid == vmid && s.params["delete"] != "" {
|
||||
del = s.params["delete"]
|
||||
}
|
||||
}
|
||||
if del != "unused0,unused1" {
|
||||
t.Fatalf("displaced volumes not deleted: delete=%q sets=%+v", del, api.sets)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5 Scenario B (the regression contract): provision passes NO MountOverrides (nil) and never
|
||||
// touches the host runner — the golden path is behavior-identical. COMPANION RED-PROOF: making the
|
||||
// override synthesis unconditional fails this (run→fail→revert, recorded in the REPORT).
|
||||
func TestRunBringUp_ProvisionNoMountOverrides(t *testing.T) {
|
||||
const vmid = 8201
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, fr, _, q := newDREngine(t, api) // runner present but must stay UNUSED
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm", Hostname: "h",
|
||||
})
|
||||
if res.Err != nil || !res.Pass {
|
||||
t.Fatalf("provision must pass, got %+v", res)
|
||||
}
|
||||
if api.restores[0].MountOverrides != nil {
|
||||
t.Fatalf("provision must pass NO MountOverrides, got %+v", api.restores[0].MountOverrides)
|
||||
}
|
||||
if len(fr.cmds) != 0 {
|
||||
t.Fatalf("provision must not touch the host runner, got %v", fr.cmds)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5 C2: a mid-swap failure (here: the mp9 pct set) fails the bring-up with the exact mpN state
|
||||
// named and compensating-rolls-back the guest — never a silent half-wired success.
|
||||
func TestRunBringUp_DRSwapFailureRollsBack(t *testing.T) {
|
||||
const vmid = 8202
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, fr, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
fr.failOn = "-mp9"
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm", KeepMAC: true,
|
||||
})
|
||||
if res.Err == nil || res.Pass {
|
||||
t.Fatalf("mid-swap failure must fail the bring-up, got %+v", res)
|
||||
}
|
||||
if !strings.Contains(res.Err.Error(), "structural-bind swap") ||
|
||||
!strings.Contains(res.Err.Error(), "mp9") || !strings.Contains(res.Err.Error(), "mp8 already landed") {
|
||||
t.Fatalf("error must name the swap + the exact mpN state, got: %v", res.Err)
|
||||
}
|
||||
if len(api.destroys) != 1 || api.destroys[0] != vmid {
|
||||
t.Fatalf("mid-swap failure must compensating-roll-back (destroy %d), got %+v", vmid, api.destroys)
|
||||
}
|
||||
if len(api.starts) != 0 {
|
||||
t.Fatalf("a half-wired guest must never be started, got %+v", api.starts)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5 C3: an OLDER archive without mp9 — the overrides are platform constants, not archive-derived,
|
||||
// so the restore still names BOTH mpN (PVE simply creates the missing one) and 4d normalizes; the
|
||||
// end state is identical, with no unusedN residue (here only ONE displaced volume shows up).
|
||||
func TestRunBringUp_DRArchiveWithoutMp9(t *testing.T) {
|
||||
const vmid = 8203
|
||||
cfg := scratchCfg() // no mp9 in Extra — the pre-override archive shape
|
||||
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8203-disk-2"`)
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
||||
e, fr, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeDRGuestLoss, Archive: "local:backup/old-customer.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm", KeepMAC: true,
|
||||
})
|
||||
if res.Err != nil || !res.Pass {
|
||||
t.Fatalf("dr bring-up of an mp9-less archive must pass, got %+v", res)
|
||||
}
|
||||
ov := api.restores[0].MountOverrides
|
||||
if len(ov) != 2 || ov["mp8"] == "" || ov["mp9"] == "" {
|
||||
t.Fatalf("overrides are constants — both mpN must be named regardless of the archive: %+v", ov)
|
||||
}
|
||||
if len(fr.cmds) != 3 {
|
||||
t.Fatalf("4d must run identically (mkdir + 2 pct sets), got %v", fr.cmds)
|
||||
}
|
||||
del := ""
|
||||
for _, s := range api.sets {
|
||||
if s.vmid == vmid && s.params["delete"] != "" {
|
||||
del = s.params["delete"]
|
||||
}
|
||||
}
|
||||
if del != "unused0" {
|
||||
t.Fatalf("the one displaced volume must be deleted: delete=%q", del)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5: DR on an API-only engine (no host runner) refuses UP FRONT — before any restore — because
|
||||
// the structural-bind swap is a root pct op the API token cannot perform.
|
||||
func TestRunBringUp_DRWithoutRunnerRefuses(t *testing.T) {
|
||||
const vmid = 8204
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{}) // no HostRunner
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm",
|
||||
})
|
||||
if res.Err == nil || !strings.Contains(res.Err.Error(), "host runner") {
|
||||
t.Fatalf("dr without a runner must refuse naming the gap, got %+v", res)
|
||||
}
|
||||
if len(api.restores) != 0 {
|
||||
t.Fatalf("the refusal must fire BEFORE any restore, got %+v", api.restores)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5: a refused unusedN delete (the scoped-token-403 class) must NOT fail the correctly-wired
|
||||
// guest — the residue is surfaced as a LOUD result warning instead (privileges never widen silently).
|
||||
func TestRunBringUp_DRUnusedDeleteFailureWarns(t *testing.T) {
|
||||
const vmid = 8205
|
||||
cfg := scratchCfg()
|
||||
cfg.Extra["unused0"] = json.RawMessage(`"local-lvm:vm-8205-disk-2"`)
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: cfg}}
|
||||
api.setFunc = func(_ int, params map[string]string) (string, error) {
|
||||
if params["delete"] != "" {
|
||||
return "", errors.New("proxmox: PUT config -> HTTP 403: Permission check failed")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
e, _, _, q := newDREngine(t, api)
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunBringUp(context.Background(), BringUpSpec{
|
||||
Mode: ModeDRGuestLoss, Archive: "local:backup/customer.tar.zst", VMID: vmid,
|
||||
RestoreStorage: "local-lvm", KeepMAC: true,
|
||||
})
|
||||
if res.Err != nil || !res.Pass {
|
||||
t.Fatalf("a refused residue delete must NOT fail a wired guest, got %+v", res)
|
||||
}
|
||||
found := false
|
||||
for _, w := range res.StartWarnings {
|
||||
if strings.Contains(w, "displaced volumes not deleted") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("residue must surface as a warning: %+v", res.StartWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,15 @@ type Engine struct {
|
||||
hostID string
|
||||
logger *slog.Logger
|
||||
|
||||
// hostRun executes host-root commands for the DR structural-bind swap (bring-up 4d): a
|
||||
// bind-mount `pct set` is root@pam-only, so it cannot go through the API token (the same
|
||||
// constraint the provision back-half documents). nil on API-only engines — RunBringUp
|
||||
// refuses ModeDRGuestLoss then (GL-5).
|
||||
hostRun proxmox.Runner
|
||||
// stateDir is the agent state dir the mp9 bootstrap host dir lives under
|
||||
// (<stateDir>/guests/<vmid>/bootstrap); "" → /var/lib/felhom-agent (mirrors provision.NewBackHalf).
|
||||
stateDir string
|
||||
|
||||
opSeq uint64 // atomic; makes each op id unique per attempt
|
||||
}
|
||||
|
||||
@@ -46,6 +55,11 @@ type EngineOptions struct {
|
||||
Gate *Gate
|
||||
HostID string
|
||||
Logger *slog.Logger
|
||||
// HostRunner enables the DR structural-bind swap (bring-up 4d, root pct ops). Optional —
|
||||
// engines that never run ModeDRGuestLoss may leave it nil.
|
||||
HostRunner proxmox.Runner
|
||||
// StateDir is the agent state dir ("" → /var/lib/felhom-agent); only the 4d swap reads it.
|
||||
StateDir string
|
||||
}
|
||||
|
||||
// NewEngine builds an Engine. The Queue is shared (the single §10 choke point); the
|
||||
@@ -69,6 +83,10 @@ func NewEngine(opts EngineOptions) *Engine {
|
||||
// the common slice-4 daemon state (no signers pinned, no desired state).
|
||||
gate = NewGate(nil, opts.HostID, nil, logger)
|
||||
}
|
||||
stateDir := opts.StateDir
|
||||
if stateDir == "" {
|
||||
stateDir = "/var/lib/felhom-agent"
|
||||
}
|
||||
return &Engine{
|
||||
api: opts.API,
|
||||
queue: opts.Queue,
|
||||
@@ -78,6 +96,8 @@ func NewEngine(opts EngineOptions) *Engine {
|
||||
gate: gate,
|
||||
hostID: opts.HostID,
|
||||
logger: logger,
|
||||
hostRun: opts.HostRunner,
|
||||
stateDir: stateDir,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -342,7 +342,7 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string
|
||||
if mp == "" {
|
||||
mp = volPart // fall back to the host path if no explicit in-guest mp=
|
||||
}
|
||||
out[key] = fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, mp)
|
||||
out[key] = throwawayVolumeOverride(restoreStorage, mp)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
@@ -350,6 +350,15 @@ func bindMountOverrides(mps map[string]string, restoreStorage string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G
|
||||
// volume on restoreStorage at the given in-guest path, excluded from backup. Used by the
|
||||
// restore-test's source-config-derived overrides (above) and by the DR bring-up's synthesized
|
||||
// structural-bind overrides (GL-5, bringup.go — its mpN are platform constants, so it skips this
|
||||
// file's is-a-bind filtering and calls the format directly).
|
||||
func throwawayVolumeOverride(restoreStorage, guestPath string) string {
|
||||
return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath)
|
||||
}
|
||||
|
||||
// mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent).
|
||||
func mountPathOf(opts string) string {
|
||||
for _, kv := range strings.Split(opts, ",") {
|
||||
|
||||
Reference in New Issue
Block a user