felhom-agent/internal/reconcile/bringup.go

package reconcile

import (
	"context"
	"errors"
	"fmt"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// The unified bring-up reconcile job (doc 03 §9, slice 7 Phase 1). It is the shared FRONT
// HALF of provision and guest-loss DR: restore an archive → reset identity (scenario-specific)
// → size → attach mounts → start LINK-UP. It mirrors RunRestoreTest's crash-safety (journal the
// owning entry BEFORE any mutation; Recover reaps a leak via ListLXC; idempotent, fail-safe) but
// DIFFERS in two load-bearing ways: it KEEPS the guest on success (no teardown), and its
// identity policy is scenario-specific (provision = fresh identity; DR = preserve continuity).
//
// Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993): F1 (restore
// preserves the archived MAC → provision MUST reset it), F3 (machine-id + SSH host keys
// regenerate guest-side on first boot from the clean golden + its baked unit — the agent does
// NOT touch guest internals here), F4 (a transient PVE config-lock 500 → bounded retry).
//
// Out of scope (deferred — see REPORT): the provisioning BACK HALF (controller deploy, bootstrap,
// per-guest token mint → slice 8); host-loss DR + escrow consumption → slice 10; the SOURCE of a
// BringUpSpec (hub desired-state) → slice 10 (this job takes the spec as input).

// BringUpMode selects the identity policy + archive semantics.
type BringUpMode string

const (
	// ModeProvision: from the golden base — a NEW guest, so reset identity fully.
	ModeProvision BringUpMode = "provision"
	// ModeDRGuestLoss: from a customer backup — CONTINUE the customer's world, so preserve
	// continuity identity (hostname, host keys, and by default MAC), reset only what collides.
	ModeDRGuestLoss BringUpMode = "dr_guest_loss"
)

const bringUpKind = "bring_up"

// DefaultDataVolMount is the mpN slot the golden bakes the Docker-data volume (/var/lib/docker) at.
const DefaultDataVolMount = "mp0"

// DefaultPool is the PVE pool every Felhom-managed guest is restored INTO. Under the pool-scoped agent
// token, restore-into-pool is how a fresh vmid gets allocated (VM.Allocate + Pool.Allocate at
// /pool/felhom) and how the guest becomes reachable by the scoped token (SPIKE-pool-scoped-acl-2026-07-01).
// Single source of truth for both restore sites (provision bring-up + restore-test).
const DefaultPool = "felhom"

// DefaultSysDataMount is the mpN slot the golden bakes the SSD user-data volume (/mnt/sys_drive) at.
// This is the controller's system_data_path; provision grows it (SysDataGrowGB) like the Docker-data
// volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half).
const DefaultSysDataMount = "mp1"

// Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of
// SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8,
// /mnt/felhom-drives, same in-guest path) and the bootstrap config bind (mp9,
// <stateDir>/guests/<vmid>/bootstrap -> /etc/felhom-bootstrap, read-only). These are PLATFORM
// CONSTANTS - backhalf.go names mp8 "the single permanent parent bind" and mp9's host path is
// templated only by vmid - which is exactly why DR can synthesize them without reading the lost
// guest's config. Mirrors provision/backhalf.go's stableParentDir/parentBindSlot/DefaultGuestPath/
// DefaultMountIndex as literals (the same avoid-the-import-edge rationale backhalf itself uses for
// its localapi mirror). A customer archive CARRIES these mpN entries, and a pct restore of a
// bind mount is root@pam-only ("restoring 'mpN' to bind mount is only possible for root") - so a
// DR restore under the privsep token MUST override them (throwaway volumes) and swap the real
// binds back post-restore (step 4d).
const (
	structuralParentSlot    = "mp8"
	structuralParentDir     = "/mnt/felhom-drives"
	structuralBootSlot      = "mp9"
	structuralBootGuestPath = "/etc/felhom-bootstrap"
)

// structuralBootHostDir is the mp9 bind's host source for vmid (mirrors the back-half's
// <stateDir>/guests/<vmid>/bootstrap layout). Joined with "/" explicitly: this is a HOST (Linux)
// path that flows into pct arguments - filepath.Join would mangle it on a non-Linux test runner.
func structuralBootHostDir(stateDir string, vmid int) string {
	return strings.TrimRight(stateDir, "/") + "/guests/" + strconv.Itoa(vmid) + "/bootstrap"
}

// configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
// tests can shrink it (the production value gives PVE time to release its async config lock).
const configLockMaxAttempts = 5

var configLockBackoff = 1500 * time.Millisecond

// GuestMount is one additive mountpoint to attach as mpN. Defined minimally now; slice 10 wires
// the hub storage manifest into this (do NOT couple to a not-yet-existing hub desired-state type).
type GuestMount struct {
	Storage    string // PVE storage id (e.g. "local-lvm")
	SizeGB     int    // new-volume size in GiB
	MountPoint string // in-guest path (e.g. "/mnt/data")
	// Backup includes this mountpoint in vzdump/PBS. MANDATORY for any data-bearing mount (DB
	// volumes), because extra LXC mountpoints default to backup=0 = EXCLUDED from the snapshot
	// (storage-split finding B3). The Docker-data volume normally rides in from the golden archive
	// (already backup=1) and is grown via DataVolGrowGB rather than attached here, but any data
	// mount attached through spec.Mounts MUST set this or its contents silently fall out of PBS.
	Backup bool
}

// BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10
// hub desired-state); this job does not decide WHAT to provision.
type BringUpSpec struct {
	Mode           BringUpMode // provision | dr_guest_loss
	Archive        string      // source volid (golden for provision; customer backup for DR)
	VMID           int         // caller-provided target VMID (NOT the restore-test band / 9999)
	RestoreStorage string      // rootfs target storage
	Hostname       string      // hostname to set (provision); ignored for DR (continuity)
	Pool           string      // restore the guest INTO this PVE pool ("" = none); required under a pool-scoped token
	Cores          int         // 0 = leave as restored
	MemoryMB       int         // 0 = leave as restored
	RootfsGrowGB   int         // optional grow-only rootfs resize (0 = skip)
	// DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the
	// per-customer target. The golden ships a small data volume with the baked images; provision
	// grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that
	// would shadow the baked images. 0 = skip (keep the golden's size).
	DataVolGrowGB int
	// DataVolMount is the mpN slot of the golden's Docker-data volume to grow; "" → DefaultDataVolMount ("mp0").
	DataVolMount string
	// SysDataGrowGB grows the golden-carried SSD user-data volume (SysDataMount, default mp1, mounted at
	// /mnt/sys_drive = the controller's system_data_path) to the per-customer target. Same online,
	// grow-only mechanism as DataVolGrowGB. 0 = skip (keep the golden's small size — the volume is still
	// a separate mount, so the controller's "not a separate drive" warning clears regardless of grow).
	SysDataGrowGB int
	// SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1").
	SysDataMount string
	Mounts       []GuestMount  // additive mpN mounts (slice 7 may pass empty/test)
	KeepMAC      bool          // DR knob: keep the archived MAC (true) unless a source may be live
	BootTimeout  time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
}

// BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface
// (StartWarnings/WarningsRecognized) — the start step here is the same liveness-anchored boot.
type BringUpResult struct {
	VMID               int
	AssignedMAC        string // the guest's net0 MAC after identity reset (fresh for provision)
	Hostname           string
	Pass               bool
	Verified           string // "boot+running"
	Err                error
	StartedAt          time.Time
	Duration           time.Duration
	StartWarnings      []string
	WarningsRecognized bool
}

// IntentForRollbackDestroy builds the benign compensating-rollback teardown intent for a guest
// the agent created earlier in THIS journaled bring-up transaction: ClassGuestDestroy made benign
// by SameTxnCreated provenance (classify.go) — a rollback, not data loss. Gate-authorized unsigned
// but genuinely in-path (wrong provenance → pending_signature). Distinct from the restore-test's
// IntentForScratchDestroy (AgentTaggedScratch) — different audit label, same destroy machinery.
func IntentForRollbackDestroy(hostID string, vmid int) Intent {
	return Intent{
		Class:      ClassGuestDestroy,
		HostID:     hostID,
		GuestID:    strconv.Itoa(vmid),
		VMID:       vmid,
		Provenance: Provenance{SameTxnCreated: true},
		Source:     SourceOneShotJob,
	}
}

// RunBringUp runs one bring-up on the target VMID's queue lane (inherits §10 per-guest
// serialization). On success the guest is KEPT; on any mid-flight failure it is
// compensating-rolled-back (destroyed). The returned Err is the job verdict's error.
func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult {
	now := time.Now().UTC()
	res := BringUpResult{VMID: spec.VMID, Hostname: spec.Hostname, StartedAt: now}

	if spec.Archive == "" || spec.RestoreStorage == "" {
		res.Err = fmt.Errorf("reconcile: bring-up needs an archive and a restore storage")
		return res
	}
	if spec.VMID <= 0 {
		res.Err = fmt.Errorf("reconcile: bring-up needs a target VMID")
		return res
	}
	// Fence the reserved bands: never provision over the standing scratch (9999) or the
	// restore-test scratch band (990000–990009).
	if spec.VMID == 9999 || (spec.VMID >= 990000 && spec.VMID <= 990009) {
		res.Err = fmt.Errorf("reconcile: bring-up VMID %d is reserved (9999 / restore-test scratch band)", spec.VMID)
		return res
	}
	switch spec.Mode {
	case ModeProvision, ModeDRGuestLoss:
	default:
		res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode)
		return res
	}
	// DR needs the host runner: the structural-bind swap (4d) is a root pct op the API token
	// cannot perform. Refuse up front rather than fail after a restore (GL-5).
	if spec.Mode == ModeDRGuestLoss && e.hostRun == nil {
		res.Err = fmt.Errorf("reconcile: dr bring-up needs a host runner (the mp8/mp9 structural-bind swap is a root pct op) — wire EngineOptions.HostRunner")
		return res
	}

	ch := e.queue.Submit(spec.VMID, func() error {
		e.runBringUp(ctx, spec, &res)
		return res.Err
	})
	<-ch
	res.Duration = time.Since(now)
	return res
}

// runBringUp is the journaled body (runs on the target VMID's queue lane).
func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpResult) {
	// PROVISION is to a NEW VMID. Restoring OVER an existing guest is ClassRestoreOverwrite
	// (destructive, slice 10) — never this benign path. Refuse before journaling.
	lxc, err := e.api.ListLXC(ctx)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: bring-up list guests: %w", err)
		return
	}
	for _, g := range lxc {
		if g.VMID == spec.VMID {
			res.Err = fmt.Errorf("reconcile: bring-up VMID %d already exists (restore-over-existing is a signed op)", spec.VMID)
			return
		}
	}

	base := JournalEntry{OpID: e.bringUpOpID(spec.VMID), VMID: spec.VMID, Kind: bringUpKind, Rollback: true}

	// OWN the rollback BEFORE any mutation. From here a crash leaves an in-flight Rollback
	// entry; Recover destroys the vmid ONLY when the entry carries a restore UPID
	// (proof-of-launch — see Recover's no-UPID abandon path).
	e.append(withState(base, OpStarted))

	// Compensating rollback on every non-committed exit AFTER the restore launched (defer):
	// destroy the just-created guest. On success we set committed and KEEP it (the key
	// difference from the restore-test). `launched` is the proof-of-launch gate (campaign
	// pool-effects F1a): a restore that failed synchronously (no UPID — e.g. PVE refusing a
	// vmid that already holds a guest the pool-blind duplicate guard can't see) created
	// NOTHING, so the rollback must NEVER destroy the vmid — a pre-existing guest, possibly
	// another customer's, may sit there. This must hold WITHOUT the pool ACL (that 403 is
	// defense-in-depth, not the guard). The owning entry is then closed terminal-failed
	// in-process (nothing exists to recover).
	committed := false
	launched := false
	defer func() {
		if committed {
			return
		}
		if !launched {
			e.append(withState(base, OpFailed))
			return
		}
		e.rollbackBringUp(ctx, base)
	}()

	// 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike).
	//    DR (GL-5): the customer archive carries the two STRUCTURAL host-bind mountpoints (mp8
	//    parent bind + mp9 bootstrap bind) that a restore under the privsep token cannot recreate
	//    — without overrides the whole restore FAILS ("restoring 'mp8' to bind mount is only
	//    possible for root"). Synthesize throwaway-volume overrides for the two known-constant mpN
	//    via the shared format helper (bindMountOverrides' is-a-bind filter is for reading real
	//    configs, which DR by definition cannot do — the guest is gone); step 4d swaps the real
	//    binds back post-restore. An archive that LACKS one of them (older backup) is fine: the
	//    override simply creates that mpN at restore and 4d normalizes it — the end state is
	//    identical (C3). Provision stays override-free (nil): the golden has no mp8/mp9 (the
	//    back-half adds them post-bring-up) — that asymmetry is the whole GL-5 bug.
	var overrides map[string]string
	if spec.Mode == ModeDRGuestLoss {
		overrides = map[string]string{
			structuralParentSlot: throwawayVolumeOverride(spec.RestoreStorage, structuralParentDir),
			structuralBootSlot:   throwawayVolumeOverride(spec.RestoreStorage, structuralBootGuestPath),
		}
	}
	upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
		VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
		MountOverrides: overrides,
	})
	if err != nil {
		// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
		res.Err = fmt.Errorf("reconcile: bring-up restore: %w", err)
		return
	}
	// Proof-of-launch: the POST was accepted — from here a failure means a half-built guest the
	// compensating rollback (or Recover, via the journaled UPID) must destroy. Accepted residual:
	// a crash in the one-statement window before the UPID is journaled leaks a half-built guest
	// that Recover won't destroy — cleanable, and preferable to destroying an innocent guest.
	launched = true
	e.append(withUPID(base, upid, OpTaskRunning))
	if _, err := e.waitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
		res.Err = fmt.Errorf("reconcile: bring-up restore task: %w", err)
		return
	}

	// 2. Read as-restored config (the net0 the MAC handling keys off).
	cfg, err := e.api.GuestConfig(ctx, spec.VMID)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: bring-up read config: %w", err)
		return
	}

	// 3+5 coalesced (F4): identity reset + cores/mem sizing + additive mounts in ONE config PUT,
	// with the bounded retry that fires ONLY on the transient PVE config-lock 500.
	if params := buildBringUpConfig(spec, cfg); len(params) > 0 {
		if err := e.setConfigWithLockRetry(ctx, spec.VMID, params); err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up config: %w", err)
			return
		}
	}

	// 4. rootfs grow-only resize as its OWN call (F4: kept separate from the config PUT).
	if spec.RootfsGrowGB > 0 {
		rupid, err := e.api.ResizeLXC(ctx, spec.VMID, "rootfs", fmt.Sprintf("+%dG", spec.RootfsGrowGB))
		if err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up resize: %w", err)
			return
		}
		if _, err := e.waitTask(ctx, rupid, proxmox.WaitOptions{}); err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up resize task: %w", err)
			return
		}
	}

	// 4b. Grow the golden-carried Docker-data volume (mp0) to the per-customer target. Grow-only,
	//     online (storage-split B4); its OWN call like the rootfs resize. The volume + baked images
	//     came in with the restore, so we grow it rather than attach a fresh one that would shadow
	//     the baked images.
	if spec.DataVolGrowGB > 0 {
		mount := spec.DataVolMount
		if mount == "" {
			mount = DefaultDataVolMount
		}
		dupid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.DataVolGrowGB))
		if err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up data-volume resize (%s): %w", mount, err)
			return
		}
		if _, err := e.waitTask(ctx, dupid, proxmox.WaitOptions{}); err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up data-volume resize task (%s): %w", mount, err)
			return
		}
	}

	// 4c. Grow the golden-carried SSD user-data volume (mp1, /mnt/sys_drive = the controller's
	//     system_data_path) to the per-customer target. Same shape as the Docker-data grow: grow-only,
	//     online, its OWN call. The volume came in with the restore (separate mount, backup=1), so we
	//     grow it rather than attach a fresh one.
	if spec.SysDataGrowGB > 0 {
		mount := spec.SysDataMount
		if mount == "" {
			mount = DefaultSysDataMount
		}
		supid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.SysDataGrowGB))
		if err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up sys-data resize (%s): %w", mount, err)
			return
		}
		if _, err := e.waitTask(ctx, supid, proxmox.WaitOptions{}); err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up sys-data resize task (%s): %w", mount, err)
			return
		}
	}

	// 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the
	//     restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR
	//     guest carries no unusedN residue. Bind-mount pct sets are root@pam-only — the swap goes
	//     through the host runner, never the API. Runs BEFORE start so the first boot already sees
	//     the real binds (the golden's baked bootstrap unit + the drives parent). A failure here is
	//     surfaced with the exact mpN state and rolls back per the envelope (committed is still
	//     false) — never a silent half-wired success (C2).
	if spec.Mode == ModeDRGuestLoss {
		if err := e.swapStructuralBinds(ctx, spec, res); err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up structural-bind swap: %w", err)
			return
		}
	}

	// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
	if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
		res.AssignedMAC = net0MAC(cfg2)
	}

	// 6. Start LINK-UP (ClassStart). The VERDICT is liveness (waitRunning), NEVER the start
	//    exitstatus — same liveness-anchoring as the restore-test fix; AllowWarnings so a benign
	//    start advisory (systemd-nesting) is surfaced, not failed.
	startUPID, err := e.api.Start(ctx, spec.VMID)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: bring-up start: %w", err)
		return
	}
	if startUPID != "" {
		st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
		if err != nil {
			res.Err = fmt.Errorf("reconcile: bring-up start task: %w", err)
			return
		}
		if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
			tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
			if logErr != nil {
				e.logger.Warn("bring-up: could not read start-task log for warnings", "vmid", spec.VMID, "err", logErr)
			}
			res.StartWarnings = extractWarningLines(tail)
			res.WarningsRecognized = warningsRecognized(res.StartWarnings)
		}
	}
	bootTO := spec.BootTimeout
	if bootTO <= 0 {
		bootTO = DefaultBootTimeout
	}
	if err := e.waitRunning(ctx, spec.VMID, bootTO); err != nil {
		res.Err = err
		return
	}

	// 6b. Re-assert pool membership (campaign-2 R2). `pct restore --pool` sets membership only at
	//     CREATE; a restore OVER AN EXISTING VMID (host-loss finale) does not re-apply it, silently
	//     dropping the guest from the pool and 403-ing the NEXT restore-test/DR. Idempotent on a
	//     fresh-VMID restore that already got membership. This runs AFTER liveness is proven: a
	//     pool-add hiccup is surfaced LOUD as a warning but must NOT flip a healthy, running guest's
	//     verdict to fail (membership matters for the next op, not this guest's boot).
	if spec.Pool != "" {
		if err := e.api.PoolAddVMID(ctx, spec.Pool, spec.VMID); err != nil {
			e.logger.Error("bring-up: pool membership re-assert FAILED (next restore-test/DR may 403); guest is healthy",
				"vmid", spec.VMID, "pool", spec.Pool, "err", err)
			res.StartWarnings = append(res.StartWarnings, fmt.Sprintf("pool re-assert failed (pool=%s): %v", spec.Pool, err))
		} else {
			e.logger.Info("bring-up: pool membership re-asserted", "vmid", spec.VMID, "pool", spec.Pool)
		}
	}

	// 7. Success — KEEP the guest; mark the owning entry terminal so Recover ignores it.
	res.Pass = true
	res.Verified = "boot+running"
	committed = true
	e.append(withState(base, OpSucceeded))
}

// swapStructuralBinds is bring-up step 4d (DR only): set mp8/mp9 to the real host binds via the
// root runner (one slot per call so an error names the exact mpN that failed — C2), then delete
// the displaced throwaway volumes PVE parked as unusedN. The mp9 host dir is created first (pct
// validates the bind source; agent-owned path, plain MkdirAll like the back-half's own bootstrap
// dir) — idempotent: on a same-host guest-loss the dir usually still exists WITH bootstrap.json,
// which the swap must not touch. The unusedN delete goes through the API config PUT
// (VM.Config.Disk + Datastore.Allocate cover it); if the scoped token refuses, the residue is
// logged LOUDLY + surfaced as a result warning and the bring-up continues — a correctly-wired
// guest with a stray volume beats a rollback, and privileges are never widened silently.
func (e *Engine) swapStructuralBinds(ctx context.Context, spec BringUpSpec, res *BringUpResult) error {
	bootDir := structuralBootHostDir(e.stateDir, spec.VMID)
	if err := os.MkdirAll(bootDir, 0o700); err != nil {
		return fmt.Errorf("mp9 bootstrap host dir %s: %w", bootDir, err)
	}
	if _, stderr, err := e.hostRun.Run(ctx, "mkdir", "-p", structuralParentDir); err != nil {
		return fmt.Errorf("parent dir %s: %w: %s", structuralParentDir, err, stderr)
	}
	parentSpec := structuralParentDir + ",mp=" + structuralParentDir
	if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralParentSlot, parentSpec); err != nil {
		return fmt.Errorf("set %s (parent bind; neither bind landed): %w: %s", structuralParentSlot, err, stderr)
	}
	bootSpec := bootDir + ",mp=" + structuralBootGuestPath + ",ro=1"
	if _, stderr, err := e.hostRun.Run(ctx, "pct", "set", strconv.Itoa(spec.VMID), "-"+structuralBootSlot, bootSpec); err != nil {
		return fmt.Errorf("set %s (bootstrap bind; %s already landed): %w: %s", structuralBootSlot, structuralParentSlot, err, stderr)
	}
	e.logger.Info("bring-up: structural binds swapped in", "vmid", spec.VMID,
		structuralParentSlot, parentSpec, structuralBootSlot, bootSpec)

	// The displaced throwaway volumes now sit as unusedN — read the config and delete them.
	cfg, err := e.api.GuestConfig(ctx, spec.VMID)
	if err != nil {
		return fmt.Errorf("read config after bind swap: %w", err)
	}
	var unused []string
	for k := range cfg.Unused() {
		unused = append(unused, k)
	}
	if len(unused) == 0 {
		return nil
	}
	sort.Strings(unused)
	if err := e.setConfigWithLockRetry(ctx, spec.VMID, map[string]string{"delete": strings.Join(unused, ",")}); err != nil {
		e.logger.Error("bring-up: could not delete displaced throwaway volumes (guest is correctly wired; residue remains)",
			"vmid", spec.VMID, "unused", unused, "err", err)
		res.StartWarnings = append(res.StartWarnings,
			fmt.Sprintf("structural-bind swap: displaced volumes not deleted (%s): %v", strings.Join(unused, ","), err))
		return nil
	}
	e.logger.Info("bring-up: displaced throwaway volumes deleted", "vmid", spec.VMID, "unused", unused)
	return nil
}

// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
// provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
// was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not
// assumed. On any teardown failure it leaves the entry in-flight so Recover reaps the guest later
// (via the journaled UPID) — never force-destroys.
func (e *Engine) rollbackBringUp(ctx context.Context, base JournalEntry) {
	tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
	defer cancel()

	dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, base.VMID), nil)
	if !dec.Allowed {
		e.logger.Error("bring-up: rollback destroy refused by gate (unexpected); left for Recover",
			"vmid", base.VMID, "reason", dec.Reason)
		return
	}
	upid, err := e.api.DestroyLXC(tctx, base.VMID)
	if err != nil {
		e.logger.Error("bring-up: rollback destroy failed; left for Recover", "vmid", base.VMID, "err", err)
		return
	}
	if _, err := e.waitTask(tctx, upid, proxmox.WaitOptions{}); err != nil {
		e.logger.Error("bring-up: rollback destroy task failed; left for Recover", "vmid", base.VMID, "err", err)
		return
	}
	e.append(withState(base, OpSucceeded))
	e.logger.Warn("bring-up: rolled back (destroyed half-built guest)", "vmid", base.VMID)
}

// buildBringUpConfig assembles the coalesced config PUT params per the scenario-specific identity
// policy (doc 03 §9). Provision: fresh MAC (strip hwaddr → PVE regenerates, F1) + hostname.
// DR: preserve continuity — keep MAC (unless KeepMAC=false: a source may be live) and keep
// hostname (no force-reset). Both: cores/mem sizing + additive mpN mounts. machine-id and SSH
// host keys are NOT touched here — they regenerate guest-side on first boot (golden bake + the
// baked first-boot unit), keeping the agent's front half host-side-only.
func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]string {
	params := map[string]string{}

	resetMAC := spec.Mode == ModeProvision || (spec.Mode == ModeDRGuestLoss && !spec.KeepMAC)
	if resetMAC {
		if net0, ok := cfg.Nets()["net0"]; ok && net0 != "" {
			params["net0"] = withoutHwaddr(net0) // omit hwaddr → PVE generates a fresh MAC (F1)
		}
	}
	if spec.Mode == ModeProvision && spec.Hostname != "" {
		params["hostname"] = spec.Hostname
	}
	if spec.Cores > 0 {
		params["cores"] = strconv.Itoa(spec.Cores)
	}
	if spec.MemoryMB > 0 {
		params["memory"] = strconv.Itoa(spec.MemoryMB)
	}
	for i, m := range spec.Mounts {
		// backup=1 for data-bearing mounts: extra LXC mountpoints default to backup=0 = EXCLUDED
		// from vzdump/PBS (storage-split B3), which would silently drop their DBs from the snapshot.
		spec := fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint)
		if m.Backup {
			spec += ",backup=1"
		}
		params[fmt.Sprintf("mp%d", i)] = spec
	}
	return params
}

// setConfigWithLockRetry issues the coalesced config PUT, retrying ONLY the transient PVE
// config-lock 500 (F4) with bounded backoff. A non-lock error (any other 500 included) fails
// immediately — never retry a real error. The slice-4 per-guest serializer prevents cross-op
// contention; this covers PVE releasing its own async config lock within one job.
func (e *Engine) setConfigWithLockRetry(ctx context.Context, vmid int, params map[string]string) error {
	var lastErr error
	for attempt := 1; attempt <= configLockMaxAttempts; attempt++ {
		if _, err := e.api.SetConfig(ctx, vmid, params); err == nil {
			return nil
		} else if !pveConfigLock(err) {
			return err // real error — never retry
		} else {
			lastErr = err
			e.logger.Warn("bring-up: transient PVE config-lock; retrying",
				"vmid", vmid, "attempt", attempt, "max", configLockMaxAttempts, "err", err)
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(configLockBackoff):
		}
	}
	return fmt.Errorf("reconcile: bring-up config still lock-contended after %d attempts: %w", configLockMaxAttempts, lastErr)
}

// pveConfigLock reports whether err is the transient PVE config-lock 500 (F4) — and ONLY that.
// The lock surfaces as a proxmox.APIError 500 whose body carries the lock signature.
func pveConfigLock(err error) bool {
	var ae *proxmox.APIError
	if !errors.As(err, &ae) || ae.StatusCode != 500 {
		return false
	}
	b := strings.ToLower(ae.Body)
	return strings.Contains(b, "can't lock file") || strings.Contains(b, "got timeout")
}

// pveAlreadyExists reports whether err is PVE's synchronous refusal to create over an existing
// vmid ("CT <vmid> already exists on node '<node>'" — an APIError 500, observed live in the
// pool-effects campaign). By construction such a refusal returned no UPID: nothing was created.
// Used by the restore-test band-advance (F2) to distinguish "band vmid occupied by a guest the
// pool-blind list can't see" from a real restore failure — never misclassify the latter.
func pveAlreadyExists(err error) bool {
	var ae *proxmox.APIError
	if !errors.As(err, &ae) || ae.StatusCode != 500 {
		return false
	}
	return strings.Contains(strings.ToLower(ae.Body), "already exists")
}

// waitTask waits a (possibly empty) UPID — "" is the clean synchronous path.
func (e *Engine) waitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) {
	if upid == "" {
		return proxmox.TaskStatus{}, nil
	}
	return e.api.WaitTask(ctx, upid, opts)
}

func (e *Engine) bringUpOpID(vmid int) string {
	return "bring-up-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq)
}

// withoutHwaddr strips the hwaddr token from a netN config string. PUTting a netN with NO hwaddr
// makes PVE generate a fresh MAC (slice-7 spike F1: a restore preserves the archived MAC, so a
// provision MUST strip it to avoid a fleet-wide MAC collision). Mirrors withLinkDown's approach.
func withoutHwaddr(netN string) string {
	parts := strings.Split(netN, ",")
	out := parts[:0]
	for _, p := range parts {
		if p == "" || strings.HasPrefix(p, "hwaddr=") {
			continue
		}
		out = append(out, p)
	}
	return strings.Join(out, ",")
}

// net0MAC extracts the hwaddr from a guest's net0 config ("" if absent).
func net0MAC(cfg proxmox.GuestConfig) string {
	net0, ok := cfg.Nets()["net0"]
	if !ok {
		return ""
	}
	for _, p := range strings.Split(net0, ",") {
		if strings.HasPrefix(p, "hwaddr=") {
			return strings.TrimPrefix(p, "hwaddr=")
		}
	}
	return ""
}