slice 7 Phase 1: unified bring-up reconcile job (provision + guest-loss DR) (v0.8.0)

The shared front half of provision and guest-loss DR as a journaled reconcile job (internal/reconcile/bringup.go), mirroring the restore-test's crash-safety but keeping the guest on success and applying a scenario-specific identity policy. Agent-only; no hub/wire change. Grounded by the slice-7 bring-up spike (commit 3342993): F1/F3/F4. - RunBringUp: restore -> reset identity -> size -> attach mounts -> start link-up; verdict is liveness (waitRunning), success KEEPS the guest. - identity policy: provision = fresh MAC (net0 sans hwaddr -> PVE regen) + hostname, host-side; machine-id/host-keys regenerate guest-side (systemd + baked golden unit). dr_guest_loss = preserve continuity (keep hostname; keep MAC unless KeepMAC=false). - compensating rollback: mid-flight failure destroys the just-created guest (SameTxnCreated provenance, gated); new Rollback journal flag + Recover.recoverBringUp reap a half-built guest from a crash. - F4: coalesced config PUT + bounded retry on the transient PVE config-lock 500 only. - --selftest=bring-up (mode/archive/vmid/hostname/keep). - configs/build-golden.sh: validated golden recipe incl. the F3 first-boot host-key unit. - doc-03 §9 + identity-reset settled/implemented. Deferred (stated): provisioning back half -> slice 8; host-loss DR + escrow consumption and the BringUpSpec source (hub desired-state) -> slice 10. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 21:27:49 +02:00
parent 9f6753de0f
commit 57405c1a99
10 changed files with 1184 additions and 73 deletions
@@ -0,0 +1,393 @@
+package reconcile
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// The unified bring-up reconcile job (doc 03 §9, slice 7 Phase 1). It is the shared FRONT
+// HALF of provision and guest-loss DR: restore an archive → reset identity (scenario-specific)
+// → size → attach mounts → start LINK-UP. It mirrors RunRestoreTest's crash-safety (journal the
+// owning entry BEFORE any mutation; Recover reaps a leak via ListLXC; idempotent, fail-safe) but
+// DIFFERS in two load-bearing ways: it KEEPS the guest on success (no teardown), and its
+// identity policy is scenario-specific (provision = fresh identity; DR = preserve continuity).
+//
+// Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993): F1 (restore
+// preserves the archived MAC → provision MUST reset it), F3 (machine-id + SSH host keys
+// regenerate guest-side on first boot from the clean golden + its baked unit — the agent does
+// NOT touch guest internals here), F4 (a transient PVE config-lock 500 → bounded retry).
+//
+// Out of scope (deferred — see REPORT): the provisioning BACK HALF (controller deploy, bootstrap,
+// per-guest token mint → slice 8); host-loss DR + escrow consumption → slice 10; the SOURCE of a
+// BringUpSpec (hub desired-state) → slice 10 (this job takes the spec as input).
+
+// BringUpMode selects the identity policy + archive semantics.
+type BringUpMode string
+
+const (
+	// ModeProvision: from the golden base — a NEW guest, so reset identity fully.
+	ModeProvision BringUpMode = "provision"
+	// ModeDRGuestLoss: from a customer backup — CONTINUE the customer's world, so preserve
+	// continuity identity (hostname, host keys, and by default MAC), reset only what collides.
+	ModeDRGuestLoss BringUpMode = "dr_guest_loss"
+)
+
+const bringUpKind = "bring_up"
+
+// configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
+// tests can shrink it (the production value gives PVE time to release its async config lock).
+const configLockMaxAttempts = 5
+
+var configLockBackoff = 1500 * time.Millisecond
+
+// GuestMount is one additive mountpoint to attach as mpN. Defined minimally now; slice 10 wires
+// the hub storage manifest into this (do NOT couple to a not-yet-existing hub desired-state type).
+type GuestMount struct {
+	Storage    string // PVE storage id (e.g. "local-lvm")
+	SizeGB     int    // new-volume size in GiB
+	MountPoint string // in-guest path (e.g. "/mnt/data")
+}
+
+// BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10
+// hub desired-state); this job does not decide WHAT to provision.
+type BringUpSpec struct {
+	Mode           BringUpMode   // provision | dr_guest_loss
+	Archive        string        // source volid (golden for provision; customer backup for DR)
+	VMID           int           // caller-provided target VMID (NOT the restore-test band / 9999)
+	RestoreStorage string        // rootfs target storage
+	Hostname       string        // hostname to set (provision); ignored for DR (continuity)
+	Cores          int           // 0 = leave as restored
+	MemoryMB       int           // 0 = leave as restored
+	RootfsGrowGB   int           // optional grow-only rootfs resize (0 = skip)
+	Mounts         []GuestMount  // additive mpN mounts (slice 7 may pass empty/test)
+	KeepMAC        bool          // DR knob: keep the archived MAC (true) unless a source may be live
+	BootTimeout    time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
+}
+
+// BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface
+// (StartWarnings/WarningsRecognized) — the start step here is the same liveness-anchored boot.
+type BringUpResult struct {
+	VMID               int
+	AssignedMAC        string // the guest's net0 MAC after identity reset (fresh for provision)
+	Hostname           string
+	Pass               bool
+	Verified           string // "boot+running"
+	Err                error
+	StartedAt          time.Time
+	Duration           time.Duration
+	StartWarnings      []string
+	WarningsRecognized bool
+}
+
+// IntentForRollbackDestroy builds the benign compensating-rollback teardown intent for a guest
+// the agent created earlier in THIS journaled bring-up transaction: ClassGuestDestroy made benign
+// by SameTxnCreated provenance (classify.go) — a rollback, not data loss. Gate-authorized unsigned
+// but genuinely in-path (wrong provenance → pending_signature). Distinct from the restore-test's
+// IntentForScratchDestroy (AgentTaggedScratch) — different audit label, same destroy machinery.
+func IntentForRollbackDestroy(hostID string, vmid int) Intent {
+	return Intent{
+		Class:      ClassGuestDestroy,
+		HostID:     hostID,
+		GuestID:    strconv.Itoa(vmid),
+		VMID:       vmid,
+		Provenance: Provenance{SameTxnCreated: true},
+		Source:     SourceOneShotJob,
+	}
+}
+
+// RunBringUp runs one bring-up on the target VMID's queue lane (inherits §10 per-guest
+// serialization). On success the guest is KEPT; on any mid-flight failure it is
+// compensating-rolled-back (destroyed). The returned Err is the job verdict's error.
+func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult {
+	now := time.Now().UTC()
+	res := BringUpResult{VMID: spec.VMID, Hostname: spec.Hostname, StartedAt: now}
+
+	if spec.Archive == "" || spec.RestoreStorage == "" {
+		res.Err = fmt.Errorf("reconcile: bring-up needs an archive and a restore storage")
+		return res
+	}
+	if spec.VMID <= 0 {
+		res.Err = fmt.Errorf("reconcile: bring-up needs a target VMID")
+		return res
+	}
+	// Fence the reserved bands: never provision over the standing scratch (9999) or the
+	// restore-test scratch band (990000–990009).
+	if spec.VMID == 9999 || (spec.VMID >= 990000 && spec.VMID <= 990009) {
+		res.Err = fmt.Errorf("reconcile: bring-up VMID %d is reserved (9999 / restore-test scratch band)", spec.VMID)
+		return res
+	}
+	switch spec.Mode {
+	case ModeProvision, ModeDRGuestLoss:
+	default:
+		res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode)
+		return res
+	}
+
+	ch := e.queue.Submit(spec.VMID, func() error {
+		e.runBringUp(ctx, spec, &res)
+		return res.Err
+	})
+	<-ch
+	res.Duration = time.Since(now)
+	return res
+}
+
+// runBringUp is the journaled body (runs on the target VMID's queue lane).
+func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpResult) {
+	// PROVISION is to a NEW VMID. Restoring OVER an existing guest is ClassRestoreOverwrite
+	// (destructive, slice 10) — never this benign path. Refuse before journaling.
+	lxc, err := e.api.ListLXC(ctx)
+	if err != nil {
+		res.Err = fmt.Errorf("reconcile: bring-up list guests: %w", err)
+		return
+	}
+	for _, g := range lxc {
+		if g.VMID == spec.VMID {
+			res.Err = fmt.Errorf("reconcile: bring-up VMID %d already exists (restore-over-existing is a signed op)", spec.VMID)
+			return
+		}
+	}
+
+	base := JournalEntry{OpID: e.bringUpOpID(spec.VMID), VMID: spec.VMID, Kind: bringUpKind, Rollback: true}
+
+	// OWN the rollback BEFORE any mutation. From here a crash leaves an in-flight Rollback
+	// entry meaning "VMID may be a half-built guest → destroy it" (Recover.recoverBringUp).
+	e.append(withState(base, OpStarted))
+
+	// Compensating rollback on EVERY non-committed exit (defer): destroy the just-created
+	// guest. On success we set committed and KEEP it (the key difference from the restore-test).
+	committed := false
+	defer func() {
+		if committed {
+			return
+		}
+		e.rollbackBringUp(ctx, base)
+	}()
+
+	// 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike).
+	upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
+		VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage,
+	})
+	if err != nil {
+		res.Err = fmt.Errorf("reconcile: bring-up restore: %w", err)
+		return
+	}
+	e.append(withUPID(base, upid, OpTaskRunning))
+	if _, err := e.waitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
+		res.Err = fmt.Errorf("reconcile: bring-up restore task: %w", err)
+		return
+	}
+
+	// 2. Read as-restored config (the net0 the MAC handling keys off).
+	cfg, err := e.api.GuestConfig(ctx, spec.VMID)
+	if err != nil {
+		res.Err = fmt.Errorf("reconcile: bring-up read config: %w", err)
+		return
+	}
+
+	// 3+5 coalesced (F4): identity reset + cores/mem sizing + additive mounts in ONE config PUT,
+	// with the bounded retry that fires ONLY on the transient PVE config-lock 500.
+	if params := buildBringUpConfig(spec, cfg); len(params) > 0 {
+		if err := e.setConfigWithLockRetry(ctx, spec.VMID, params); err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up config: %w", err)
+			return
+		}
+	}
+
+	// 4. rootfs grow-only resize as its OWN call (F4: kept separate from the config PUT).
+	if spec.RootfsGrowGB > 0 {
+		rupid, err := e.api.ResizeLXC(ctx, spec.VMID, "rootfs", fmt.Sprintf("+%dG", spec.RootfsGrowGB))
+		if err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up resize: %w", err)
+			return
+		}
+		if _, err := e.waitTask(ctx, rupid, proxmox.WaitOptions{}); err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up resize task: %w", err)
+			return
+		}
+	}
+
+	// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
+	if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
+		res.AssignedMAC = net0MAC(cfg2)
+	}
+
+	// 6. Start LINK-UP (ClassStart). The VERDICT is liveness (waitRunning), NEVER the start
+	//    exitstatus — same liveness-anchoring as the restore-test fix; AllowWarnings so a benign
+	//    start advisory (systemd-nesting) is surfaced, not failed.
+	startUPID, err := e.api.Start(ctx, spec.VMID)
+	if err != nil {
+		res.Err = fmt.Errorf("reconcile: bring-up start: %w", err)
+		return
+	}
+	if startUPID != "" {
+		st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
+		if err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up start task: %w", err)
+			return
+		}
+		if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
+			tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
+			if logErr != nil {
+				e.logger.Warn("bring-up: could not read start-task log for warnings", "vmid", spec.VMID, "err", logErr)
+			}
+			res.StartWarnings = extractWarningLines(tail)
+			res.WarningsRecognized = warningsRecognized(res.StartWarnings)
+		}
+	}
+	bootTO := spec.BootTimeout
+	if bootTO <= 0 {
+		bootTO = DefaultBootTimeout
+	}
+	if err := e.waitRunning(ctx, spec.VMID, bootTO); err != nil {
+		res.Err = err
+		return
+	}
+
+	// 7. Success — KEEP the guest; mark the owning entry terminal so Recover ignores it.
+	res.Pass = true
+	res.Verified = "boot+running"
+	committed = true
+	e.append(withState(base, OpSucceeded))
+}
+
+// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
+// provenance) and records the owning entry terminal. Mirrors teardownScratch: ALWAYS attempts the
+// destroy (idempotent — a restore-POST failure that created no guest just errors harmlessly and is
+// left in-flight for Recover, which existence-checks). On any teardown failure it leaves the entry
+// in-flight so Recover reaps the guest later — never force-destroys.
+func (e *Engine) rollbackBringUp(ctx context.Context, base JournalEntry) {
+	tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
+	defer cancel()
+
+	dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, base.VMID), nil)
+	if !dec.Allowed {
+		e.logger.Error("bring-up: rollback destroy refused by gate (unexpected); left for Recover",
+			"vmid", base.VMID, "reason", dec.Reason)
+		return
+	}
+	upid, err := e.api.DestroyLXC(tctx, base.VMID)
+	if err != nil {
+		e.logger.Error("bring-up: rollback destroy failed; left for Recover", "vmid", base.VMID, "err", err)
+		return
+	}
+	if _, err := e.waitTask(tctx, upid, proxmox.WaitOptions{}); err != nil {
+		e.logger.Error("bring-up: rollback destroy task failed; left for Recover", "vmid", base.VMID, "err", err)
+		return
+	}
+	e.append(withState(base, OpSucceeded))
+	e.logger.Warn("bring-up: rolled back (destroyed half-built guest)", "vmid", base.VMID)
+}
+
+// buildBringUpConfig assembles the coalesced config PUT params per the scenario-specific identity
+// policy (doc 03 §9). Provision: fresh MAC (strip hwaddr → PVE regenerates, F1) + hostname.
+// DR: preserve continuity — keep MAC (unless KeepMAC=false: a source may be live) and keep
+// hostname (no force-reset). Both: cores/mem sizing + additive mpN mounts. machine-id and SSH
+// host keys are NOT touched here — they regenerate guest-side on first boot (golden bake + the
+// baked first-boot unit), keeping the agent's front half host-side-only.
+func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]string {
+	params := map[string]string{}
+
+	resetMAC := spec.Mode == ModeProvision || (spec.Mode == ModeDRGuestLoss && !spec.KeepMAC)
+	if resetMAC {
+		if net0, ok := cfg.Nets()["net0"]; ok && net0 != "" {
+			params["net0"] = withoutHwaddr(net0) // omit hwaddr → PVE generates a fresh MAC (F1)
+		}
+	}
+	if spec.Mode == ModeProvision && spec.Hostname != "" {
+		params["hostname"] = spec.Hostname
+	}
+	if spec.Cores > 0 {
+		params["cores"] = strconv.Itoa(spec.Cores)
+	}
+	if spec.MemoryMB > 0 {
+		params["memory"] = strconv.Itoa(spec.MemoryMB)
+	}
+	for i, m := range spec.Mounts {
+		params[fmt.Sprintf("mp%d", i)] = fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint)
+	}
+	return params
+}
+
+// setConfigWithLockRetry issues the coalesced config PUT, retrying ONLY the transient PVE
+// config-lock 500 (F4) with bounded backoff. A non-lock error (any other 500 included) fails
+// immediately — never retry a real error. The slice-4 per-guest serializer prevents cross-op
+// contention; this covers PVE releasing its own async config lock within one job.
+func (e *Engine) setConfigWithLockRetry(ctx context.Context, vmid int, params map[string]string) error {
+	var lastErr error
+	for attempt := 1; attempt <= configLockMaxAttempts; attempt++ {
+		if _, err := e.api.SetConfig(ctx, vmid, params); err == nil {
+			return nil
+		} else if !pveConfigLock(err) {
+			return err // real error — never retry
+		} else {
+			lastErr = err
+			e.logger.Warn("bring-up: transient PVE config-lock; retrying",
+				"vmid", vmid, "attempt", attempt, "max", configLockMaxAttempts, "err", err)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(configLockBackoff):
+		}
+	}
+	return fmt.Errorf("reconcile: bring-up config still lock-contended after %d attempts: %w", configLockMaxAttempts, lastErr)
+}
+
+// pveConfigLock reports whether err is the transient PVE config-lock 500 (F4) — and ONLY that.
+// The lock surfaces as a proxmox.APIError 500 whose body carries the lock signature.
+func pveConfigLock(err error) bool {
+	var ae *proxmox.APIError
+	if !errors.As(err, &ae) || ae.StatusCode != 500 {
+		return false
+	}
+	b := strings.ToLower(ae.Body)
+	return strings.Contains(b, "can't lock file") || strings.Contains(b, "got timeout")
+}
+
+// waitTask waits a (possibly empty) UPID — "" is the clean synchronous path.
+func (e *Engine) waitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) {
+	if upid == "" {
+		return proxmox.TaskStatus{}, nil
+	}
+	return e.api.WaitTask(ctx, upid, opts)
+}
+
+func (e *Engine) bringUpOpID(vmid int) string {
+	return "bring-up-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq)
+}
+
+// withoutHwaddr strips the hwaddr token from a netN config string. PUTting a netN with NO hwaddr
+// makes PVE generate a fresh MAC (slice-7 spike F1: a restore preserves the archived MAC, so a
+// provision MUST strip it to avoid a fleet-wide MAC collision). Mirrors withLinkDown's approach.
+func withoutHwaddr(netN string) string {
+	parts := strings.Split(netN, ",")
+	out := parts[:0]
+	for _, p := range parts {
+		if p == "" || strings.HasPrefix(p, "hwaddr=") {
+			continue
+		}
+		out = append(out, p)
+	}
+	return strings.Join(out, ",")
+}
+
+// net0MAC extracts the hwaddr from a guest's net0 config ("" if absent).
+func net0MAC(cfg proxmox.GuestConfig) string {
+	net0, ok := cfg.Nets()["net0"]
+	if !ok {
+		return ""
+	}
+	for _, p := range strings.Split(net0, ",") {
+		if strings.HasPrefix(p, "hwaddr=") {
+			return strings.TrimPrefix(p, "hwaddr=")
+		}
+	}
+	return ""
+}