package reconcile import ( "context" "errors" "fmt" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // The unified bring-up reconcile job (doc 03 §9, slice 7 Phase 1). It is the shared FRONT // HALF of provision and guest-loss DR: restore an archive → reset identity (scenario-specific) // → size → attach mounts → start LINK-UP. It mirrors RunRestoreTest's crash-safety (journal the // owning entry BEFORE any mutation; Recover reaps a leak via ListLXC; idempotent, fail-safe) but // DIFFERS in two load-bearing ways: it KEEPS the guest on success (no teardown), and its // identity policy is scenario-specific (provision = fresh identity; DR = preserve continuity). // // Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993): F1 (restore // preserves the archived MAC → provision MUST reset it), F3 (machine-id + SSH host keys // regenerate guest-side on first boot from the clean golden + its baked unit — the agent does // NOT touch guest internals here), F4 (a transient PVE config-lock 500 → bounded retry). // // Out of scope (deferred — see REPORT): the provisioning BACK HALF (controller deploy, bootstrap, // per-guest token mint → slice 8); host-loss DR + escrow consumption → slice 10; the SOURCE of a // BringUpSpec (hub desired-state) → slice 10 (this job takes the spec as input). // BringUpMode selects the identity policy + archive semantics. type BringUpMode string const ( // ModeProvision: from the golden base — a NEW guest, so reset identity fully. ModeProvision BringUpMode = "provision" // ModeDRGuestLoss: from a customer backup — CONTINUE the customer's world, so preserve // continuity identity (hostname, host keys, and by default MAC), reset only what collides. ModeDRGuestLoss BringUpMode = "dr_guest_loss" ) const bringUpKind = "bring_up" // DefaultDataVolMount is the mpN slot the golden bakes the Docker-data volume (/var/lib/docker) at. const DefaultDataVolMount = "mp0" // configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so // tests can shrink it (the production value gives PVE time to release its async config lock). const configLockMaxAttempts = 5 var configLockBackoff = 1500 * time.Millisecond // GuestMount is one additive mountpoint to attach as mpN. Defined minimally now; slice 10 wires // the hub storage manifest into this (do NOT couple to a not-yet-existing hub desired-state type). type GuestMount struct { Storage string // PVE storage id (e.g. "local-lvm") SizeGB int // new-volume size in GiB MountPoint string // in-guest path (e.g. "/mnt/data") // Backup includes this mountpoint in vzdump/PBS. MANDATORY for any data-bearing mount (DB // volumes), because extra LXC mountpoints default to backup=0 = EXCLUDED from the snapshot // (storage-split finding B3). The Docker-data volume normally rides in from the golden archive // (already backup=1) and is grown via DataVolGrowGB rather than attached here, but any data // mount attached through spec.Mounts MUST set this or its contents silently fall out of PBS. Backup bool } // BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10 // hub desired-state); this job does not decide WHAT to provision. type BringUpSpec struct { Mode BringUpMode // provision | dr_guest_loss Archive string // source volid (golden for provision; customer backup for DR) VMID int // caller-provided target VMID (NOT the restore-test band / 9999) RestoreStorage string // rootfs target storage Hostname string // hostname to set (provision); ignored for DR (continuity) Cores int // 0 = leave as restored MemoryMB int // 0 = leave as restored RootfsGrowGB int // optional grow-only rootfs resize (0 = skip) // DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the // per-customer target. The golden ships a small data volume with the baked images; provision // grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that // would shadow the baked images. 0 = skip (keep the golden's size). DataVolGrowGB int // DataVolMount is the mpN slot of the golden's Docker-data volume to grow; "" → DefaultDataVolMount ("mp0"). DataVolMount string Mounts []GuestMount // additive mpN mounts (slice 7 may pass empty/test) KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live BootTimeout time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait } // BringUpResult is the outcome. It reuses the restore-test's WARNINGS surface // (StartWarnings/WarningsRecognized) — the start step here is the same liveness-anchored boot. type BringUpResult struct { VMID int AssignedMAC string // the guest's net0 MAC after identity reset (fresh for provision) Hostname string Pass bool Verified string // "boot+running" Err error StartedAt time.Time Duration time.Duration StartWarnings []string WarningsRecognized bool } // IntentForRollbackDestroy builds the benign compensating-rollback teardown intent for a guest // the agent created earlier in THIS journaled bring-up transaction: ClassGuestDestroy made benign // by SameTxnCreated provenance (classify.go) — a rollback, not data loss. Gate-authorized unsigned // but genuinely in-path (wrong provenance → pending_signature). Distinct from the restore-test's // IntentForScratchDestroy (AgentTaggedScratch) — different audit label, same destroy machinery. func IntentForRollbackDestroy(hostID string, vmid int) Intent { return Intent{ Class: ClassGuestDestroy, HostID: hostID, GuestID: strconv.Itoa(vmid), VMID: vmid, Provenance: Provenance{SameTxnCreated: true}, Source: SourceOneShotJob, } } // RunBringUp runs one bring-up on the target VMID's queue lane (inherits §10 per-guest // serialization). On success the guest is KEPT; on any mid-flight failure it is // compensating-rolled-back (destroyed). The returned Err is the job verdict's error. func (e *Engine) RunBringUp(ctx context.Context, spec BringUpSpec) BringUpResult { now := time.Now().UTC() res := BringUpResult{VMID: spec.VMID, Hostname: spec.Hostname, StartedAt: now} if spec.Archive == "" || spec.RestoreStorage == "" { res.Err = fmt.Errorf("reconcile: bring-up needs an archive and a restore storage") return res } if spec.VMID <= 0 { res.Err = fmt.Errorf("reconcile: bring-up needs a target VMID") return res } // Fence the reserved bands: never provision over the standing scratch (9999) or the // restore-test scratch band (990000–990009). if spec.VMID == 9999 || (spec.VMID >= 990000 && spec.VMID <= 990009) { res.Err = fmt.Errorf("reconcile: bring-up VMID %d is reserved (9999 / restore-test scratch band)", spec.VMID) return res } switch spec.Mode { case ModeProvision, ModeDRGuestLoss: default: res.Err = fmt.Errorf("reconcile: bring-up unknown mode %q", spec.Mode) return res } ch := e.queue.Submit(spec.VMID, func() error { e.runBringUp(ctx, spec, &res) return res.Err }) <-ch res.Duration = time.Since(now) return res } // runBringUp is the journaled body (runs on the target VMID's queue lane). func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpResult) { // PROVISION is to a NEW VMID. Restoring OVER an existing guest is ClassRestoreOverwrite // (destructive, slice 10) — never this benign path. Refuse before journaling. lxc, err := e.api.ListLXC(ctx) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up list guests: %w", err) return } for _, g := range lxc { if g.VMID == spec.VMID { res.Err = fmt.Errorf("reconcile: bring-up VMID %d already exists (restore-over-existing is a signed op)", spec.VMID) return } } base := JournalEntry{OpID: e.bringUpOpID(spec.VMID), VMID: spec.VMID, Kind: bringUpKind, Rollback: true} // OWN the rollback BEFORE any mutation. From here a crash leaves an in-flight Rollback // entry meaning "VMID may be a half-built guest → destroy it" (Recover.recoverBringUp). e.append(withState(base, OpStarted)) // Compensating rollback on EVERY non-committed exit (defer): destroy the just-created // guest. On success we set committed and KEEP it (the key difference from the restore-test). committed := false defer func() { if committed { return } e.rollbackBringUp(ctx, base) }() // 1. Restore archive → VMID (token-covered ClassCreate; keyctl preserved — phase3 + spike). upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{ VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, }) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up restore: %w", err) return } e.append(withUPID(base, upid, OpTaskRunning)) if _, err := e.waitTask(ctx, upid, proxmox.WaitOptions{}); err != nil { res.Err = fmt.Errorf("reconcile: bring-up restore task: %w", err) return } // 2. Read as-restored config (the net0 the MAC handling keys off). cfg, err := e.api.GuestConfig(ctx, spec.VMID) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up read config: %w", err) return } // 3+5 coalesced (F4): identity reset + cores/mem sizing + additive mounts in ONE config PUT, // with the bounded retry that fires ONLY on the transient PVE config-lock 500. if params := buildBringUpConfig(spec, cfg); len(params) > 0 { if err := e.setConfigWithLockRetry(ctx, spec.VMID, params); err != nil { res.Err = fmt.Errorf("reconcile: bring-up config: %w", err) return } } // 4. rootfs grow-only resize as its OWN call (F4: kept separate from the config PUT). if spec.RootfsGrowGB > 0 { rupid, err := e.api.ResizeLXC(ctx, spec.VMID, "rootfs", fmt.Sprintf("+%dG", spec.RootfsGrowGB)) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up resize: %w", err) return } if _, err := e.waitTask(ctx, rupid, proxmox.WaitOptions{}); err != nil { res.Err = fmt.Errorf("reconcile: bring-up resize task: %w", err) return } } // 4b. Grow the golden-carried Docker-data volume (mp0) to the per-customer target. Grow-only, // online (storage-split B4); its OWN call like the rootfs resize. The volume + baked images // came in with the restore, so we grow it rather than attach a fresh one that would shadow // the baked images. if spec.DataVolGrowGB > 0 { mount := spec.DataVolMount if mount == "" { mount = DefaultDataVolMount } dupid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.DataVolGrowGB)) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up data-volume resize (%s): %w", mount, err) return } if _, err := e.waitTask(ctx, dupid, proxmox.WaitOptions{}); err != nil { res.Err = fmt.Errorf("reconcile: bring-up data-volume resize task (%s): %w", mount, err) return } } // Capture the post-reset MAC for the result (fresh for provision; archived for DR keep). if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil { res.AssignedMAC = net0MAC(cfg2) } // 6. Start LINK-UP (ClassStart). The VERDICT is liveness (waitRunning), NEVER the start // exitstatus — same liveness-anchoring as the restore-test fix; AllowWarnings so a benign // start advisory (systemd-nesting) is surfaced, not failed. startUPID, err := e.api.Start(ctx, spec.VMID) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up start: %w", err) return } if startUPID != "" { st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true}) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up start task: %w", err) return } if strings.HasPrefix(st.ExitStatus, "WARNINGS") { tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50) if logErr != nil { e.logger.Warn("bring-up: could not read start-task log for warnings", "vmid", spec.VMID, "err", logErr) } res.StartWarnings = extractWarningLines(tail) res.WarningsRecognized = warningsRecognized(res.StartWarnings) } } bootTO := spec.BootTimeout if bootTO <= 0 { bootTO = DefaultBootTimeout } if err := e.waitRunning(ctx, spec.VMID, bootTO); err != nil { res.Err = err return } // 7. Success — KEEP the guest; mark the owning entry terminal so Recover ignores it. res.Pass = true res.Verified = "boot+running" committed = true e.append(withState(base, OpSucceeded)) } // rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated // provenance) and records the owning entry terminal. Mirrors teardownScratch: ALWAYS attempts the // destroy (idempotent — a restore-POST failure that created no guest just errors harmlessly and is // left in-flight for Recover, which existence-checks). On any teardown failure it leaves the entry // in-flight so Recover reaps the guest later — never force-destroys. func (e *Engine) rollbackBringUp(ctx context.Context, base JournalEntry) { tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) defer cancel() dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, base.VMID), nil) if !dec.Allowed { e.logger.Error("bring-up: rollback destroy refused by gate (unexpected); left for Recover", "vmid", base.VMID, "reason", dec.Reason) return } upid, err := e.api.DestroyLXC(tctx, base.VMID) if err != nil { e.logger.Error("bring-up: rollback destroy failed; left for Recover", "vmid", base.VMID, "err", err) return } if _, err := e.waitTask(tctx, upid, proxmox.WaitOptions{}); err != nil { e.logger.Error("bring-up: rollback destroy task failed; left for Recover", "vmid", base.VMID, "err", err) return } e.append(withState(base, OpSucceeded)) e.logger.Warn("bring-up: rolled back (destroyed half-built guest)", "vmid", base.VMID) } // buildBringUpConfig assembles the coalesced config PUT params per the scenario-specific identity // policy (doc 03 §9). Provision: fresh MAC (strip hwaddr → PVE regenerates, F1) + hostname. // DR: preserve continuity — keep MAC (unless KeepMAC=false: a source may be live) and keep // hostname (no force-reset). Both: cores/mem sizing + additive mpN mounts. machine-id and SSH // host keys are NOT touched here — they regenerate guest-side on first boot (golden bake + the // baked first-boot unit), keeping the agent's front half host-side-only. func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]string { params := map[string]string{} resetMAC := spec.Mode == ModeProvision || (spec.Mode == ModeDRGuestLoss && !spec.KeepMAC) if resetMAC { if net0, ok := cfg.Nets()["net0"]; ok && net0 != "" { params["net0"] = withoutHwaddr(net0) // omit hwaddr → PVE generates a fresh MAC (F1) } } if spec.Mode == ModeProvision && spec.Hostname != "" { params["hostname"] = spec.Hostname } if spec.Cores > 0 { params["cores"] = strconv.Itoa(spec.Cores) } if spec.MemoryMB > 0 { params["memory"] = strconv.Itoa(spec.MemoryMB) } for i, m := range spec.Mounts { // backup=1 for data-bearing mounts: extra LXC mountpoints default to backup=0 = EXCLUDED // from vzdump/PBS (storage-split B3), which would silently drop their DBs from the snapshot. spec := fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint) if m.Backup { spec += ",backup=1" } params[fmt.Sprintf("mp%d", i)] = spec } return params } // setConfigWithLockRetry issues the coalesced config PUT, retrying ONLY the transient PVE // config-lock 500 (F4) with bounded backoff. A non-lock error (any other 500 included) fails // immediately — never retry a real error. The slice-4 per-guest serializer prevents cross-op // contention; this covers PVE releasing its own async config lock within one job. func (e *Engine) setConfigWithLockRetry(ctx context.Context, vmid int, params map[string]string) error { var lastErr error for attempt := 1; attempt <= configLockMaxAttempts; attempt++ { if _, err := e.api.SetConfig(ctx, vmid, params); err == nil { return nil } else if !pveConfigLock(err) { return err // real error — never retry } else { lastErr = err e.logger.Warn("bring-up: transient PVE config-lock; retrying", "vmid", vmid, "attempt", attempt, "max", configLockMaxAttempts, "err", err) } select { case <-ctx.Done(): return ctx.Err() case <-time.After(configLockBackoff): } } return fmt.Errorf("reconcile: bring-up config still lock-contended after %d attempts: %w", configLockMaxAttempts, lastErr) } // pveConfigLock reports whether err is the transient PVE config-lock 500 (F4) — and ONLY that. // The lock surfaces as a proxmox.APIError 500 whose body carries the lock signature. func pveConfigLock(err error) bool { var ae *proxmox.APIError if !errors.As(err, &ae) || ae.StatusCode != 500 { return false } b := strings.ToLower(ae.Body) return strings.Contains(b, "can't lock file") || strings.Contains(b, "got timeout") } // waitTask waits a (possibly empty) UPID — "" is the clean synchronous path. func (e *Engine) waitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) { if upid == "" { return proxmox.TaskStatus{}, nil } return e.api.WaitTask(ctx, upid, opts) } func (e *Engine) bringUpOpID(vmid int) string { return "bring-up-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq) } // withoutHwaddr strips the hwaddr token from a netN config string. PUTting a netN with NO hwaddr // makes PVE generate a fresh MAC (slice-7 spike F1: a restore preserves the archived MAC, so a // provision MUST strip it to avoid a fleet-wide MAC collision). Mirrors withLinkDown's approach. func withoutHwaddr(netN string) string { parts := strings.Split(netN, ",") out := parts[:0] for _, p := range parts { if p == "" || strings.HasPrefix(p, "hwaddr=") { continue } out = append(out, p) } return strings.Join(out, ",") } // net0MAC extracts the hwaddr from a guest's net0 config ("" if absent). func net0MAC(cfg proxmox.GuestConfig) string { net0, ok := cfg.Nets()["net0"] if !ok { return "" } for _, p := range strings.Split(net0, ",") { if strings.HasPrefix(p, "hwaddr=") { return strings.TrimPrefix(p, "hwaddr=") } } return "" }