slice 7 Phase 1: unified bring-up reconcile job (provision + guest-loss DR) (v0.8.0)

The shared front half of provision and guest-loss DR as a journaled reconcile job
(internal/reconcile/bringup.go), mirroring the restore-test's crash-safety but keeping
the guest on success and applying a scenario-specific identity policy. Agent-only; no
hub/wire change. Grounded by the slice-7 bring-up spike (commit 3342993): F1/F3/F4.

- RunBringUp: restore -> reset identity -> size -> attach mounts -> start link-up;
  verdict is liveness (waitRunning), success KEEPS the guest.
- identity policy: provision = fresh MAC (net0 sans hwaddr -> PVE regen) + hostname,
  host-side; machine-id/host-keys regenerate guest-side (systemd + baked golden unit).
  dr_guest_loss = preserve continuity (keep hostname; keep MAC unless KeepMAC=false).
- compensating rollback: mid-flight failure destroys the just-created guest
  (SameTxnCreated provenance, gated); new Rollback journal flag + Recover.recoverBringUp
  reap a half-built guest from a crash.
- F4: coalesced config PUT + bounded retry on the transient PVE config-lock 500 only.
- --selftest=bring-up (mode/archive/vmid/hostname/keep).
- configs/build-golden.sh: validated golden recipe incl. the F3 first-boot host-key unit.
- doc-03 §9 + identity-reset settled/implemented.

Deferred (stated): provisioning back half -> slice 8; host-loss DR + escrow consumption
and the BringUpSpec source (hub desired-state) -> slice 10.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 21:27:49 +02:00
parent 9f6753de0f
commit 57405c1a99
10 changed files with 1184 additions and 73 deletions
+79 -8
View File
@@ -44,6 +44,15 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
continue
}
// Rollback entries (slice-7 bring-up) own a guest the agent was CREATING. An in-flight
// one means "VMID may be a half-built guest → destroy it" (compensating rollback) — same
// reason the Scratch path runs before the generic UPID path: the restore sub-task's OK
// status would otherwise mark the entry succeeded and leave a half-provisioned guest.
if entry.Rollback {
e.recoverBringUp(ctx, entry, &res)
continue
}
if entry.UPID == "" {
// POST never confirmed → abandon (fail-safe).
e.append(terminal(entry, OpFailed))
@@ -144,16 +153,77 @@ func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *Re
"op_id", entry.OpID, "vmid", entry.VMID)
}
// recoverBringUp rolls back a half-built bring-up guest left in-flight by a mid-job crash
// (slice 7, doc 03 §9). Invariant: a Rollback entry in-flight at startup means "VMID may be a
// half-provisioned guest and MUST be destroyed" (compensating rollback — the guest is only kept
// when the bring-up reached its terminal OpSucceeded). Idempotent: already-gone records
// terminal-clean. Routes the destroy through the gate as benign ClassGuestDestroy (SameTxnCreated
// provenance) — the same audit-bearing path the in-job rollback uses.
func (e *Engine) recoverBringUp(ctx context.Context, entry JournalEntry, res *RecoverResult) {
lxc, err := e.api.ListLXC(ctx)
if err != nil {
res.Unresolved++
e.logger.Warn("recover: cannot list guests to resolve half-built bring-up; left in-flight",
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
return
}
exists := false
for _, g := range lxc {
if g.VMID == entry.VMID {
exists = true
break
}
}
if !exists {
// Already gone (e.g. the restore POST never created it) → no leak.
e.append(terminal(entry, OpSucceeded))
res.BringUpClean++
e.logger.Info("recover: half-built bring-up resolved; guest already gone",
"op_id", entry.OpID, "vmid", entry.VMID)
return
}
dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, entry.VMID), nil)
if !dec.Allowed {
// Should be benign (SameTxnCreated); if not, fail-safe — do NOT force a destroy.
res.Unresolved++
e.logger.Error("recover: bring-up rollback refused by gate (unexpected); left in-flight",
"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
return
}
upid, err := e.api.DestroyLXC(ctx, entry.VMID)
if err != nil {
res.Unresolved++
e.logger.Warn("recover: destroying half-built bring-up failed; left in-flight (will retry)",
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
return
}
if upid != "" {
if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
res.Unresolved++
e.logger.Warn("recover: half-built bring-up destroy task failed; left in-flight (will retry)",
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
return
}
}
e.append(terminal(entry, OpSucceeded))
res.BringUpRolledBack++
e.logger.Warn("recover: rolled back half-built bring-up guest",
"op_id", entry.OpID, "vmid", entry.VMID)
}
// RecoverResult summarizes a startup recovery pass.
type RecoverResult struct {
Examined int
Resumed int // task found completed OK and recorded succeeded
Failed int // task found ended non-OK and recorded failed
RolledBack int // no task id → abandoned (fail-safe)
StillRunning int // task still executing → left in-flight
Unresolved int // task status unreadable → left in-flight
ScratchClean int // scratch entry resolved: guest already gone (no leak)
ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
Examined int
Resumed int // task found completed OK and recorded succeeded
Failed int // task found ended non-OK and recorded failed
RolledBack int // no task id → abandoned (fail-safe)
StillRunning int // task still executing → left in-flight
Unresolved int // task status unreadable → left in-flight
ScratchClean int // scratch entry resolved: guest already gone (no leak)
ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
BringUpClean int // bring-up rollback entry resolved: guest already gone (no leak)
BringUpRolledBack int // bring-up rollback entry resolved: half-built guest destroyed
}
// terminal builds a terminal journal record preserving the op's identity, with the
@@ -167,6 +237,7 @@ func terminal(e JournalEntry, state OpState) JournalEntry {
State: state,
IdempKey: e.IdempKey,
Scratch: e.Scratch,
Rollback: e.Rollback,
At: time.Now().UTC(),
}
}