slice 7 Phase 1: unified bring-up reconcile job (provision + guest-loss DR) (v0.8.0)
The shared front half of provision and guest-loss DR as a journaled reconcile job (internal/reconcile/bringup.go), mirroring the restore-test's crash-safety but keeping the guest on success and applying a scenario-specific identity policy. Agent-only; no hub/wire change. Grounded by the slice-7 bring-up spike (commit 3342993): F1/F3/F4. - RunBringUp: restore -> reset identity -> size -> attach mounts -> start link-up; verdict is liveness (waitRunning), success KEEPS the guest. - identity policy: provision = fresh MAC (net0 sans hwaddr -> PVE regen) + hostname, host-side; machine-id/host-keys regenerate guest-side (systemd + baked golden unit). dr_guest_loss = preserve continuity (keep hostname; keep MAC unless KeepMAC=false). - compensating rollback: mid-flight failure destroys the just-created guest (SameTxnCreated provenance, gated); new Rollback journal flag + Recover.recoverBringUp reap a half-built guest from a crash. - F4: coalesced config PUT + bounded retry on the transient PVE config-lock 500 only. - --selftest=bring-up (mode/archive/vmid/hostname/keep). - configs/build-golden.sh: validated golden recipe incl. the F3 first-boot host-key unit. - doc-03 §9 + identity-reset settled/implemented. Deferred (stated): provisioning back half -> slice 8; host-loss DR + escrow consumption and the BringUpSpec source (hub desired-state) -> slice 10. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -44,6 +44,15 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
|
||||
continue
|
||||
}
|
||||
|
||||
// Rollback entries (slice-7 bring-up) own a guest the agent was CREATING. An in-flight
|
||||
// one means "VMID may be a half-built guest → destroy it" (compensating rollback) — same
|
||||
// reason the Scratch path runs before the generic UPID path: the restore sub-task's OK
|
||||
// status would otherwise mark the entry succeeded and leave a half-provisioned guest.
|
||||
if entry.Rollback {
|
||||
e.recoverBringUp(ctx, entry, &res)
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.UPID == "" {
|
||||
// POST never confirmed → abandon (fail-safe).
|
||||
e.append(terminal(entry, OpFailed))
|
||||
@@ -144,16 +153,77 @@ func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *Re
|
||||
"op_id", entry.OpID, "vmid", entry.VMID)
|
||||
}
|
||||
|
||||
// recoverBringUp rolls back a half-built bring-up guest left in-flight by a mid-job crash
|
||||
// (slice 7, doc 03 §9). Invariant: a Rollback entry in-flight at startup means "VMID may be a
|
||||
// half-provisioned guest and MUST be destroyed" (compensating rollback — the guest is only kept
|
||||
// when the bring-up reached its terminal OpSucceeded). Idempotent: already-gone records
|
||||
// terminal-clean. Routes the destroy through the gate as benign ClassGuestDestroy (SameTxnCreated
|
||||
// provenance) — the same audit-bearing path the in-job rollback uses.
|
||||
func (e *Engine) recoverBringUp(ctx context.Context, entry JournalEntry, res *RecoverResult) {
|
||||
lxc, err := e.api.ListLXC(ctx)
|
||||
if err != nil {
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: cannot list guests to resolve half-built bring-up; left in-flight",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
exists := false
|
||||
for _, g := range lxc {
|
||||
if g.VMID == entry.VMID {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
// Already gone (e.g. the restore POST never created it) → no leak.
|
||||
e.append(terminal(entry, OpSucceeded))
|
||||
res.BringUpClean++
|
||||
e.logger.Info("recover: half-built bring-up resolved; guest already gone",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID)
|
||||
return
|
||||
}
|
||||
|
||||
dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, entry.VMID), nil)
|
||||
if !dec.Allowed {
|
||||
// Should be benign (SameTxnCreated); if not, fail-safe — do NOT force a destroy.
|
||||
res.Unresolved++
|
||||
e.logger.Error("recover: bring-up rollback refused by gate (unexpected); left in-flight",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
|
||||
return
|
||||
}
|
||||
upid, err := e.api.DestroyLXC(ctx, entry.VMID)
|
||||
if err != nil {
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: destroying half-built bring-up failed; left in-flight (will retry)",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: half-built bring-up destroy task failed; left in-flight (will retry)",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
e.append(terminal(entry, OpSucceeded))
|
||||
res.BringUpRolledBack++
|
||||
e.logger.Warn("recover: rolled back half-built bring-up guest",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID)
|
||||
}
|
||||
|
||||
// RecoverResult summarizes a startup recovery pass.
|
||||
type RecoverResult struct {
|
||||
Examined int
|
||||
Resumed int // task found completed OK and recorded succeeded
|
||||
Failed int // task found ended non-OK and recorded failed
|
||||
RolledBack int // no task id → abandoned (fail-safe)
|
||||
StillRunning int // task still executing → left in-flight
|
||||
Unresolved int // task status unreadable → left in-flight
|
||||
ScratchClean int // scratch entry resolved: guest already gone (no leak)
|
||||
ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
|
||||
Examined int
|
||||
Resumed int // task found completed OK and recorded succeeded
|
||||
Failed int // task found ended non-OK and recorded failed
|
||||
RolledBack int // no task id → abandoned (fail-safe)
|
||||
StillRunning int // task still executing → left in-flight
|
||||
Unresolved int // task status unreadable → left in-flight
|
||||
ScratchClean int // scratch entry resolved: guest already gone (no leak)
|
||||
ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
|
||||
BringUpClean int // bring-up rollback entry resolved: guest already gone (no leak)
|
||||
BringUpRolledBack int // bring-up rollback entry resolved: half-built guest destroyed
|
||||
}
|
||||
|
||||
// terminal builds a terminal journal record preserving the op's identity, with the
|
||||
@@ -167,6 +237,7 @@ func terminal(e JournalEntry, state OpState) JournalEntry {
|
||||
State: state,
|
||||
IdempKey: e.IdempKey,
|
||||
Scratch: e.Scratch,
|
||||
Rollback: e.Rollback,
|
||||
At: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user