v0.60.0: proof-of-launch destroy gating (F1a/b/c) + restore-test band-advance (F2)

Campaign pool-effects F1 (HIGH): the bring-up compensating rollback and the
restore-test teardown destroyed the target vmid even when RestoreLXC failed
synchronously without creating anything — destroying a guest the transaction
never made (only the pool ACL 403 contained it). A RestoreLXC UPID is now the
sole destroy authorization in all three destroy paths (in-process bring-up
defer, in-process restore-test teardown, Recover). F2: the restore-test
advances past an 'already exists' band vmid (invisible squatter) instead of
failing + false-alerting; a fully-occupied band Skips.

Red-proof verified: with the gates reverted, the four new tests fail with the
innocent-guest destroy. go build/vet/test clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 10:18:51 +02:00
parent 55ade9e254
commit b9356d60ab
8 changed files with 400 additions and 67 deletions
+39 -7
View File
@@ -192,16 +192,29 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
base := JournalEntry{OpID: e.bringUpOpID(spec.VMID), VMID: spec.VMID, Kind: bringUpKind, Rollback: true}
// OWN the rollback BEFORE any mutation. From here a crash leaves an in-flight Rollback
// entry meaning "VMID may be a half-built guest → destroy it" (Recover.recoverBringUp).
// entry; Recover destroys the vmid ONLY when the entry carries a restore UPID
// (proof-of-launch — see Recover's no-UPID abandon path).
e.append(withState(base, OpStarted))
// Compensating rollback on EVERY non-committed exit (defer): destroy the just-created
// guest. On success we set committed and KEEP it (the key difference from the restore-test).
// Compensating rollback on every non-committed exit AFTER the restore launched (defer):
// destroy the just-created guest. On success we set committed and KEEP it (the key
// difference from the restore-test). `launched` is the proof-of-launch gate (campaign
// pool-effects F1a): a restore that failed synchronously (no UPID — e.g. PVE refusing a
// vmid that already holds a guest the pool-blind duplicate guard can't see) created
// NOTHING, so the rollback must NEVER destroy the vmid — a pre-existing guest, possibly
// another customer's, may sit there. This must hold WITHOUT the pool ACL (that 403 is
// defense-in-depth, not the guard). The owning entry is then closed terminal-failed
// in-process (nothing exists to recover).
committed := false
launched := false
defer func() {
if committed {
return
}
if !launched {
e.append(withState(base, OpFailed))
return
}
e.rollbackBringUp(ctx, base)
}()
@@ -210,9 +223,15 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
})
if err != nil {
// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
res.Err = fmt.Errorf("reconcile: bring-up restore: %w", err)
return
}
// Proof-of-launch: the POST was accepted — from here a failure means a half-built guest the
// compensating rollback (or Recover, via the journaled UPID) must destroy. Accepted residual:
// a crash in the one-statement window before the UPID is journaled leaks a half-built guest
// that Recover won't destroy — cleanable, and preferable to destroying an innocent guest.
launched = true
e.append(withUPID(base, upid, OpTaskRunning))
if _, err := e.waitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
res.Err = fmt.Errorf("reconcile: bring-up restore task: %w", err)
@@ -333,10 +352,10 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
}
// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
// provenance) and records the owning entry terminal. Mirrors teardownScratch: ALWAYS attempts the
// destroy (idempotent — a restore-POST failure that created no guest just errors harmlessly and is
// left in-flight for Recover, which existence-checks). On any teardown failure it leaves the entry
// in-flight so Recover reaps the guest later — never force-destroys.
// provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
// was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not
// assumed. On any teardown failure it leaves the entry in-flight so Recover reaps the guest later
// (via the journaled UPID) — never force-destroys.
func (e *Engine) rollbackBringUp(ctx context.Context, base JournalEntry) {
tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
@@ -432,6 +451,19 @@ func pveConfigLock(err error) bool {
return strings.Contains(b, "can't lock file") || strings.Contains(b, "got timeout")
}
// pveAlreadyExists reports whether err is PVE's synchronous refusal to create over an existing
// vmid ("CT <vmid> already exists on node '<node>'" — an APIError 500, observed live in the
// pool-effects campaign). By construction such a refusal returned no UPID: nothing was created.
// Used by the restore-test band-advance (F2) to distinguish "band vmid occupied by a guest the
// pool-blind list can't see" from a real restore failure — never misclassify the latter.
func pveAlreadyExists(err error) bool {
var ae *proxmox.APIError
if !errors.As(err, &ae) || ae.StatusCode != 500 {
return false
}
return strings.Contains(strings.ToLower(ae.Body), "already exists")
}
// waitTask waits a (possibly empty) UPID — "" is the clean synchronous path.
func (e *Engine) waitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) {
if upid == "" {