v0.60.0: proof-of-launch destroy gating (F1a/b/c) + restore-test band-advance (F2)
Campaign pool-effects F1 (HIGH): the bring-up compensating rollback and the restore-test teardown destroyed the target vmid even when RestoreLXC failed synchronously without creating anything — destroying a guest the transaction never made (only the pool ACL 403 contained it). A RestoreLXC UPID is now the sole destroy authorization in all three destroy paths (in-process bring-up defer, in-process restore-test teardown, Recover). F2: the restore-test advances past an 'already exists' band vmid (invisible squatter) instead of failing + false-alerting; a fully-occupied band Skips. Red-proof verified: with the gates reverted, the four new tests fail with the innocent-guest destroy. go build/vet/test clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -192,16 +192,29 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
base := JournalEntry{OpID: e.bringUpOpID(spec.VMID), VMID: spec.VMID, Kind: bringUpKind, Rollback: true}
|
||||
|
||||
// OWN the rollback BEFORE any mutation. From here a crash leaves an in-flight Rollback
|
||||
// entry meaning "VMID may be a half-built guest → destroy it" (Recover.recoverBringUp).
|
||||
// entry; Recover destroys the vmid ONLY when the entry carries a restore UPID
|
||||
// (proof-of-launch — see Recover's no-UPID abandon path).
|
||||
e.append(withState(base, OpStarted))
|
||||
|
||||
// Compensating rollback on EVERY non-committed exit (defer): destroy the just-created
|
||||
// guest. On success we set committed and KEEP it (the key difference from the restore-test).
|
||||
// Compensating rollback on every non-committed exit AFTER the restore launched (defer):
|
||||
// destroy the just-created guest. On success we set committed and KEEP it (the key
|
||||
// difference from the restore-test). `launched` is the proof-of-launch gate (campaign
|
||||
// pool-effects F1a): a restore that failed synchronously (no UPID — e.g. PVE refusing a
|
||||
// vmid that already holds a guest the pool-blind duplicate guard can't see) created
|
||||
// NOTHING, so the rollback must NEVER destroy the vmid — a pre-existing guest, possibly
|
||||
// another customer's, may sit there. This must hold WITHOUT the pool ACL (that 403 is
|
||||
// defense-in-depth, not the guard). The owning entry is then closed terminal-failed
|
||||
// in-process (nothing exists to recover).
|
||||
committed := false
|
||||
launched := false
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
if !launched {
|
||||
e.append(withState(base, OpFailed))
|
||||
return
|
||||
}
|
||||
e.rollbackBringUp(ctx, base)
|
||||
}()
|
||||
|
||||
@@ -210,9 +223,15 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
VMID: spec.VMID, Archive: spec.Archive, Storage: spec.RestoreStorage, Pool: spec.Pool,
|
||||
})
|
||||
if err != nil {
|
||||
// No UPID ⇒ nothing was created ⇒ the defer closes the entry WITHOUT a destroy.
|
||||
res.Err = fmt.Errorf("reconcile: bring-up restore: %w", err)
|
||||
return
|
||||
}
|
||||
// Proof-of-launch: the POST was accepted — from here a failure means a half-built guest the
|
||||
// compensating rollback (or Recover, via the journaled UPID) must destroy. Accepted residual:
|
||||
// a crash in the one-statement window before the UPID is journaled leaks a half-built guest
|
||||
// that Recover won't destroy — cleanable, and preferable to destroying an innocent guest.
|
||||
launched = true
|
||||
e.append(withUPID(base, upid, OpTaskRunning))
|
||||
if _, err := e.waitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: bring-up restore task: %w", err)
|
||||
@@ -333,10 +352,10 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
|
||||
}
|
||||
|
||||
// rollbackBringUp destroys the just-created guest (benign ClassGuestDestroy via SameTxnCreated
|
||||
// provenance) and records the owning entry terminal. Mirrors teardownScratch: ALWAYS attempts the
|
||||
// destroy (idempotent — a restore-POST failure that created no guest just errors harmlessly and is
|
||||
// left in-flight for Recover, which existence-checks). On any teardown failure it leaves the entry
|
||||
// in-flight so Recover reaps the guest later — never force-destroys.
|
||||
// provenance) and records the owning entry terminal. Called ONLY launch-proven (the restore POST
|
||||
// was accepted — campaign pool-effects F1a): the SameTxnCreated provenance is then real, not
|
||||
// assumed. On any teardown failure it leaves the entry in-flight so Recover reaps the guest later
|
||||
// (via the journaled UPID) — never force-destroys.
|
||||
func (e *Engine) rollbackBringUp(ctx context.Context, base JournalEntry) {
|
||||
tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||||
defer cancel()
|
||||
@@ -432,6 +451,19 @@ func pveConfigLock(err error) bool {
|
||||
return strings.Contains(b, "can't lock file") || strings.Contains(b, "got timeout")
|
||||
}
|
||||
|
||||
// pveAlreadyExists reports whether err is PVE's synchronous refusal to create over an existing
|
||||
// vmid ("CT <vmid> already exists on node '<node>'" — an APIError 500, observed live in the
|
||||
// pool-effects campaign). By construction such a refusal returned no UPID: nothing was created.
|
||||
// Used by the restore-test band-advance (F2) to distinguish "band vmid occupied by a guest the
|
||||
// pool-blind list can't see" from a real restore failure — never misclassify the latter.
|
||||
func pveAlreadyExists(err error) bool {
|
||||
var ae *proxmox.APIError
|
||||
if !errors.As(err, &ae) || ae.StatusCode != 500 {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(ae.Body), "already exists")
|
||||
}
|
||||
|
||||
// waitTask waits a (possibly empty) UPID — "" is the clean synchronous path.
|
||||
func (e *Engine) waitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) {
|
||||
if upid == "" {
|
||||
|
||||
Reference in New Issue
Block a user