v0.60.0: proof-of-launch destroy gating (F1a/b/c) + restore-test band-advance (F2)

Campaign pool-effects F1 (HIGH): the bring-up compensating rollback and the
restore-test teardown destroyed the target vmid even when RestoreLXC failed
synchronously without creating anything — destroying a guest the transaction
never made (only the pool ACL 403 contained it). A RestoreLXC UPID is now the
sole destroy authorization in all three destroy paths (in-process bring-up
defer, in-process restore-test teardown, Recover). F2: the restore-test
advances past an 'already exists' band vmid (invisible squatter) instead of
failing + false-alerting; a fully-occupied band Skips.

Red-proof verified: with the gates reverted, the four new tests fail with the
innocent-guest destroy. go build/vet/test clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

internal/reconcile/restoretest.go

@@ -105,9 +105,11 @@ func IntentForScratchDestroy(hostID string, vmid int) Intent {
 
 // RunRestoreTest runs one restore-test on the per-guest queue lane of a fresh scratch VMID.
 // It journals a Scratch-owned entry BEFORE any mutation, so a crash anywhere after this
-// point is recoverable (Recover destroys the scratch guest). Teardown runs on EVERY path
-// (defer), including a failed verify. The returned Err is the TEST verdict's error (restore
-// or boot failure), independent of teardown success.
+// point is recoverable (Recover destroys a launch-proven scratch guest via its journaled
+// UPID). Teardown runs on every launch-proven path (defer), including a failed verify — but
+// NEVER when the restore failed before creating anything (proof-of-launch, campaign F1b). A
+// band vmid PVE reports "already exists" is advanced past, not failed (F2). The returned Err
+// is the TEST verdict's error (restore or boot failure), independent of teardown success.
 func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) RestoreTestResult {
 	now := time.Now().UTC()
 	res := RestoreTestResult{Archive: spec.Archive, SourceTier: spec.SourceTier, StartedAt: now}
@@ -126,38 +128,68 @@ func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) Resto
 		res.Err = fmt.Errorf("reconcile: restore-test list guests: %w", err)
 		return res
 	}
-	vmid, ok := pickScratchVMID(lxc, spec.ScratchMin, spec.ScratchMax)
-	if !ok {
-		// Full band (e.g. an accumulation of un-torn-down scratch guests) → skip, never
-		// panic or pick out-of-band. Recover will reap any genuinely leaked ones.
-		e.logger.Warn("restore-test skipped: no free scratch VMID in band",
-			"min", spec.ScratchMin, "max", spec.ScratchMax)
-		res.Skipped = true
-		return res
-	}
-	res.ScratchVMID = vmid
+	// Band-advance loop (campaign pool-effects F2): the band scan below is POOL-BLIND (the
+	// scoped token's ListLXC can't see non-pool guests), so a band vmid can look free while a
+	// squatter sits on it. PVE tells us at restore time ("already exists"); we then advance to
+	// the next band vmid instead of failing — one squatter must not permanently break the
+	// restore-test or raise a false "backup unrestorable" alert. Bounded by the band width.
+	occupied := make(map[int]bool)
+	for {
+		vmid, ok := pickScratchVMID(lxc, spec.ScratchMin, spec.ScratchMax, occupied)
+		if !ok {
+			// Band exhausted (in-use and/or invisible squatters) → skip, never panic, never
+			// pick out-of-band, never FAIL. Recover will reap any genuinely leaked ones.
+			e.logger.Warn("restore-test skipped: no free scratch VMID in band",
+				"min", spec.ScratchMin, "max", spec.ScratchMax, "occupied_invisible", len(occupied))
+			res.Skipped = true
+			res.ScratchVMID = 0
+			res.Err = nil
+			return res
+		}
+		res.ScratchVMID = vmid
 
-	// Serialize on the scratch VMID's lane (inherits §10), and capture the result.
-	ch := e.queue.Submit(vmid, func() error {
-		e.runScratchTest(ctx, vmid, spec, &res)
-		return res.Err
-	})
-	<-ch
+		// Serialize on the scratch VMID's lane (inherits §10), and capture the result.
+		var vmidOccupied bool
+		ch := e.queue.Submit(vmid, func() error {
+			vmidOccupied = e.runScratchTest(ctx, vmid, spec, &res)
+			return res.Err
+		})
+		<-ch
+		if !vmidOccupied {
+			break
+		}
+		e.logger.Warn("restore-test: band VMID occupied by a guest invisible to the token; advancing",
+			"vmid", vmid)
+		occupied[vmid] = true
+	}
 	res.Duration = time.Since(now)
 	return res
 }
 
-// runScratchTest is the journaled body (runs on vmid's queue lane).
-func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) {
+// runScratchTest is the journaled body (runs on vmid's queue lane). The occupied return is true
+// ONLY when PVE synchronously refused the restore because the vmid already holds a guest (one
+// the pool-blind band scan couldn't see) — the caller then advances to the next band vmid (F2).
+func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) (occupied bool) {
 	base := JournalEntry{OpID: e.scratchOpID(vmid), VMID: vmid, Kind: scratchKind, Scratch: true}
 
 	// OWN the scratch guest's cleanup BEFORE any mutation. From here, a crash is recoverable.
 	e.append(withState(base, OpStarted))
 
-	// Teardown ALWAYS runs (even on a failed verify). Uses a cancel-immune context so a
-	// daemon shutdown mid-test still tears down; if teardown fails, the entry stays
-	// in-flight and Recover reaps the guest on the next start.
-	defer e.teardownScratch(ctx, base)
+	// Teardown runs on every exit AFTER the restore launched (even on a failed verify), using a
+	// cancel-immune context so a daemon shutdown mid-test still tears down; if teardown fails,
+	// the entry stays in-flight and Recover reaps the guest on the next start (via the journaled
+	// UPID). `launched` is the proof-of-launch gate (campaign pool-effects F1b): a restore that
+	// failed synchronously (no UPID) created NOTHING, so teardown must NEVER destroy the vmid —
+	// an invisible pre-existing guest may sit there. The entry is then closed terminal-failed
+	// (nothing exists to recover).
+	launched := false
+	defer func() {
+		if launched {
+			e.teardownScratch(ctx, base)
+			return
+		}
+		e.append(withState(base, OpFailed))
+	}()
 
 	// 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error
 	//    detection only — it does NOT make the Scratch entry terminal (teardown does).
@@ -197,9 +229,18 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
 		VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage, MountOverrides: mountOverrides, Pool: DefaultPool,
 	})
 	if err != nil {
+		if pveAlreadyExists(err) {
+			// The band vmid holds a guest the pool-blind scan couldn't see. Nothing was
+			// created; NOT a test verdict — the caller advances to the next band vmid (F2).
+			return true
+		}
 		res.Err = fmt.Errorf("reconcile: restore-test restore: %w", err)
-		return
+		return false
 	}
+	// Proof-of-launch: the POST was accepted — from here teardown owns the guest. Accepted
+	// residual: a crash before the next append leaks a scratch guest Recover won't destroy
+	// (no journaled UPID) — cleanable, and preferable to destroying an innocent guest.
+	launched = true
 	e.append(withUPID(base, upid, OpTaskRunning))
 	if upid != "" {
 		if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
@@ -258,6 +299,7 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
 	}
 	res.Pass = true
 	res.Verified = "boot+running"
+	return false
 }
 
 // archiveVMID extracts the source VMID from a backup archive volid. Handles PBS volids
@@ -414,15 +456,16 @@ func (e *Engine) waitRunning(ctx context.Context, vmid int, timeout time.Duratio
 }
 
 // pickScratchVMID returns the lowest free VMID in [min,max], excluding the standing 9999
-// scratch and any in-use guest. ok=false when the band is fully occupied (the test is then
-// skipped, never run out-of-band).
-func pickScratchVMID(lxc []proxmox.Guest, min, max int) (int, bool) {
+// scratch, any in-use guest, and the caller's exclude set (band vmids PVE reported occupied by
+// guests the pool-blind list can't see — the F2 band-advance). ok=false when the band is fully
+// occupied (the test is then skipped, never run out-of-band).
+func pickScratchVMID(lxc []proxmox.Guest, min, max int, exclude map[int]bool) (int, bool) {
 	used := make(map[int]bool, len(lxc))
 	for _, g := range lxc {
 		used[g.VMID] = true
 	}
 	for id := min; id <= max; id++ {
-		if id == 9999 || used[id] {
+		if id == 9999 || used[id] || exclude[id] {
 			continue
 		}
 		return id, true