v0.175.0 — R-82: a tier that overruns the quiesce bound defers the rest

Operator ruling 2026-07-26: let the first backup run as long as needed; other
backups shouldn't start until finished.

A first FULL offsite snapshot runs for hours, far past max_quiesce. When that
bound elapses the app resumes (unchanged), but the loop then started the NEXT
tier while the first was still uploading. Now it breaks and defers the rest to
a later poll — vzdump still holds the guest lock, so the second start would be
refused by the agent (409, v0.99.0) or fail on the lock, and a failed backup
never satisfies a cadence, so the tier would retry into the same wall forever.

pollTier returns (phase, stillRunning, err). The app still resumes exactly once.

Red-proof observed and restored; full suite green.

controller/internal/quiesce/quiesce.go

@@ -354,16 +354,24 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 		_ = l.writeMarker(marker) // best-effort: record the CURRENT tier's job id for diagnosis
 		l.logger.Printf("[INFO] [quiesce] tier %s: backup job %s started — polling", label, jobID)
 
-		phase, perr := l.pollTier(ctx, t.target, jobID, label, deadline, last, &unquiesced, unquiesce)
+		phase, stillRunning, perr := l.pollTier(ctx, t.target, jobID, label, deadline, last, &unquiesced, unquiesce)
 		if perr != nil && firstErr == nil {
 			firstErr = perr
 		}
 		if phase == phaseFailed {
 			l.logger.Printf("[WARN] [quiesce] tier %s: backup job %s failed", label, jobID)
 		}
-		// The max-quiesce guard already unquiesced; keep going so the remaining tiers still run
-		// (the app is up — the backups simply continue without the quiesce guarantee, which is
-		// strictly better than skipping the DR tier entirely).
+		if stillRunning {
+			// The max-quiesce guard fired while THIS tier's backup is still going (a first full
+			// offsite snapshot legitimately runs for hours). The app is already back up. We must
+			// NOT start the next tier: ONE BACKUP AT A TIME PER GUEST (operator ruling
+			// 2026-07-26) — vzdump still holds the guest lock, so a second start would be refused
+			// by the agent (409) or fail on the lock and record a spurious failure. The remaining
+			// tiers simply run on a later poll, once this one has finished.
+			l.logger.Printf("[INFO] [quiesce] tier %s still running past the quiesce bound — deferring %d remaining tier(s) to a later cycle",
+				label, len(tiers)-i-1)
+			break
+		}
 	}
 
 	// Belt: if every tier failed to start, nothing above unquiesced. The deferred call covers it,
@@ -378,19 +386,21 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 // app-consistent while still costing exactly one stop/start pair. For a non-last tier the loop
 // waits for a TERMINAL phase (done/failed), because vzdump holds the guest lock and the next tier
 // cannot start until this one truly finishes.
+// Returns (phase, stillRunning, err). stillRunning=true means the quiesce bound elapsed while the
+// backup is STILL going — the caller must not start another tier (one backup at a time per guest).
 func (l *Loop) pollTier(ctx context.Context, target, jobID, label string, deadline time.Time,
-	last bool, unquiesced *bool, unquiesce func(string)) (string, error) {
+	last bool, unquiesced *bool, unquiesce func(string)) (string, bool, error) {
 	for {
 		if !l.now().Before(deadline) {
 			l.logger.Printf("[WARN] [quiesce] max-quiesce-duration (%s) exceeded on tier %s (job %s) — unquiescing while the backup continues on the agent",
 				l.maxQuiesce, label, jobID)
 			unquiesce("max-quiesce guard")
-			return "", nil
+			return "", true, nil
 		}
 		phase, err := l.backupStatusOn(ctx, target)
 		if err != nil {
 			unquiesce("status poll failed")
-			return "", fmt.Errorf("poll backup status on %s: %w", label, err)
+			return "", false, fmt.Errorf("poll backup status on %s: %w", label, err)
 		}
 		switch phase {
 		case phaseSnapshotted:
@@ -408,17 +418,17 @@ func (l *Loop) pollTier(ctx context.Context, target, jobID, label string, deadli
 			} else {
 				l.logger.Printf("[INFO] [quiesce] tier %s: backup job %s done — next tier may start (app still quiesced)", label, jobID)
 			}
-			return phaseDone, nil
+			return phaseDone, false, nil
 		case phaseFailed:
 			if last {
 				unquiesce("backup failed")
 			}
-			return phaseFailed, nil
+			return phaseFailed, false, nil
 		}
 		select {
 		case <-ctx.Done():
 			unquiesce("controller shutting down")
-			return "", ctx.Err()
+			return "", false, ctx.Err()
 		case <-time.After(l.statusPoll):
 		}
 	}