R-893: hold the app after ANY failure once the definition or a volume moved; hold persisted before the stop; run_job done says it ran, not what it found (security review)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/backup/offbox_reconstitute.go

@@ -501,6 +501,23 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
 // held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
 // notification tells this case from R-379's double failure.
 func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
+	// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
+	// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
+	// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
+	held := false
+	if m.settings == nil {
+		m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
+	} else {
+		h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
+		if replayErr != nil {
+			h.ReplayError = replayErr.Error()
+		}
+		if err := m.settings.SetRestoreHold(h); err != nil {
+			m.logger.Printf("[ERROR] [offbox] %s: persisting the restore hold FAILED: %v — the app is stopped and the app-stop marker is kept", stack, err)
+		} else {
+			held = true
+		}
+	}
 	if live != nil {
 		if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
 			m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the app is held at the snapshot's definition", stack, err)
@@ -509,21 +526,10 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
 		}
 	}
 	if err := m.stackProvider.StopStack(stack); err != nil {
-		m.logger.Printf("[WARN] [offbox] %s: stopping the database service before the hold failed: %v", stack, err)
+		m.logger.Printf("[WARN] [offbox] %s: stopping the app before the hold failed: %v", stack, err)
 	}
-	m.logger.Printf("[ERROR] [offbox] %s: database rolled back, but files/volumes/version had already moved — HOLDING the app stopped for support (R-893); replay error was: %v", stack, replayErr)
-	if m.settings == nil {
-		m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
-		return
-	}
-	h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
-	if replayErr != nil {
-		h.ReplayError = replayErr.Error()
-	}
-	if err := m.settings.SetRestoreHold(h); err != nil {
-		m.logger.Printf("[ERROR] [offbox] %s: persisting the restore hold FAILED: %v — the app is stopped and unguarded", stack, err)
-	}
-	if m.appStop != nil {
+	m.logger.Printf("[ERROR] [offbox] %s: the restore stopped half-way after files/volumes/version had moved — HOLDING the app stopped for support (R-893); error was: %v", stack, replayErr)
+	if held && m.appStop != nil {
 		m.appStop.End()
 	}
 	if m.restoreHoldNotify != nil {
@@ -935,8 +941,13 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 		}
 		n, cErr := copier(pl.src, pl.dst)
 		if cErr != nil {
-			// Best-effort bring-up: leaving the app stopped after a partial copy would turn a failed
-			// restore into an outage.
+			// R-893 option C (security review 2026-10-08): once the snapshot's (possibly older) definition
+			// is written, a start would run that version on the live data — HOLD the app instead.
+			if defineFromSnapshot {
+				m.holdAppAfterMixedRestore(stack, cErr, liveDef)
+				return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
+			}
+			// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
 			if sErr := restartStack(); sErr != nil {
 				m.logger.Printf("[WARN] [offbox] %s: restart after failed placement also failed: %v", stack, sErr)
 			}
@@ -961,8 +972,16 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 	nVols, vErr := volReplay(stack, filepath.Join(scratchUnit, "volume-dumps"))
 	res.VolumesReplayed = nVols
 	if vErr != nil {
-		// A partial replay must never read as a completion. Bring the app back up rather than leaving
-		// an outage, then surface it — the same shape the file leg above uses.
+		// A partial replay must never read as a completion. R-893 (security review 2026-10-08): a volume
+		// replay that failed may have removed or half-replaced a volume, and the definition and files may
+		// already be the snapshot's — HOLD the app rather than start it on a mix.
+		// R-893 option C (security review 2026-10-08): a volume already replaced, or the snapshot's
+		// definition written → HOLD (the volume detail reaches the operator through the hold notice).
+		if defineFromSnapshot || nVols > 0 {
+			m.holdAppAfterMixedRestore(stack, vErr, liveDef)
+			return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
+		}
+		// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
 		if sErr := restartStack(); sErr != nil {
 			m.logger.Printf("[WARN] [offbox] %s: restart after failed volume replay also failed: %v", stack, sErr)
 		}
@@ -977,6 +996,11 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 	// under ON_ERROR_STOP=1 (H4). Starting only the database service closes that window entirely.
 	if hasDB {
 		if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
+			// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
+			if defineFromSnapshot || res.VolumesReplayed > 0 {
+				m.holdAppAfterMixedRestore(stack, err, liveDef)
+				return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
+			}
 			// Best-effort bring-up: a failed restore must not also be an outage.
 			if sErr := restartStack(); sErr != nil {
 				m.logger.Printf("[WARN] [offbox] %s: full start after failed DB-only start also failed: %v", stack, sErr)
@@ -1018,6 +1042,8 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 			// files and volumes (and maybe its older version) under the newer database. Write the
 			// live definition back, stop the database service again, and HOLD the app for support.
 			// Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA.
+			// Placed FILES alone do not hold (option C as ruled): that mix is what „put back exactly as it
+			// was" (option A, the next slice) removes.
 			if defineFromSnapshot || res.VolumesReplayed > 0 {
 				m.holdAppAfterMixedRestore(stack, iErr, liveDef)
 				return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack)