R-893 slice 1: a failed off-site replay after files/volumes/version moved holds the app

Decision 192 (D8, option C). After a failed off-site database replay whose
rollback worked, but where the snapshot's definition was written or a named
volume was replaced, the app is no longer started on the mixed state:
the live definition is written back, the database service stopped, and the
app held (HoldReasonRestoreMixed) for support. The sentence (hu+en) says the
app needs help and no longer claims the data is back as it was. The operator
gets a backup_run_failures mail (leg restore-hold-mixed).

The plain case (no version change, no volume replaced) keeps today's
behaviour (TestR379_ScenarioA). Red-proofed: r893_mixed_restore_test.go.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/backup/offbox_reconstitute.go

@@ -369,6 +369,9 @@ func (m *Manager) RestoreHoldForLang(stack, lang string) (bool, string) {
 	if t, err := time.Parse(time.RFC3339, h.At); err == nil {
 		when = t.Format("2006-01-02 15:04")
 	}
+	if h.Reason == settings.HoldReasonRestoreMixed { // R-893
+		return true, util.Text(lang, "note.reconstitute.held_mixed", stack, when)
+	}
 	return true, util.Text(lang, "note.reconstitute.held", stack, when)
 }
 
@@ -441,6 +444,93 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
 	}
 }
 
+// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
+// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
+type liveDefinition struct {
+	dir string
+	env map[string]string
+}
+
+func (d *liveDefinition) cleanup() {
+	if d != nil && d.dir != "" {
+		_ = os.RemoveAll(d.dir)
+	}
+}
+
+// captureLiveDefinition copies the live docker-compose.yml (required) and .felhom.yml (when present) and
+// reads the live env (non-secret values plus the live secrets, decrypted from the guest's app.yaml) — the
+// same two sources the restore path uses. Values are never logged.
+func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
+	composePath, ok := m.stackProvider.GetStackComposePath(stack)
+	if !ok || composePath == "" {
+		return nil, fmt.Errorf("no live compose path for %s", stack)
+	}
+	dir, err := os.MkdirTemp("", "felhom-livedef-")
+	if err != nil {
+		return nil, err
+	}
+	d := &liveDefinition{dir: dir, env: map[string]string{}}
+	for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} {
+		data, rErr := os.ReadFile(filepath.Join(filepath.Dir(composePath), fname))
+		if rErr != nil {
+			if fname == "docker-compose.yml" {
+				d.cleanup()
+				return nil, rErr
+			}
+			continue
+		}
+		if wErr := os.WriteFile(filepath.Join(dir, fname), data, 0o600); wErr != nil {
+			d.cleanup()
+			return nil, wErr
+		}
+	}
+	if info, ok := m.stackProvider.GetStackRecoveryInfo(stack); ok {
+		for k, v := range info.NonSecretEnv {
+			d.env[k] = v
+		}
+		for k, v := range m.stackProvider.RecoverStackSecrets(stack, info.SecretEnvVars) {
+			d.env[k] = v
+		}
+	}
+	return d, nil
+}
+
+// holdAppAfterMixedRestore (R-893, `09` §3 decision 192, option C) is the hold for a failed replay whose
+// database rollback WORKED but which had already moved more than the database: the live definition is
+// written back (when the snapshot's was written), the database service is stopped again, and the app is
+// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
+// notification tells this case from R-379's double failure.
+func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
+	if live != nil {
+		if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
+			m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the app is held at the snapshot's definition", stack, err)
+		} else {
+			m.logger.Printf("[INFO] [offbox] %s: the live definition is written back", stack)
+		}
+	}
+	if err := m.stackProvider.StopStack(stack); err != nil {
+		m.logger.Printf("[WARN] [offbox] %s: stopping the database service before the hold failed: %v", stack, err)
+	}
+	m.logger.Printf("[ERROR] [offbox] %s: database rolled back, but files/volumes/version had already moved — HOLDING the app stopped for support (R-893); replay error was: %v", stack, replayErr)
+	if m.settings == nil {
+		m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
+		return
+	}
+	h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
+	if replayErr != nil {
+		h.ReplayError = replayErr.Error()
+	}
+	if err := m.settings.SetRestoreHold(h); err != nil {
+		m.logger.Printf("[ERROR] [offbox] %s: persisting the restore hold FAILED: %v — the app is stopped and unguarded", stack, err)
+	}
+	if m.appStop != nil {
+		m.appStop.End()
+	}
+	if m.restoreHoldNotify != nil {
+		m.restoreHoldNotify(stack, replayErr, nil)
+	}
+}
+
 // SetRestoreHoldNotify wires the operator notification for a held app (cmd/controller/main.go).
 func (m *Manager) SetRestoreHoldNotify(fn func(stack string, replayErr, rollbackErr error)) {
 	m.restoreHoldNotify = fn
@@ -732,6 +822,21 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 		}
 	}
 
+	// --- THE LIVE DEFINITION, KEPT FOR A FAILED REPLAY (R-893, `09` §3 decision 192) -------------
+	// When the snapshot's definition is about to be written, keep the LIVE one first, so a failed
+	// replay below can write it back before the app is held. Taken before the first mutation; a
+	// failure here refuses with nothing touched — without it the failure branch could not be undone.
+	var liveDef *liveDefinition
+	if defineFromSnapshot {
+		ld, ldErr := m.captureLiveDefinition(stack)
+		if ldErr != nil {
+			m.logger.Printf("[ERROR] [offbox] Restore REFUSED for %s: could not keep the live definition: %v — nothing was touched", stack, ldErr)
+			return res, fmt.Errorf("restoring %s: keeping the live definition failed: %w", stack, ldErr)
+		}
+		liveDef = ld
+		defer liveDef.cleanup()
+	}
+
 	// --- WHICH SERVICE HOLDS THE DATABASE (R-47) ------------------------------------------------
 	// Read from the compose that will RUN: the live one, or — when the app comes back at the snapshot's
 	// version — the snapshot unit's, which is written into the stack dir before the first start. Resolved
@@ -907,6 +1012,16 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
 				return res, util.MsgError("err.backup.db_restore_and_rollback_failed", stack, m.undoCopyPhrase(safetySet))
 			}
 			res.RolledBack = true
+			// --- R-893: A ROLLED-BACK DATABASE IS NOT „BACK AS IT WAS" WHEN MORE MOVED ---------
+			// The rollback puts back the database rows only. When the snapshot's definition was
+			// written or a named volume was replaced, the app would start on a mix: the snapshot's
+			// files and volumes (and maybe its older version) under the newer database. Write the
+			// live definition back, stop the database service again, and HOLD the app for support.
+			// Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA.
+			if defineFromSnapshot || res.VolumesReplayed > 0 {
+				m.holdAppAfterMixedRestore(stack, iErr, liveDef)
+				return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack)
+			}
 			if sErr := restartStack(); sErr != nil {
 				m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
 			}