R-893 slice 1: a failed off-site replay after files/volumes/version moved holds the app

Decision 192 (D8, option C). After a failed off-site database replay whose
rollback worked, but where the snapshot's definition was written or a named
volume was replaced, the app is no longer started on the mixed state:
the live definition is written back, the database service stopped, and the
app held (HoldReasonRestoreMixed) for support. The sentence (hu+en) says the
app needs help and no longer claims the data is back as it was. The operator
gets a backup_run_failures mail (leg restore-hold-mixed).

The plain case (no version change, no volume replaced) keeps today's
behaviour (TestR379_ScenarioA). Red-proofed: r893_mixed_restore_test.go.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/backup/r893_mixed_restore_test.go

@@ -0,0 +1,135 @@
+package backup
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
+)
+
+// R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md).
+//
+// An off-site restore of one app writes the snapshot's definition (when the version differs), copies the
+// snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails,
+// the rollback puts back only the database rows. Until this slice the app was started on that mix —
+// an older definition, the snapshot's files and volumes, the newer database — and the household was told
+// „your data is back as it was before the restore". That is true only of the database rows.
+//
+// Now: the database is still rolled back, the LIVE definition is written back, and the app is HELD
+// stopped with a sentence that says it needs support. Pinned here:
+//   (a) the definition write is called a second time, with the live definition;
+//   (b) the app is NOT started;
+//   (c) a hold is left (kind restore_mixed), and the operator is notified;
+//   (d) neither the error nor the hold sentence claims the data is back as it was (hu and en).
+// The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA.
+
+// r893Provider records EVERY definition write (vtReconProvider keeps only the last).
+type r893Provider struct {
+	*vtReconProvider
+	defs [][]string
+}
+
+func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error {
+	p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml")))
+	return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env)
+}
+
+func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) {
+	t.Helper()
+	m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1))
+	m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") }
+	rolled := 0
+	m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })
+	notified := 0
+	m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) {
+		notified++
+		if replayErr == nil || rollbackErr != nil {
+			t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr)
+		}
+	})
+	m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil }
+	live := "16"
+	if versionChange {
+		live = "18"
+	}
+	// The LIVE definition, on disk where the app runs.
+	if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}}
+	m.SetStackProvider(vp)
+	vtSnapshotUnit(t, m, "16", true)
+	return m, vp, &rolled, &notified
+}
+
+func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) {
+	t.Helper()
+	if err == nil {
+		t.Fatal("a failed replay must be surfaced as a failure")
+	}
+	if rolled != 1 {
+		t.Fatalf("the database rollback must still run exactly once, ran %d", rolled)
+	}
+	// (b)
+	if vp.fullStarted {
+		t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls)
+	}
+	if last := vp.calls[len(vp.calls)-1]; last != "stop" {
+		t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls)
+	}
+	// (c)
+	held, sentence := m.RestoreHoldFor("immich")
+	if !held {
+		t.Fatal("no hold was left — every start path would start the app on the mixed state")
+	}
+	if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed {
+		t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed)
+	}
+	if notified != 1 {
+		t.Errorf("the operator must be notified once, got %d", notified)
+	}
+	// (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too).
+	for _, s := range []string{err.Error(), sentence} {
+		low := strings.ToLower(s)
+		if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") {
+			t.Errorf("the sentence still claims the data is back / the app runs: %q", s)
+		}
+		if !strings.Contains(low, "kapcsolatot") {
+			t.Errorf("the sentence must send the household to support: %q", s)
+		}
+	}
+	_, en := m.RestoreHoldForLang("immich", "en")
+	if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") {
+		t.Errorf("English hold sentence: %q", en)
+	}
+	if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") {
+		t.Errorf("English error sentence: %q", enErr)
+	}
+}
+
+func TestR893_AVersionChangeThenAFailedReplayHoldsTheAppAtItsLiveDefinition(t *testing.T) {
+	m, vp, rolled, notified := r893Fixture(t, true, 0)
+	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	assertR893Held(t, m, vp, err, *rolled, *notified)
+	// (a)
+	if len(vp.defs) != 2 {
+		t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs)
+	}
+	if got := strings.Join(vp.defs[1], " "); !strings.Contains(got, "postgres:18-alpine") {
+		t.Fatalf("the definition written back is %q — want the live 18", got)
+	}
+}
+
+func TestR893_AReplacedVolumeThenAFailedReplayHoldsTheApp(t *testing.T) {
+	m, vp, rolled, notified := r893Fixture(t, false, 1)
+	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	assertR893Held(t, m, vp, err, *rolled, *notified)
+	if len(vp.defs) != 0 {
+		t.Fatalf("no version changed, yet a definition was written: %v", vp.defs)
+	}
+}