package backup import ( "context" "errors" "os" "path/filepath" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" "gitea.dooplex.hu/admin/felhom-controller/internal/util" ) // R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md). // // An off-site restore of one app writes the snapshot's definition (when the version differs), copies the // snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails, // the rollback puts back only the database rows. Until this slice the app was started on that mix — // an older definition, the snapshot's files and volumes, the newer database — and the household was told // „your data is back as it was before the restore". That is true only of the database rows. // // Slice 1 (option C) HELD the app. Since option A (operator D20 = A, 2026-10-20) the box puts the app back // EXACTLY as it was and starts it: the live definition is written back and the copy of the live volumes is // poured back (r893_putback_test.go pins the volume half). The HOLD remains for the case nothing can undo: // volumes replaced with no copy to put back. Pinned here: // (a) the definition write is called a second time, with the live definition; // (b) put back → the app IS started, no hold, the operator told once (put-back event); // (c) no copy → a hold is left (kind restore_mixed), the app is NOT started, the operator notified; // (d) the held sentences claim neither that the data is back nor that the app runs (hu and en). // The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA. // r893Provider records EVERY definition write (vtReconProvider keeps only the last). type r893Provider struct { *vtReconProvider defs [][]string onStop func() } func (p *r893Provider) StopStack(name string) error { if p.onStop != nil { p.onStop() } return p.vtReconProvider.StopStack(name) } func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error { p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml"))) return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env) } func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) { m, vp, rolled, notified, _ := r893FixtureA(t, versionChange, volumes) return m, vp, rolled, notified } // r893FixtureA is r893Fixture plus the put-back notification counter (option A). func r893FixtureA(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int, *int) { t.Helper() m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1)) m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") } rolled := 0 m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil }) notified := 0 m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) { notified++ if replayErr == nil || rollbackErr != nil { t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr) } }) putBack := 0 m.SetRestorePutBackNotify(func(_ string, replayErr error) { putBack++ if replayErr == nil { t.Error("the put-back notification must carry the error that failed the restore") } }) m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil } live := "16" if versionChange { live = "18" } // The LIVE definition, on disk where the app runs. if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil { t.Fatal(err) } vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}} m.SetStackProvider(vp) vtSnapshotUnit(t, m, "16", true) return m, vp, &rolled, ¬ified, &putBack } // assertR893PutBack is option A's outcome: the app runs again at its live definition, nothing is held. func assertR893PutBack(t *testing.T, m *Manager, vp *r893Provider, err error, notified, putBack int) { t.Helper() if err == nil { t.Fatal("a failed restore must still be surfaced as a failure") } if !vp.fullStarted { t.Fatalf("option A: the app must be started again after the put-back — calls %v", vp.calls) } if held, _ := m.RestoreHoldFor("immich"); held { t.Fatal("option A: a put-back app must NOT be held") } if notified != 0 || putBack != 1 { t.Errorf("hold notices %d (want 0), put-back notices %d (want 1)", notified, putBack) } if len(vp.defs) != 2 || !strings.Contains(strings.Join(vp.defs[1], " "), "postgres:18-alpine") { t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs) } low := strings.ToLower(err.Error()) if !strings.Contains(low, "sikertelen") || !strings.Contains(low, "visszaker") || strings.Contains(low, "kapcsolatot") { t.Errorf("the sentence must say the restore failed and the data is back, with no support call: %q", err.Error()) } } func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) { t.Helper() if err == nil { t.Fatal("a failed replay must be surfaced as a failure") } if rolled != 1 { t.Fatalf("the database rollback must still run exactly once, ran %d", rolled) } // (b) if vp.fullStarted { t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls) } if last := vp.calls[len(vp.calls)-1]; last != "stop" { t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls) } // (c) held, sentence := m.RestoreHoldFor("immich") if !held { t.Fatal("no hold was left — every start path would start the app on the mixed state") } if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed { t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed) } if notified != 1 { t.Errorf("the operator must be notified once, got %d", notified) } // (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too). for _, s := range []string{err.Error(), sentence} { low := strings.ToLower(s) if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") { t.Errorf("the sentence still claims the data is back / the app runs: %q", s) } if !strings.Contains(low, "kapcsolatot") { t.Errorf("the sentence must send the household to support: %q", s) } } _, en := m.RestoreHoldForLang("immich", "en") if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") { t.Errorf("English hold sentence: %q", en) } if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") { t.Errorf("English error sentence: %q", enErr) } } // RED-PROOF (option A): restore the slice-1 branch (holdAppAfterMixedRestore + return held_mixed) after the // rollback → the app is not started and a hold is left → FAILS. func TestR893_AVersionChangeThenAFailedReplayStartsTheAppAtItsLiveDefinition(t *testing.T) { m, vp, rolled, notified, putBack := r893FixtureA(t, true, 0) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if *rolled != 1 { t.Fatalf("the database rollback must still run exactly once, ran %d", *rolled) } assertR893PutBack(t, m, vp, err, *notified, *putBack) } // A volume replaced with NO copy to pour back (no snapshot volume tar was measured) cannot be undone: the hold // stays (option C). In production the copy exists whenever the snapshot holds a volume tar. func TestR893_AReplacedVolumeWithNoCopyThenAFailedReplayHoldsTheApp(t *testing.T) { m, vp, rolled, notified := r893Fixture(t, false, 1) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) assertR893Held(t, m, vp, err, *rolled, *notified) if len(vp.defs) != 0 { t.Fatalf("no version changed, yet a definition was written: %v", vp.defs) } } // Security review 2026-10-08 (G1): EVERY failure after the snapshot's definition is written is undone, not only a // failed replay. Here the DB-only start fails after a version change: before G1 the app was started at the // snapshot's (older) definition on the live database. Since option A the live definition is written back FIRST and // the app then starts at it (calls: … recreate, start — the live definition before the start). // RED-PROOF: restore `restartStack()` as the only action in the StartStackServices failure branch → only one // definition write (the snapshot's) → FAILS. func TestR893_AVersionChangeThenAFailedDBStartStartsTheAppAtItsLiveDefinition(t *testing.T) { m, vp, rolled, notified, putBack := r893FixtureA(t, true, 0) vp.startSvcErr = context.DeadlineExceeded _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if *rolled != 0 { t.Errorf("rollback ran %d (want 0: nothing was replayed)", *rolled) } if err == nil { t.Fatal("a failed DB-only start must be surfaced") } n := len(vp.calls) if n < 2 || vp.calls[n-2] != "recreate" || vp.calls[n-1] != "start" { t.Fatalf("the live definition must be written back BEFORE the start — calls %v", vp.calls) } assertR893PutBack(t, m, vp, err, *notified, *putBack) } // Security review 2026-10-08 (G3): the hold is persisted BEFORE the app is stopped, so a controller that dies in // between cannot restart the app from its app-stop marker with nothing refusing it. The provider's StopStack asserts // the hold is already on disk at the moment it is called. // RED-PROOF: move SetRestoreHold after StopStack in holdAppAfterMixedRestore → FAILS. func TestR893_HoldIsPersistedBeforeTheStop(t *testing.T) { m, vp, _, _ := r893Fixture(t, true, 1) // a replaced volume with no copy: the case that still holds seenAtStop := []bool{} vp.onStop = func() { held, _ := m.RestoreHoldFor("immich") seenAtStop = append(seenAtStop, held) } _, _ = m.ReconstituteFromOffsite(context.Background(), "immich", false) if len(seenAtStop) == 0 || !seenAtStop[len(seenAtStop)-1] { t.Fatalf("the last stop ran before the hold was persisted (held at each stop: %v)", seenAtStop) } }