package backup import ( "context" "os" "path/filepath" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // R-43/R-44 (v0.148.0) — the coherent-pair + true-restore tests. // // These exist because the product shipped a restore button for months that could not restore. // DIAG-immich-restore-2026-07-19: 11 photos, files intact, timeline empty, two "successful" // restores that merged 0 files and never touched postgres. Every test below asserts a behaviour // whose absence produced that outcome, so each one is a regression guard for a real incident // rather than a description of the current implementation. // recordingProvider records stop/start call ORDER so the reconstitution sequence can be asserted. // // R-47 widened it: it now also records the DB-ONLY bring-up and, critically, whether a FULL start // has happened yet — the state the replay must observe as `false`. That single flag is what // separates the fixed sequence from the one that produced H4, in which the whole stack was already // up (and rebuilding its own schema) when the dump replay began. type recordingProvider struct { offbox3aProvider calls []string composePath string // the LIVE compose the DB-service resolver reads gotServices []string // services passed to StartStackServices fullStarted bool // a FULL StartStack has happened startSvcErr error // injected StartStackServices failure } func (p *recordingProvider) StopStack(string) error { p.calls = append(p.calls, "stop"); return nil } func (p *recordingProvider) StartStack(string) error { p.fullStarted = true p.calls = append(p.calls, "start") return nil } func (p *recordingProvider) StartStackServices(_ string, services []string) error { p.gotServices = append([]string(nil), services...) p.calls = append(p.calls, "startsvc:"+strings.Join(services, ",")) return p.startSvcErr } func (p *recordingProvider) GetStackComposePath(string) (string, bool) { return p.composePath, p.composePath != "" } // The app really is up again after StartStack, so the post-restore health wait returns at once. // Leaving it false would make each test sit through the full 90s deadline. func (p *recordingProvider) RefreshAndIsRunning(string) bool { return true } // recoveryProvider adds the recovery info CaptureRecoveryUnit needs (the shared 3a provider has none). type recoveryProvider struct { offbox3aProvider stackDir string } func (p *recoveryProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) { return RecoveryInfo{DisplayName: "Immich", StackDir: p.stackDir}, name == "immich" } // pgDump builds a structurally valid postgres dump big enough to clear ValidateDump's 100-byte // floor, with the accounts-table COPY block carrying `rows` rows. The R-44 sniff runs only on a // dump that already passes structural validation, so a toy fixture would silently skip it. func pgDump(rows int) string { const head = `-- PostgreSQL database dump -- Dumped from database version 16.10 SET statement_timeout = 0; SET lock_timeout = 0; SET client_encoding = 'UTF8'; CREATE TABLE public.asset (id uuid NOT NULL); CREATE TABLE public."user" (id uuid NOT NULL, email text); COPY public."user" (id, email) FROM stdin; ` var b strings.Builder b.WriteString(head) for i := 0; i < rows; i++ { b.WriteString("id-x\tuser@example.invalid\n") } b.WriteString("\\.\n") // the COPY-block terminator return b.String() } // reconFixture builds a manager with a COMPLETED full scratch for `immich`, a snapshot whose unit // carries the given coherence stamp, and injectable copy/dump/import seams. func reconFixture(t *testing.T, runID, dumpsAt string, dumpBody string) (*Manager, *recordingProvider, *[]string) { t.Helper() drive := t.TempDir() m, sett := newOffboxManager(t) prov := &recordingProvider{offbox3aProvider: offbox3aProvider{ hdd: map[string]string{"immich": drive}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{}, }} m.SetStackProvider(prov) if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "USB", Schedulable: true}); err != nil { t.Fatal(err) } // The LIVE compose the reconstitution reads to learn WHICH service holds the database (R-47). // Immich-shaped on purpose: an app service, a redis service that must never be mistaken for a // database, and a top-level `volumes:` key whose entry looks exactly like a service to a line scan. liveStackDir := t.TempDir() prov.composePath = filepath.Join(liveStackDir, "docker-compose.yml") if err := os.WriteFile(prov.composePath, []byte(immichLikeCompose), 0o644); err != nil { t.Fatal(err) } scratch, liveNs, err := m.offboxRestoreScratchDir("immich") if err != nil { t.Fatal(err) } oldNs := "/felhomdata/ns" unitP := oldNs + "/backups/primary/immich" dataP := oldNs + "/appdata/immich" placements, err := mapOffsiteRestorePaths([]string{unitP, dataP}, "immich", scratch, liveNs) if err != nil { t.Fatal(err) } for _, pl := range placements { if err := os.MkdirAll(pl.src, 0o755); err != nil { t.Fatal(err) } if pl.isUnit { dd := filepath.Join(pl.src, "db-dumps") if err := os.MkdirAll(dd, 0o755); err != nil { t.Fatal(err) } if dumpBody != "" { if err := os.WriteFile(filepath.Join(dd, "immich-postgres.sql"), []byte(dumpBody), 0o644); err != nil { t.Fatal(err) } } man := &RecoveryManifest{SchemaVersion: 1, AppName: "immich", OffsiteRunID: runID, DumpsAt: dumpsAt} if err := writeManifest(filepath.Join(pl.src, "manifest.json"), man); err != nil { t.Fatal(err) } } } m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 }) m.SetOffboxSizer(func(string) int64 { return 1 << 20 }) m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) { if contains(args, "snapshots") { return []byte(`[{"short_id":"snap1","time":"2026-07-19T06:00:00Z","paths":["` + unitP + `","` + dataP + `"]}]`), nil } return nil, nil }) // Seams: one DB, a safety dump that really writes a file, and a recording importer. db := DiscoveredDB{StackName: "immich", ContainerName: "immich-postgres", DBType: DBTypePostgres} m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return []DiscoveredDB{db}, nil } m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, dir string) DumpResult { p := filepath.Join(dir, "immich-postgres.sql") _ = os.MkdirAll(dir, 0o755) _ = os.WriteFile(p, []byte(pgDump(1)), 0o644) return DumpResult{DB: d, FilePath: p, Size: 42} }) var imported []string m.importDBDump = func(_ context.Context, _ DiscoveredDB, p string) error { imported = append(imported, p) return nil } m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 3, nil }) return m, prov, &imported } // TestReconstituteReplaysDBAndOrdersOperations is Scenario C: the whole point of R-43. A restore of // a DB-indexed app must stop the app, place files, restart it and REPLAY the snapshot's dump — and // the safety dump must exist before any of it. Before v0.148.0 the replay simply did not happen, // which is why the photos never came back. func TestReconstituteReplaysDBAndOrdersOperations(t *testing.T) { m, prov, imported := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1)) res, err := m.ReconstituteFromOffsite(context.Background(), "immich") if err != nil { t.Fatalf("reconstitute: %v", err) } if res.DBsReplayed != 1 { t.Fatalf("expected the snapshot dump to be replayed exactly once, got %d — this is the R-43 defect", res.DBsReplayed) } if len(*imported) != 1 || !strings.Contains((*imported)[0], "immich-postgres.sql") { t.Fatalf("expected an import of the snapshot dump, got %v", *imported) } // The dump replayed must come from the SCRATCH unit, never the live one: the live unit is // deliberately not overwritten, so replaying from it would replay the CURRENT database back over // itself and restore nothing. if !strings.Contains((*imported)[0], "offsite-restore") { t.Fatalf("replay source must be the restored scratch unit, got %s", (*imported)[0]) } if res.FilesPlaced != 3 { t.Fatalf("expected the userdata placement to be counted, got %d", res.FilesPlaced) } // stop BEFORE the file copy; then ONLY the database service up for the replay (R-47 — a full // start here is the H4 race); the full start comes last. if got := strings.Join(prov.calls, ","); got != "stop,startsvc:immich-postgres,start" { t.Fatalf("expected stop → db-only start → full start around the restore, got %q", got) } if res.SafetyDump == "" { t.Fatal("no safety dump recorded — the undo must exist") } if _, err := os.Stat(res.SafetyDump); err != nil { t.Fatalf("safety dump not on disk: %v", err) } if !strings.HasPrefix(filepath.Base(res.SafetyDump), preRestoreDumpPrefix) { t.Fatalf("safety dump must carry the pre-restore prefix so it is never replayed as a source, got %s", filepath.Base(res.SafetyDump)) } } // TestReconstituteRefusesWhenSafetyDumpFails is the RED-PROOF for the undo invariant: a replay whose // previous state was not captured is an overwrite with no way back, so it must not happen at all — // and it must abort with the live app untouched (no stop, no copy). func TestReconstituteRefusesWhenSafetyDumpFails(t *testing.T) { m, prov, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1)) m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult { return DumpResult{DB: d, Error: context.DeadlineExceeded} }) var copied bool m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { copied = true; return 1, nil }) _, err := m.ReconstituteFromOffsite(context.Background(), "immich") if err == nil { t.Fatal("expected a refusal when the safety dump cannot be taken") } if len(*imported) != 0 { t.Fatalf("REPLAYED WITHOUT AN UNDO — the exact thing the invariant forbids: %v", *imported) } if copied { t.Fatal("files were overwritten despite the refusal — the abort must leave live data untouched") } if len(prov.calls) != 0 { t.Fatalf("the app was stopped despite the refusal, got %v", prov.calls) } } // TestReconstituteNoDBAppMakesNoDumpOrImportCalls is Scenario E: an app without a database must flow // exactly as before — no safety dump, no replay — so the new leg cannot regress the simple case. func TestReconstituteNoDBAppMakesNoDumpOrImportCalls(t *testing.T) { m, _, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "") m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil } dumped := 0 m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult { dumped++ return DumpResult{DB: d} }) res, err := m.ReconstituteFromOffsite(context.Background(), "immich") if err != nil { t.Fatalf("reconstitute: %v", err) } if dumped != 0 { t.Fatalf("a no-DB app must not produce a safety dump, got %d call(s)", dumped) } if len(*imported) != 0 { t.Fatalf("a no-DB app must not import anything, got %v", *imported) } if res.SafetyDump != "" || res.DBsReplayed != 0 { t.Fatalf("unexpected DB activity: safety=%q replayed=%d", res.SafetyDump, res.DBsReplayed) } } // TestReconstituteSurfacesLegacySkewedPair is Scenario D: a pre-v0.148 snapshot carries no coherence // stamp, so its two halves may be from different times. That must be SURFACED (and reversible), never // blocked — the customer's own judgement is the gate, and refusing would deny a legitimate restore. func TestReconstituteSurfacesLegacySkewedPair(t *testing.T) { m, _, imported := reconFixture(t, "", "", pgDump(1)) res, err := m.ReconstituteFromOffsite(context.Background(), "immich") if err != nil { t.Fatalf("a legacy pair must still be restorable, got refusal: %v", err) } if !res.Skewed { t.Fatal("an unstamped (pre-v0.148) snapshot must report Skewed so the confirm can say so") } if len(*imported) != 1 { t.Fatalf("the legacy restore must still replay, got %v", *imported) } } // TestReconstituteFlagsCustomerEmptyDump is the R-44 sniff at the restore end: the immich dump that // started all of this was structurally valid and contained zero users. Restoring it is allowed, but // the customer must be told before they commit. func TestReconstituteFlagsCustomerEmptyDump(t *testing.T) { // A valid postgres dump whose accounts table has NO rows — the 2026-07-19 shape exactly. m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(0)) res, err := m.ReconstituteFromOffsite(context.Background(), "immich") if err != nil { t.Fatalf("the sniff must never block a restore: %v", err) } if !res.LooksEmpty { t.Fatal("a dump with an empty accounts table must raise the warn-level signal") } } // TestOffsiteScratchPairReportsWhatTheConfirmNeeds covers the page-render surface: the confirm can // only be honest if this reports the pair's age and warnings before anything is started. func TestOffsiteScratchPairReportsWhatTheConfirmNeeds(t *testing.T) { m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "-- PostgreSQL database dump\nCREATE TABLE a();\nCOPY public.\"user\" (id) FROM stdin;\n7\n\\.\n") info := m.OffsiteScratchPair("immich") if !info.Ready || !info.HasDump { t.Fatalf("expected a ready pair with a dump, got %+v", info) } if info.Skewed { t.Fatal("a stamped snapshot must not be reported as skewed") } if info.LooksEmpty { t.Fatal("a dump with account rows must not be flagged empty") } want, _ := time.Parse(time.RFC3339, "2026-07-19T06:00:00Z") if !info.DumpsAt.Equal(want) { t.Fatalf("DumpsAt = %v, want %v", info.DumpsAt, want) } } // --- R-44: the coherence pre-phase ----------------------------------------------------------- // TestOffsiteRunDumpsBeforeCapture is Scenarios A + B. The ORDER is the entire mechanism: dumps // must be refreshed BEFORE restic captures, so the snapshot pairs this run's database with this // run's files. Reversed, the snapshot would hold rows pointing at files that were never captured. // // It also asserts the ordering on the NIGHTLY entry point (RunOffboxBackup, no progress sink), not // just the manual one — before v0.148.0 the nightly ordering was an accident of two independent // scheduler entries at 02:30 and 04:15, which a schedule edit could silently invert. func TestOffsiteRunDumpsBeforeCapture(t *testing.T) { drive := t.TempDir() m, sett, prov := classifiedOffboxManager(t, drive) mkUnit(t, drive, "immich") if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil { t.Fatal(err) } prov.hdd["immich"] = drive prov.has["immich"] = true prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")} _ = sett.SetAppOffbox("immich", true) var order []string m.SetOffsitePreDumpFn(func(context.Context) error { order = append(order, "dump") return nil }) m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) { switch { case contains(args, "cat") && contains(args, "config"): return []byte(`{"version":2}`), nil case contains(args, "backup"): order = append(order, "capture") return nil, nil case contains(args, "snapshots"): return []byte(`[]`), nil case contains(args, "stats"): return []byte(`{"total_size":123}`), nil } return nil, nil }) if err := m.RunOffboxBackup(context.Background()); err != nil { t.Fatalf("run: %v", err) } if len(order) < 2 { t.Fatalf("expected both a dump and a capture, got %v", order) } if order[0] != "dump" { t.Fatalf("the dump leg MUST precede the capture (R-44); got %v", order) } if order[1] != "capture" { t.Fatalf("expected the capture immediately after the dump, got %v", order) } } // TestOffsiteRunContinuesWhenDumpLegFails is the data-first rule: a dump failure degrades the // snapshot's DB half but must NOT abort the push. Refusing to ship the files would turn a partial // backup into no backup at all — strictly worse for the customer. func TestOffsiteRunContinuesWhenDumpLegFails(t *testing.T) { drive := t.TempDir() m, sett, prov := classifiedOffboxManager(t, drive) mkUnit(t, drive, "immich") if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil { t.Fatal(err) } prov.hdd["immich"] = drive prov.has["immich"] = true prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")} _ = sett.SetAppOffbox("immich", true) m.SetOffsitePreDumpFn(func(context.Context) error { return context.DeadlineExceeded }) cap := &backupCapture{} m.SetOffboxRunner(cap.runner()) if err := m.RunOffboxBackup(context.Background()); err != nil { t.Fatalf("a dump failure must not fail the whole run: %v", err) } if cap.backups != 1 { t.Fatalf("the files must still be pushed after a dump failure, got %d capture(s)", cap.backups) } } // TestCaptureRecoveryUnitStampsAndCarriesRunID covers the stamp that makes a pair verifiable at // restore time, and the trap beside it: the PERIODIC refresh must neither invent a coherence claim // nor erase one a real run established. func TestCaptureRecoveryUnitStampsAndCarriesRunID(t *testing.T) { drive := t.TempDir() m, _, base := classifiedOffboxManager(t, drive) base.hdd["immich"] = drive // CaptureRecoveryUnit needs real recovery info + a compose dir to read; the shared fixture // provider returns none, so wrap it rather than widening a struct four other test files use. stackDir := filepath.Join(t.TempDir(), "immich") if err := os.MkdirAll(stackDir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(stackDir, "docker-compose.yml"), []byte("services: {}\n"), 0o644); err != nil { t.Fatal(err) } m.SetStackProvider(&recoveryProvider{offbox3aProvider: *base, stackDir: stackDir}) // 1) A run in flight stamps the manifest. end := m.beginOffsiteRunStamp("run-A") if err := m.CaptureRecoveryUnit("immich"); err != nil { t.Fatalf("capture: %v", err) } end() man := readManifest(RecoveryUnitManifestPath(drive, "immich")) if man == nil || man.OffsiteRunID != "run-A" { t.Fatalf("expected the in-flight run id to be stamped, got %+v", man) } if man.DumpsAt == "" { t.Fatal("a stamped unit must record when its dumps were taken") } // 2) A periodic refresh (no run in flight) must CARRY the stamp forward, not blank it — a unit // that silently lost its stamp would be re-reported as a skewed legacy pair at restore time. if err := m.CaptureRecoveryUnit("immich"); err != nil { t.Fatalf("refresh: %v", err) } man2 := readManifest(RecoveryUnitManifestPath(drive, "immich")) if man2 == nil || man2.OffsiteRunID != "run-A" { t.Fatalf("the periodic refresh erased the coherence stamp: %+v", man2) } // 3) A NEW run re-stamps even though nothing else about the unit changed — the idempotent-skip // must not swallow the one field the restore path reads. end2 := m.beginOffsiteRunStamp("run-B") if err := m.CaptureRecoveryUnit("immich"); err != nil { t.Fatalf("capture 2: %v", err) } end2() man3 := readManifest(RecoveryUnitManifestPath(drive, "immich")) if man3 == nil || man3.OffsiteRunID != "run-B" { t.Fatalf("a new run must re-stamp the unit, got %+v", man3) } }