package backup import ( "context" "crypto/sha256" "encoding/hex" "fmt" "os" "path/filepath" "sort" "strings" "testing" ) // R-354. The off-site reconstitution replayed the snapshot's DATABASE and never its named VOLUMES, // because the volume archives live inside the recovery unit and the unit placement is (correctly) // skipped. Measured live 2026-08-21: calibre-web's 1 422 848-byte `calibre_web_config.tar` was in the // unit, in the off-site snapshot and in the verification folder, and the restore returned five files, // reported success, and did not replay it. For the 40 of 53 catalogue apps that declare no data drive, // that archive is the entire dataset. // seedScratchVolumes writes volume tars into the scratch unit the reconstitution will read from, and // returns that directory. func seedScratchVolumes(t *testing.T, m *Manager, stack string, names ...string) string { t.Helper() scratch, _, err := m.offboxRestoreScratchDir(stack) if err != nil { t.Fatal(err) } unit := findScratchUnitDir(scratch, stack) if unit == "" { t.Fatalf("no scratch unit dir for %s under %s", stack, scratch) } dir := filepath.Join(unit, "volume-dumps") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } for _, n := range names { if err := os.WriteFile(filepath.Join(dir, n), []byte("tar:"+n), 0o644); err != nil { t.Fatal(err) } } return dir } func fingerprintTree(t *testing.T, root string) string { t.Helper() var lines []string _ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error { if err != nil || fi.IsDir() { return nil } b, rErr := os.ReadFile(p) if rErr != nil { return nil } // The pre-restore undo copies are the ONE documented write into the live unit; everything // else in the tree must be byte-identical across the operation. if strings.HasPrefix(filepath.Base(p), preRestoreDumpPrefix) { return nil } sum := sha256.Sum256(b) rel, _ := filepath.Rel(root, p) lines = append(lines, rel+":"+hex.EncodeToString(sum[:])) return nil }) sort.Strings(lines) return strings.Join(lines, "\n") } // TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack is the case that matters: an app whose data is // entirely in a named volume. Before the fix this returned nothing and said it had succeeded. func TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack(t *testing.T) { m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", "") // A volume-only app places no files at all — the shape that reported "0 fájl" and success. m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 0, nil }) m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil } wantDir := seedScratchVolumes(t, m, "immich", "immich_immich_data.tar") var gotDir string var gotStack string m.volumeReplayFrom = func(stack, dumpDir string) (int, error) { gotStack, gotDir = stack, dumpDir return 1, nil } res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err != nil { t.Fatalf("ReconstituteFromOffsite: %v", err) } if res.VolumesReplayed != 1 { t.Errorf("VolumesReplayed = %d, want 1 — the snapshot's volume did not come back", res.VolumesReplayed) } if gotStack != "immich" { t.Errorf("replayed for stack %q, want %q", gotStack, "immich") } // It must read the SCRATCH unit, never the live one. if gotDir != wantDir { t.Errorf("volume replay read %q, want the scratch unit's %q", gotDir, wantDir) } } // TestR354_ScenarioB_BothLegsReturnAndAreCounted — declared files AND a volume. func TestR354_ScenarioB_BothLegsReturn(t *testing.T) { m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1)) seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar") m.volumeReplayFrom = func(_, _ string) (int, error) { return 2, nil } res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err != nil { t.Fatalf("ReconstituteFromOffsite: %v", err) } if res.FilesPlaced != 3 { t.Errorf("FilesPlaced = %d, want 3", res.FilesPlaced) } if res.VolumesReplayed != 2 { t.Errorf("VolumesReplayed = %d, want 2", res.VolumesReplayed) } if res.DBsReplayed != 1 { t.Errorf("DBsReplayed = %d, want 1", res.DBsReplayed) } } // TestR354_ScenarioC_NoVolumesIsUnchanged — a snapshot with no volume archives must behave exactly as // before. The real helper runs here (no seam), so the absent-directory path is the one under test. func TestR354_ScenarioC_NoVolumeArchivesIsANoOp(t *testing.T) { m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1)) // deliberately NO seedScratchVolumes and NO seam res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err != nil { t.Fatalf("a snapshot without volume archives must not fail the restore: %v", err) } if res.VolumesReplayed != 0 { t.Errorf("VolumesReplayed = %d, want 0", res.VolumesReplayed) } } // TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten. The skip that caused R-354 also protects the // local restore path's own source, and that reason still holds. Fingerprint the live unit across the // whole operation and compare — the doctrine's own answer to the R-181 class, where every test // asserted a mechanism inside one function and the tree still moved. func TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten(t *testing.T) { m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1)) _, liveNs, err := m.offboxRestoreScratchDir("immich") if err != nil { t.Fatal(err) } liveUnit := RecoveryUnitPath(liveNs, "immich") liveVols := filepath.Join(liveUnit, "volume-dumps") if err := os.MkdirAll(liveVols, 0o755); err != nil { t.Fatal(err) } // The live unit's own copy — the local restore path's source. It must survive untouched. if err := os.WriteFile(filepath.Join(liveVols, "immich_immich_data.tar"), []byte("THE LIVE UNIT COPY"), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(liveUnit, "manifest.json"), []byte(`{"app_name":"immich"}`), 0o644); err != nil { t.Fatal(err) } // Fingerprint the WHOLE live unit. Narrowing this to the volume archives made the test blind to a // placement writing into the unit ROOT — caught by red-proof 6, which passed against the narrowed // version. The excluded set is exactly the pre-restore undo copies, and those are asserted below. before := fingerprintTree(t, liveUnit) seedScratchVolumes(t, m, "immich", "immich_immich_data.tar") m.volumeReplayFrom = func(_, _ string) (int, error) { return 1, nil } // A copier that really writes, so "the unit is never written to" is observable rather than assumed. m.SetOffboxFullPlaceCopier(func(src, dst string) (int, error) { _ = os.MkdirAll(dst, 0o755) return 1, os.WriteFile(filepath.Join(dst, "PLACED"), []byte("from the copier"), 0o644) }) if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil { t.Fatalf("ReconstituteFromOffsite: %v", err) } if after := fingerprintTree(t, liveUnit); after != before { t.Errorf("the LIVE recovery unit's backup content changed across the restore — the local path's source was clobbered\nbefore:\n%s\nafter:\n%s", before, after) } // The ONE write into the live unit that IS expected: the pre-restore safety dump. It lives in the // app's own db-dumps dir deliberately (see preRestoreDumpPrefix) — it is the undo, and an undo the // customer cannot see is not much of one. Asserted here rather than merely excluded, so "the unit // is untouched" cannot quietly come to mean "the undo stopped being written". undo, _ := filepath.Glob(filepath.Join(liveUnit, "db-dumps", "pre-restore-*.sql")) if len(undo) != 1 { t.Errorf("expected exactly one pre-restore undo copy in the live unit, found %d", len(undo)) } } // TestR354_ScenarioE_PartialReplayIsAFailure — a volume replay that fails must never read as a // completion, and must name what failed. func TestR354_ScenarioE_PartialReplayIsReportedAsFailure(t *testing.T) { m, prov, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1)) seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar") m.volumeReplayFrom = func(_, _ string) (int, error) { return 1, fmt.Errorf("failed to restore 1 volume(s): [immich_b]") } res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a partial volume replay must be reported as a failure, not a completion") } if !strings.Contains(err.Error(), "immich_b") { t.Errorf("the failure must name the volume that did not come back; got %q", err.Error()) } // Best-effort bring-up: a failed restore must not also be an outage. if !prov.fullStarted { t.Error("the app was left stopped after a failed volume replay") } // The count of what DID come back is still carried, so the report can say "1 of 2". if res.VolumesReplayed != 1 { t.Errorf("VolumesReplayed = %d, want the partial count 1", res.VolumesReplayed) } }