v0.218.0: attribute a DB container by its compose project, and replay volumes on the off-site restore

R-355 (first, because it is the only one where data can be lost for good). paperless-ngx's PostgreSQL was dumped into backups/primary/paperless/db-dumps/ — a directory for a stack that does not exist, on the system drive — while the app's own unit recorded db_dumps: null. The same misattribution reached writeSafetyDump, so a destructive restore of that app took NO undo copy and the fail-closed refusal was never reached. Fixed by reading the compose project label, which is the stack name by construction (compose runs with cmd.Dir set to the stack dir and no -p). The old derivation stays as the fallback and an unresolvable attribution is now loud. Catalogue sweep, proven able to convict: one affected app of 53. The fix is in the controller, not the catalogue. R-354. ReconstituteFromOffsite skipped every unit placement and the volume archives live inside the unit, so the off-site restore had no volume leg at all — proven live with planted files: calibre-web's 1,422,848-byte config archive was in the unit, the snapshot and the checking folder, and the restore reported success without it. For the 40 of 53 apps that declare no data drive that archive is the whole dataset. restoreDockerVolumesFrom is the local path's own replay with an explicit directory: ONE implementation, two callers. Volumes replay before the database and inside the stopped window. VolumesReplayed reaches the message. The comment beside the skip was half false and is corrected; the half that still holds — the live unit is the local path's source — is named, and scenario D fingerprints the whole live unit across the operation. Seven red-proofs, each asserted applied and reverted. Two found defects in the tests, not the code: scenario D passed with the unit guard removed because the fingerprint had been narrowed and was blind to the unit root.
2026-08-22 09:43:22 +02:00
parent f94543ee5c
commit 5ce3a44645
10 changed files with 871 additions and 24 deletions
@@ -0,0 +1,216 @@
+package backup
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+// R-354. The off-site reconstitution replayed the snapshot's DATABASE and never its named VOLUMES,
+// because the volume archives live inside the recovery unit and the unit placement is (correctly)
+// skipped. Measured live 2026-08-21: calibre-web's 1 422 848-byte `calibre_web_config.tar` was in the
+// unit, in the off-site snapshot and in the verification folder, and the restore returned five files,
+// reported success, and did not replay it. For the 40 of 53 catalogue apps that declare no data drive,
+// that archive is the entire dataset.
+
+// seedScratchVolumes writes volume tars into the scratch unit the reconstitution will read from, and
+// returns that directory.
+func seedScratchVolumes(t *testing.T, m *Manager, stack string, names ...string) string {
+	t.Helper()
+	scratch, _, err := m.offboxRestoreScratchDir(stack)
+	if err != nil {
+		t.Fatal(err)
+	}
+	unit := findScratchUnitDir(scratch, stack)
+	if unit == "" {
+		t.Fatalf("no scratch unit dir for %s under %s", stack, scratch)
+	}
+	dir := filepath.Join(unit, "volume-dumps")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range names {
+		if err := os.WriteFile(filepath.Join(dir, n), []byte("tar:"+n), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+func fingerprintTree(t *testing.T, root string) string {
+	t.Helper()
+	var lines []string
+	_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
+		if err != nil || fi.IsDir() {
+			return nil
+		}
+		b, rErr := os.ReadFile(p)
+		if rErr != nil {
+			return nil
+		}
+		// The pre-restore undo copies are the ONE documented write into the live unit; everything
+		// else in the tree must be byte-identical across the operation.
+		if strings.HasPrefix(filepath.Base(p), preRestoreDumpPrefix) {
+			return nil
+		}
+		sum := sha256.Sum256(b)
+		rel, _ := filepath.Rel(root, p)
+		lines = append(lines, rel+":"+hex.EncodeToString(sum[:]))
+		return nil
+	})
+	sort.Strings(lines)
+	return strings.Join(lines, "\n")
+}
+
+// TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack is the case that matters: an app whose data is
+// entirely in a named volume. Before the fix this returned nothing and said it had succeeded.
+func TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack(t *testing.T) {
+	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", "")
+	// A volume-only app places no files at all — the shape that reported "0 fájl" and success.
+	m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 0, nil })
+	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
+	wantDir := seedScratchVolumes(t, m, "immich", "immich_immich_data.tar")
+
+	var gotDir string
+	var gotStack string
+	m.volumeReplayFrom = func(stack, dumpDir string) (int, error) {
+		gotStack, gotDir = stack, dumpDir
+		return 1, nil
+	}
+
+	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	if err != nil {
+		t.Fatalf("ReconstituteFromOffsite: %v", err)
+	}
+	if res.VolumesReplayed != 1 {
+		t.Errorf("VolumesReplayed = %d, want 1 — the snapshot's volume did not come back", res.VolumesReplayed)
+	}
+	if gotStack != "immich" {
+		t.Errorf("replayed for stack %q, want %q", gotStack, "immich")
+	}
+	// It must read the SCRATCH unit, never the live one.
+	if gotDir != wantDir {
+		t.Errorf("volume replay read %q, want the scratch unit's %q", gotDir, wantDir)
+	}
+}
+
+// TestR354_ScenarioB_BothLegsReturnAndAreCounted — declared files AND a volume.
+func TestR354_ScenarioB_BothLegsReturn(t *testing.T) {
+	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
+	seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar")
+	m.volumeReplayFrom = func(_, _ string) (int, error) { return 2, nil }
+
+	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	if err != nil {
+		t.Fatalf("ReconstituteFromOffsite: %v", err)
+	}
+	if res.FilesPlaced != 3 {
+		t.Errorf("FilesPlaced = %d, want 3", res.FilesPlaced)
+	}
+	if res.VolumesReplayed != 2 {
+		t.Errorf("VolumesReplayed = %d, want 2", res.VolumesReplayed)
+	}
+	if res.DBsReplayed != 1 {
+		t.Errorf("DBsReplayed = %d, want 1", res.DBsReplayed)
+	}
+}
+
+// TestR354_ScenarioC_NoVolumesIsUnchanged — a snapshot with no volume archives must behave exactly as
+// before. The real helper runs here (no seam), so the absent-directory path is the one under test.
+func TestR354_ScenarioC_NoVolumeArchivesIsANoOp(t *testing.T) {
+	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
+	// deliberately NO seedScratchVolumes and NO seam
+
+	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	if err != nil {
+		t.Fatalf("a snapshot without volume archives must not fail the restore: %v", err)
+	}
+	if res.VolumesReplayed != 0 {
+		t.Errorf("VolumesReplayed = %d, want 0", res.VolumesReplayed)
+	}
+}
+
+// TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten. The skip that caused R-354 also protects the
+// local restore path's own source, and that reason still holds. Fingerprint the live unit across the
+// whole operation and compare — the doctrine's own answer to the R-181 class, where every test
+// asserted a mechanism inside one function and the tree still moved.
+func TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten(t *testing.T) {
+	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
+	_, liveNs, err := m.offboxRestoreScratchDir("immich")
+	if err != nil {
+		t.Fatal(err)
+	}
+	liveUnit := RecoveryUnitPath(liveNs, "immich")
+	liveVols := filepath.Join(liveUnit, "volume-dumps")
+	if err := os.MkdirAll(liveVols, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	// The live unit's own copy — the local restore path's source. It must survive untouched.
+	if err := os.WriteFile(filepath.Join(liveVols, "immich_immich_data.tar"), []byte("THE LIVE UNIT COPY"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(liveUnit, "manifest.json"), []byte(`{"app_name":"immich"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	// Fingerprint the WHOLE live unit. Narrowing this to the volume archives made the test blind to a
+	// placement writing into the unit ROOT — caught by red-proof 6, which passed against the narrowed
+	// version. The excluded set is exactly the pre-restore undo copies, and those are asserted below.
+	before := fingerprintTree(t, liveUnit)
+
+	seedScratchVolumes(t, m, "immich", "immich_immich_data.tar")
+	m.volumeReplayFrom = func(_, _ string) (int, error) { return 1, nil }
+	// A copier that really writes, so "the unit is never written to" is observable rather than assumed.
+	m.SetOffboxFullPlaceCopier(func(src, dst string) (int, error) {
+		_ = os.MkdirAll(dst, 0o755)
+		return 1, os.WriteFile(filepath.Join(dst, "PLACED"), []byte("from the copier"), 0o644)
+	})
+
+	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
+		t.Fatalf("ReconstituteFromOffsite: %v", err)
+	}
+
+	if after := fingerprintTree(t, liveUnit); after != before {
+		t.Errorf("the LIVE recovery unit's backup content changed across the restore — the local path's source was clobbered\nbefore:\n%s\nafter:\n%s", before, after)
+	}
+
+	// The ONE write into the live unit that IS expected: the pre-restore safety dump. It lives in the
+	// app's own db-dumps dir deliberately (see preRestoreDumpPrefix) — it is the undo, and an undo the
+	// customer cannot see is not much of one. Asserted here rather than merely excluded, so "the unit
+	// is untouched" cannot quietly come to mean "the undo stopped being written".
+	undo, _ := filepath.Glob(filepath.Join(liveUnit, "db-dumps", "pre-restore-*.sql"))
+	if len(undo) != 1 {
+		t.Errorf("expected exactly one pre-restore undo copy in the live unit, found %d", len(undo))
+	}
+}
+
+// TestR354_ScenarioE_PartialReplayIsAFailure — a volume replay that fails must never read as a
+// completion, and must name what failed.
+func TestR354_ScenarioE_PartialReplayIsReportedAsFailure(t *testing.T) {
+	m, prov, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
+	seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar")
+	m.volumeReplayFrom = func(_, _ string) (int, error) {
+		return 1, fmt.Errorf("failed to restore 1 volume(s): [immich_b]")
+	}
+
+	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
+	if err == nil {
+		t.Fatal("a partial volume replay must be reported as a failure, not a completion")
+	}
+	if !strings.Contains(err.Error(), "immich_b") {
+		t.Errorf("the failure must name the volume that did not come back; got %q", err.Error())
+	}
+	// Best-effort bring-up: a failed restore must not also be an outage.
+	if !prov.fullStarted {
+		t.Error("the app was left stopped after a failed volume replay")
+	}
+	// The count of what DID come back is still carried, so the report can say "1 of 2".
+	if res.VolumesReplayed != 1 {
+		t.Errorf("VolumesReplayed = %d, want the partial count 1", res.VolumesReplayed)
+	}
+}