felhom-controller/controller/internal/backup/r354_volume_replay_test.go

package backup

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"
)

// R-354. The off-site reconstitution replayed the snapshot's DATABASE and never its named VOLUMES,
// because the volume archives live inside the recovery unit and the unit placement is (correctly)
// skipped. Measured live 2026-08-21: calibre-web's 1 422 848-byte `calibre_web_config.tar` was in the
// unit, in the off-site snapshot and in the verification folder, and the restore returned five files,
// reported success, and did not replay it. For the 40 of 53 catalogue apps that declare no data drive,
// that archive is the entire dataset.

// seedScratchVolumes writes volume tars into the scratch unit the reconstitution will read from, and
// returns that directory.
func seedScratchVolumes(t *testing.T, m *Manager, stack string, names ...string) string {
	t.Helper()
	scratch, _, err := m.offboxRestoreScratchDir(stack)
	if err != nil {
		t.Fatal(err)
	}
	unit := findScratchUnitDir(scratch, stack)
	if unit == "" {
		t.Fatalf("no scratch unit dir for %s under %s", stack, scratch)
	}
	dir := filepath.Join(unit, "volume-dumps")
	if err := os.MkdirAll(dir, 0o755); err != nil {
		t.Fatal(err)
	}
	for _, n := range names {
		if err := os.WriteFile(filepath.Join(dir, n), []byte("tar:"+n), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

func fingerprintTree(t *testing.T, root string) string {
	t.Helper()
	var lines []string
	_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
		if err != nil || fi.IsDir() {
			return nil
		}
		b, rErr := os.ReadFile(p)
		if rErr != nil {
			return nil
		}
		// The pre-restore undo copies are the ONE documented write into the live unit; everything
		// else in the tree must be byte-identical across the operation.
		if strings.HasPrefix(filepath.Base(p), preRestoreDumpPrefix) {
			return nil
		}
		sum := sha256.Sum256(b)
		rel, _ := filepath.Rel(root, p)
		lines = append(lines, rel+":"+hex.EncodeToString(sum[:]))
		return nil
	})
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

// TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack is the case that matters: an app whose data is
// entirely in a named volume. Before the fix this returned nothing and said it had succeeded.
func TestR354_ScenarioA_VolumeOnlyAppGetsItsVolumeBack(t *testing.T) {
	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", "")
	// A volume-only app places no files at all — the shape that reported "0 fájl" and success.
	m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 0, nil })
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
	wantDir := seedScratchVolumes(t, m, "immich", "immich_immich_data.tar")

	var gotDir string
	var gotStack string
	m.volumeReplayFrom = func(stack, dumpDir string) (int, error) {
		gotStack, gotDir = stack, dumpDir
		return 1, nil
	}

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("ReconstituteFromOffsite: %v", err)
	}
	if res.VolumesReplayed != 1 {
		t.Errorf("VolumesReplayed = %d, want 1 — the snapshot's volume did not come back", res.VolumesReplayed)
	}
	if gotStack != "immich" {
		t.Errorf("replayed for stack %q, want %q", gotStack, "immich")
	}
	// It must read the SCRATCH unit, never the live one.
	if gotDir != wantDir {
		t.Errorf("volume replay read %q, want the scratch unit's %q", gotDir, wantDir)
	}
}

// TestR354_ScenarioB_BothLegsReturnAndAreCounted — declared files AND a volume.
func TestR354_ScenarioB_BothLegsReturn(t *testing.T) {
	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
	seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar")
	m.volumeReplayFrom = func(_, _ string) (int, error) { return 2, nil }

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("ReconstituteFromOffsite: %v", err)
	}
	if res.FilesPlaced != 3 {
		t.Errorf("FilesPlaced = %d, want 3", res.FilesPlaced)
	}
	if res.VolumesReplayed != 2 {
		t.Errorf("VolumesReplayed = %d, want 2", res.VolumesReplayed)
	}
	if res.DBsReplayed != 1 {
		t.Errorf("DBsReplayed = %d, want 1", res.DBsReplayed)
	}
}

// TestR354_ScenarioC_NoVolumesIsUnchanged — a snapshot with no volume archives must behave exactly as
// before. The real helper runs here (no seam), so the absent-directory path is the one under test.
func TestR354_ScenarioC_NoVolumeArchivesIsANoOp(t *testing.T) {
	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
	// deliberately NO seedScratchVolumes and NO seam

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("a snapshot without volume archives must not fail the restore: %v", err)
	}
	if res.VolumesReplayed != 0 {
		t.Errorf("VolumesReplayed = %d, want 0", res.VolumesReplayed)
	}
}

// TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten. The skip that caused R-354 also protects the
// local restore path's own source, and that reason still holds. Fingerprint the live unit across the
// whole operation and compare — the doctrine's own answer to the R-181 class, where every test
// asserted a mechanism inside one function and the tree still moved.
func TestR354_ScenarioD_LiveRecoveryUnitIsNeverWritten(t *testing.T) {
	m, _, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
	_, liveNs, err := m.offboxRestoreScratchDir("immich")
	if err != nil {
		t.Fatal(err)
	}
	liveUnit := RecoveryUnitPath(liveNs, "immich")
	liveVols := filepath.Join(liveUnit, "volume-dumps")
	if err := os.MkdirAll(liveVols, 0o755); err != nil {
		t.Fatal(err)
	}
	// The live unit's own copy — the local restore path's source. It must survive untouched.
	if err := os.WriteFile(filepath.Join(liveVols, "immich_immich_data.tar"), []byte("THE LIVE UNIT COPY"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(liveUnit, "manifest.json"), []byte(`{"app_name":"immich"}`), 0o644); err != nil {
		t.Fatal(err)
	}
	// Fingerprint the WHOLE live unit. Narrowing this to the volume archives made the test blind to a
	// placement writing into the unit ROOT — caught by red-proof 6, which passed against the narrowed
	// version. The excluded set is exactly the pre-restore undo copies, and those are asserted below.
	before := fingerprintTree(t, liveUnit)

	seedScratchVolumes(t, m, "immich", "immich_immich_data.tar")
	m.volumeReplayFrom = func(_, _ string) (int, error) { return 1, nil }
	// A copier that really writes, so "the unit is never written to" is observable rather than assumed.
	m.SetOffboxFullPlaceCopier(func(src, dst string) (int, error) {
		_ = os.MkdirAll(dst, 0o755)
		return 1, os.WriteFile(filepath.Join(dst, "PLACED"), []byte("from the copier"), 0o644)
	})

	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
		t.Fatalf("ReconstituteFromOffsite: %v", err)
	}

	if after := fingerprintTree(t, liveUnit); after != before {
		t.Errorf("the LIVE recovery unit's backup content changed across the restore — the local path's source was clobbered\nbefore:\n%s\nafter:\n%s", before, after)
	}

	// The ONE write into the live unit that IS expected: the pre-restore safety dump. It lives in the
	// app's own db-dumps dir deliberately (see preRestoreDumpPrefix) — it is the undo, and an undo the
	// customer cannot see is not much of one. Asserted here rather than merely excluded, so "the unit
	// is untouched" cannot quietly come to mean "the undo stopped being written".
	undo, _ := filepath.Glob(filepath.Join(liveUnit, "db-dumps", "pre-restore-*.sql"))
	if len(undo) != 1 {
		t.Errorf("expected exactly one pre-restore undo copy in the live unit, found %d", len(undo))
	}
}

// TestR354_ScenarioE_PartialReplayIsAFailure — a volume replay that fails must never read as a
// completion, and must name what failed (to the operator).
//
// UPDATED 2026-10-08 (R-893 option C, `09` §3 decision 192): one volume HAS been replaced, so a start
// would run the app on a mix of the snapshot's volume and the live rest. The app is now HELD stopped
// for support instead of brought up; the volume that did not come back is named in the operator's
// hold notice (the household reads the plain hold sentence).
func TestR354_ScenarioE_PartialReplayIsReportedAsFailure(t *testing.T) {
	m, prov, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
	seedScratchVolumes(t, m, "immich", "immich_a.tar", "immich_b.tar")
	m.volumeReplayFrom = func(_, _ string) (int, error) {
		return 1, fmt.Errorf("failed to restore 1 volume(s): [immich_b]")
	}
	var noticed error
	m.SetRestoreHoldNotify(func(_ string, replayErr, _ error) { noticed = replayErr })

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("a partial volume replay must be reported as a failure, not a completion")
	}
	if noticed == nil || !strings.Contains(noticed.Error(), "immich_b") {
		t.Errorf("the operator's hold notice must name the volume that did not come back; got %v", noticed)
	}
	if prov.fullStarted {
		t.Error("the app was STARTED on a partly replaced set of volumes — it must be held (R-893)")
	}
	if held, _ := m.RestoreHoldFor("immich"); !held {
		t.Error("no restore hold was left after a partial volume replay")
	}
	// The count of what DID come back is still carried, so the report can say "1 of 2".
	if res.VolumesReplayed != 1 {
		t.Errorf("VolumesReplayed = %d, want the partial count 1", res.VolumesReplayed)
	}
}