felhom-controller/controller/internal/backup/offbox_reconstitute_test.go

package backup

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// R-43/R-44 (v0.148.0) — the coherent-pair + true-restore tests.
//
// These exist because the product shipped a restore button for months that could not restore.
// DIAG-immich-restore-2026-07-19: 11 photos, files intact, timeline empty, two "successful"
// restores that merged 0 files and never touched postgres. Every test below asserts a behaviour
// whose absence produced that outcome, so each one is a regression guard for a real incident
// rather than a description of the current implementation.

// recordingProvider records stop/start call ORDER so the reconstitution sequence can be asserted.
//
// R-47 widened it: it now also records the DB-ONLY bring-up and, critically, whether a FULL start
// has happened yet — the state the replay must observe as `false`. That single flag is what
// separates the fixed sequence from the one that produced H4, in which the whole stack was already
// up (and rebuilding its own schema) when the dump replay began.
type recordingProvider struct {
	offbox3aProvider
	calls       []string
	composePath string   // the LIVE compose the DB-service resolver reads
	gotServices []string // services passed to StartStackServices
	fullStarted bool     // a FULL StartStack has happened
	startSvcErr error    // injected StartStackServices failure
}

func (p *recordingProvider) StopStack(string) error { p.calls = append(p.calls, "stop"); return nil }
func (p *recordingProvider) StartStack(string) error {
	p.fullStarted = true
	p.calls = append(p.calls, "start")
	return nil
}
func (p *recordingProvider) StartStackServices(_ string, services []string) error {
	p.gotServices = append([]string(nil), services...)
	p.calls = append(p.calls, "startsvc:"+strings.Join(services, ","))
	return p.startSvcErr
}
func (p *recordingProvider) GetStackComposePath(string) (string, bool) {
	return p.composePath, p.composePath != ""
}

// The app really is up again after StartStack, so the post-restore health wait returns at once.
// Leaving it false would make each test sit through the full 90s deadline.
func (p *recordingProvider) RefreshAndIsRunning(string) bool { return true }

// recoveryProvider adds the recovery info CaptureRecoveryUnit needs (the shared 3a provider has none).
type recoveryProvider struct {
	offbox3aProvider
	stackDir string
}

func (p *recoveryProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) {
	return RecoveryInfo{DisplayName: "Immich", StackDir: p.stackDir}, name == "immich"
}

// pgDump builds a structurally valid postgres dump big enough to clear ValidateDump's 100-byte
// floor, with the accounts-table COPY block carrying `rows` rows. The R-44 sniff runs only on a
// dump that already passes structural validation, so a toy fixture would silently skip it.
func pgDump(rows int) string {
	const head = `-- PostgreSQL database dump
-- Dumped from database version 16.10
SET statement_timeout = 0;
SET lock_timeout = 0;
SET client_encoding = 'UTF8';
CREATE TABLE public.asset (id uuid NOT NULL);
CREATE TABLE public."user" (id uuid NOT NULL, email text);
COPY public."user" (id, email) FROM stdin;
`
	var b strings.Builder
	b.WriteString(head)
	for i := 0; i < rows; i++ {
		b.WriteString("id-x\tuser@example.invalid\n")
	}
	b.WriteString("\\.\n") // the COPY-block terminator
	return b.String()
}

// reconFixture builds a manager with a COMPLETED full scratch for `immich`, a snapshot whose unit
// carries the given coherence stamp, and injectable copy/dump/import seams.
func reconFixture(t *testing.T, runID, dumpsAt string, dumpBody string) (*Manager, *recordingProvider, *[]string) {
	t.Helper()
	drive := t.TempDir()
	m, sett := newOffboxManager(t)
	prov := &recordingProvider{offbox3aProvider: offbox3aProvider{
		hdd: map[string]string{"immich": drive}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{},
		// R-356: the app is DEPLOYED, stated as its own fact. It used to be implied by owning a
		// drive; the restore now asks the two questions separately.
		deployed: map[string]bool{"immich": true},
	}}
	m.SetStackProvider(prov)
	if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "USB", Schedulable: true}); err != nil {
		t.Fatal(err)
	}

	// The LIVE compose the reconstitution reads to learn WHICH service holds the database (R-47).
	// Immich-shaped on purpose: an app service, a redis service that must never be mistaken for a
	// database, and a top-level `volumes:` key whose entry looks exactly like a service to a line scan.
	liveStackDir := t.TempDir()
	prov.composePath = filepath.Join(liveStackDir, "docker-compose.yml")
	if err := os.WriteFile(prov.composePath, []byte(immichLikeCompose), 0o644); err != nil {
		t.Fatal(err)
	}

	scratch, liveNs, err := m.offboxRestoreScratchDir("immich")
	if err != nil {
		t.Fatal(err)
	}
	oldNs := "/felhomdata/ns"
	unitP := oldNs + "/backups/primary/immich"
	dataP := oldNs + "/appdata/immich"
	placements, err := mapOffsiteRestorePaths([]string{unitP, dataP}, "immich", scratch, liveNs)
	if err != nil {
		t.Fatal(err)
	}
	for _, pl := range placements {
		if err := os.MkdirAll(pl.src, 0o755); err != nil {
			t.Fatal(err)
		}
		if pl.isUnit {
			dd := filepath.Join(pl.src, "db-dumps")
			if err := os.MkdirAll(dd, 0o755); err != nil {
				t.Fatal(err)
			}
			if dumpBody != "" {
				if err := os.WriteFile(filepath.Join(dd, "immich-postgres.sql"), []byte(dumpBody), 0o644); err != nil {
					t.Fatal(err)
				}
			}
			man := &RecoveryManifest{SchemaVersion: 1, AppName: "immich", OffsiteRunID: runID, DumpsAt: dumpsAt}
			if err := writeManifest(filepath.Join(pl.src, "manifest.json"), man); err != nil {
				t.Fatal(err)
			}
		}
	}

	m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
	m.SetOffboxSizer(func(string) int64 { return 1 << 20 })
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		if contains(args, "snapshots") {
			return []byte(`[{"short_id":"snap1","time":"2026-07-19T06:00:00Z","paths":["` + unitP + `","` + dataP + `"]}]`), nil
		}
		return nil, nil
	})

	// Seams: one DB, a safety dump that really writes a file, and a recording importer.
	db := DiscoveredDB{StackName: "immich", ContainerName: "immich-postgres", DBType: DBTypePostgres}
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return []DiscoveredDB{db}, nil }
	// R-361: the seam now takes the FINAL PATH, not a directory — a directory argument is what let
	// the callee choose the canonical name, which was the defect.
	m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, finalPath string) DumpResult {
		_ = os.MkdirAll(filepath.Dir(finalPath), 0o755)
		_ = os.WriteFile(finalPath, []byte(pgDump(1)), 0o644)
		return DumpResult{DB: d, FilePath: finalPath, Size: 42}
	})
	var imported []string
	m.importDBDump = func(_ context.Context, _ DiscoveredDB, p string) error {
		imported = append(imported, p)
		return nil
	}
	m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 3, nil })
	return m, prov, &imported
}

// TestReconstituteReplaysDBAndOrdersOperations is Scenario C: the whole point of R-43. A restore of
// a DB-indexed app must stop the app, place files, restart it and REPLAY the snapshot's dump — and
// the safety dump must exist before any of it. Before v0.148.0 the replay simply did not happen,
// which is why the photos never came back.
func TestReconstituteReplaysDBAndOrdersOperations(t *testing.T) {
	m, prov, imported := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("reconstitute: %v", err)
	}
	if res.DBsReplayed != 1 {
		t.Fatalf("expected the snapshot dump to be replayed exactly once, got %d — this is the R-43 defect", res.DBsReplayed)
	}
	if len(*imported) != 1 || !strings.Contains((*imported)[0], "immich-postgres.sql") {
		t.Fatalf("expected an import of the snapshot dump, got %v", *imported)
	}
	// The dump replayed must come from the SCRATCH unit, never the live one: the live unit is
	// deliberately not overwritten, so replaying from it would replay the CURRENT database back over
	// itself and restore nothing.
	if !strings.Contains((*imported)[0], "offsite-restore") {
		t.Fatalf("replay source must be the restored scratch unit, got %s", (*imported)[0])
	}
	if res.FilesPlaced != 3 {
		t.Fatalf("expected the userdata placement to be counted, got %d", res.FilesPlaced)
	}
	// stop BEFORE the file copy; then ONLY the database service up for the replay (R-47 — a full
	// start here is the H4 race); the full start comes last.
	if got := strings.Join(prov.calls, ","); got != "stop,startsvc:immich-postgres,start" {
		t.Fatalf("expected stop → db-only start → full start around the restore, got %q", got)
	}
	if res.SafetyDump == "" {
		t.Fatal("no safety dump recorded — the undo must exist")
	}
	if _, err := os.Stat(res.SafetyDump); err != nil {
		t.Fatalf("safety dump not on disk: %v", err)
	}
	if !strings.HasPrefix(filepath.Base(res.SafetyDump), preRestoreDumpPrefix) {
		t.Fatalf("safety dump must carry the pre-restore prefix so it is never replayed as a source, got %s", filepath.Base(res.SafetyDump))
	}
}

// TestReconstituteRefusesWhenSafetyDumpFails is the RED-PROOF for the undo invariant: a replay whose
// previous state was not captured is an overwrite with no way back, so it must not happen at all —
// and it must abort with the live app untouched (no stop, no copy).
func TestReconstituteRefusesWhenSafetyDumpFails(t *testing.T) {
	m, prov, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
	m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult {
		return DumpResult{DB: d, Error: context.DeadlineExceeded}
	})
	var copied bool
	m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { copied = true; return 1, nil })

	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("expected a refusal when the safety dump cannot be taken")
	}
	if len(*imported) != 0 {
		t.Fatalf("REPLAYED WITHOUT AN UNDO — the exact thing the invariant forbids: %v", *imported)
	}
	if copied {
		t.Fatal("files were overwritten despite the refusal — the abort must leave live data untouched")
	}
	if len(prov.calls) != 0 {
		t.Fatalf("the app was stopped despite the refusal, got %v", prov.calls)
	}
}

// TestReconstituteNoDBAppMakesNoDumpOrImportCalls is Scenario E: an app without a database must flow
// exactly as before — no safety dump, no replay — so the new leg cannot regress the simple case.
func TestReconstituteNoDBAppMakesNoDumpOrImportCalls(t *testing.T) {
	m, _, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "")
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
	dumped := 0
	m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult {
		dumped++
		return DumpResult{DB: d}
	})

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("reconstitute: %v", err)
	}
	if dumped != 0 {
		t.Fatalf("a no-DB app must not produce a safety dump, got %d call(s)", dumped)
	}
	if len(*imported) != 0 {
		t.Fatalf("a no-DB app must not import anything, got %v", *imported)
	}
	if res.SafetyDump != "" || res.DBsReplayed != 0 {
		t.Fatalf("unexpected DB activity: safety=%q replayed=%d", res.SafetyDump, res.DBsReplayed)
	}
}

// TestReconstituteSurfacesLegacySkewedPair is Scenario D: a pre-v0.148 snapshot carries no coherence
// stamp, so its two halves may be from different times. That must be SURFACED (and reversible), never
// blocked — the customer's own judgement is the gate, and refusing would deny a legitimate restore.
func TestReconstituteSurfacesLegacySkewedPair(t *testing.T) {
	m, _, imported := reconFixture(t, "", "", pgDump(1))

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("a legacy pair must still be restorable, got refusal: %v", err)
	}
	if !res.Skewed {
		t.Fatal("an unstamped (pre-v0.148) snapshot must report Skewed so the confirm can say so")
	}
	if len(*imported) != 1 {
		t.Fatalf("the legacy restore must still replay, got %v", *imported)
	}
}

// TestReconstituteFlagsCustomerEmptyDump is the R-44 sniff at the restore end: the immich dump that
// started all of this was structurally valid and contained zero users. Restoring it is allowed, but
// the customer must be told before they commit.
func TestReconstituteFlagsCustomerEmptyDump(t *testing.T) {
	// A valid postgres dump whose accounts table has NO rows — the 2026-07-19 shape exactly.
	m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(0))

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("the sniff must never block a restore: %v", err)
	}
	if !res.LooksEmpty {
		t.Fatal("a dump with an empty accounts table must raise the warn-level signal")
	}
}

// TestOffsiteScratchPairReportsWhatTheConfirmNeeds covers the page-render surface: the confirm can
// only be honest if this reports the pair's age and warnings before anything is started.
func TestOffsiteScratchPairReportsWhatTheConfirmNeeds(t *testing.T) {
	m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z",
		"-- PostgreSQL database dump\nCREATE TABLE a();\nCOPY public.\"user\" (id) FROM stdin;\n7\n\\.\n")

	info := m.OffsiteScratchPair("immich")
	if !info.Ready || !info.HasDump {
		t.Fatalf("expected a ready pair with a dump, got %+v", info)
	}
	if info.Skewed {
		t.Fatal("a stamped snapshot must not be reported as skewed")
	}
	if info.LooksEmpty {
		t.Fatal("a dump with account rows must not be flagged empty")
	}
	want, _ := time.Parse(time.RFC3339, "2026-07-19T06:00:00Z")
	if !info.DumpsAt.Equal(want) {
		t.Fatalf("DumpsAt = %v, want %v", info.DumpsAt, want)
	}
}

// --- R-44: the coherence pre-phase -----------------------------------------------------------

// TestOffsiteRunDumpsBeforeCapture is Scenarios A + B. The ORDER is the entire mechanism: dumps
// must be refreshed BEFORE restic captures, so the snapshot pairs this run's database with this
// run's files. Reversed, the snapshot would hold rows pointing at files that were never captured.
//
// It also asserts the ordering on the NIGHTLY entry point (RunOffboxBackup, no progress sink), not
// just the manual one — before v0.148.0 the nightly ordering was an accident of two independent
// scheduler entries at 02:30 and 04:15, which a schedule edit could silently invert.
func TestOffsiteRunDumpsBeforeCapture(t *testing.T) {
	drive := t.TempDir()
	m, sett, prov := classifiedOffboxManager(t, drive)
	mkUnit(t, drive, "immich")
	if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
		t.Fatal(err)
	}
	prov.hdd["immich"] = drive
	prov.has["immich"] = true
	prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
	_ = sett.SetAppOffbox("immich", true)

	var order []string
	m.SetOffsitePreDumpFn(func(context.Context) error {
		order = append(order, "dump")
		return nil
	})
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case contains(args, "cat") && contains(args, "config"):
			return []byte(`{"version":2}`), nil
		case contains(args, "backup"):
			order = append(order, "capture")
			return nil, nil
		case contains(args, "snapshots"):
			return []byte(`[]`), nil
		case contains(args, "stats"):
			return []byte(`{"total_size":123}`), nil
		}
		return nil, nil
	})

	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("run: %v", err)
	}
	if len(order) < 2 {
		t.Fatalf("expected both a dump and a capture, got %v", order)
	}
	if order[0] != "dump" {
		t.Fatalf("the dump leg MUST precede the capture (R-44); got %v", order)
	}
	if order[1] != "capture" {
		t.Fatalf("expected the capture immediately after the dump, got %v", order)
	}
}

// TestOffsiteRunContinuesWhenDumpLegFails is the data-first rule: a dump failure degrades the
// snapshot's DB half but must NOT abort the push. Refusing to ship the files would turn a partial
// backup into no backup at all — strictly worse for the customer.
func TestOffsiteRunContinuesWhenDumpLegFails(t *testing.T) {
	drive := t.TempDir()
	m, sett, prov := classifiedOffboxManager(t, drive)
	mkUnit(t, drive, "immich")
	if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
		t.Fatal(err)
	}
	prov.hdd["immich"] = drive
	prov.has["immich"] = true
	prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
	_ = sett.SetAppOffbox("immich", true)

	m.SetOffsitePreDumpFn(func(context.Context) error { return context.DeadlineExceeded })
	cap := &backupCapture{}
	m.SetOffboxRunner(cap.runner())

	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("a dump failure must not fail the whole run: %v", err)
	}
	if cap.backups != 1 {
		t.Fatalf("the files must still be pushed after a dump failure, got %d capture(s)", cap.backups)
	}
}

// TestCaptureRecoveryUnitStampsAndCarriesRunID covers the stamp that makes a pair verifiable at
// restore time, and the trap beside it: the PERIODIC refresh must neither invent a coherence claim
// nor erase one a real run established.
func TestCaptureRecoveryUnitStampsAndCarriesRunID(t *testing.T) {
	drive := t.TempDir()
	m, _, base := classifiedOffboxManager(t, drive)
	base.hdd["immich"] = drive
	// CaptureRecoveryUnit needs real recovery info + a compose dir to read; the shared fixture
	// provider returns none, so wrap it rather than widening a struct four other test files use.
	stackDir := filepath.Join(t.TempDir(), "immich")
	if err := os.MkdirAll(stackDir, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(stackDir, "docker-compose.yml"), []byte("services: {}\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	m.SetStackProvider(&recoveryProvider{offbox3aProvider: *base, stackDir: stackDir})

	// 1) A run in flight stamps the manifest.
	end := m.beginOffsiteRunStamp("run-A")
	if err := m.CaptureRecoveryUnit("immich"); err != nil {
		t.Fatalf("capture: %v", err)
	}
	end()
	man := readManifest(RecoveryUnitManifestPath(drive, "immich"))
	if man == nil || man.OffsiteRunID != "run-A" {
		t.Fatalf("expected the in-flight run id to be stamped, got %+v", man)
	}
	if man.DumpsAt == "" {
		t.Fatal("a stamped unit must record when its dumps were taken")
	}

	// 2) A periodic refresh (no run in flight) must CARRY the stamp forward, not blank it — a unit
	//    that silently lost its stamp would be re-reported as a skewed legacy pair at restore time.
	if err := m.CaptureRecoveryUnit("immich"); err != nil {
		t.Fatalf("refresh: %v", err)
	}
	man2 := readManifest(RecoveryUnitManifestPath(drive, "immich"))
	if man2 == nil || man2.OffsiteRunID != "run-A" {
		t.Fatalf("the periodic refresh erased the coherence stamp: %+v", man2)
	}

	// 3) A NEW run re-stamps even though nothing else about the unit changed — the idempotent-skip
	//    must not swallow the one field the restore path reads.
	end2 := m.beginOffsiteRunStamp("run-B")
	if err := m.CaptureRecoveryUnit("immich"); err != nil {
		t.Fatalf("capture 2: %v", err)
	}
	end2()
	man3 := readManifest(RecoveryUnitManifestPath(drive, "immich"))
	if man3 == nil || man3.OffsiteRunID != "run-B" {
		t.Fatalf("a new run must re-stamp the unit, got %+v", man3)
	}
}