Files
felhom-controller/controller/internal/backup/tier2_restore_test.go
T
admin 78ff991f1c v0.153.0 — R-47: the DB replay no longer races the app, on BOTH restore paths
Closes R-47. No new agent coupling — MinAgent stays 0.90.0.

The replay needs a running DB container, so both restore paths started the
WHOLE stack first, giving the application a window to rebuild the very schema
objects the dump was about to create. Measured live on 2026-07-19 (H4,
DIAG-immich-restore-round2): immich-server rebuilt clip_index two seconds
before the dump's CREATE INDEX, the replay aborted "already exists" under
ON_ERROR_STOP=1, and immich reported schema drift. The data survived only
because pg_dump emits COPY before CREATE INDEX.

Both paths now open a DB-ONLY window: only the stack's database service(s)
come up, the dump is replayed with the app still down, and the full start
runs only after the replay exits 0. Fail-closed: a dump with no identifiable
DB service refuses BEFORE the first mutation. Every exit from the window
still does a best-effort full start, so a failed restore never leaves a box
with a database and no application.

New: appbackup.DBServiceNames (yaml.v3 services-map parse — never a line
scan; immich's top-level volume keys are the decoy) sharing dbTypeForImage
with DiscoverDatabases; stacks.Manager.StartStackServices (refuses an empty
list — argument-less `up -d` is a full start); RedeployFromEnv split into
PersistUnitRedeployConfig + its unchanged tail. StackDataProvider's
RecreateStackFromUnit becomes RecreateStackDefinitionFromUnit — the hidden
`up -d` inside the old name is what carried the defect on the local path.

19 new tests (ordering plus state-at-replay-time, zero-mutation fail-closed
effects, replay-failure bring-up, parser decoys, empty-list refusal); three
companion red-proofs run and reverted. 23/23 packages green.

Not yet live-validated: STOP-1 supervised reconstitute, golden 0.153.0.
2026-07-20 17:01:52 +02:00

309 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package backup
import (
"bytes"
"errors"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// t2rFakeProvider records the lifecycle calls RestoreTier2Files makes — the refusal tests assert
// the NON-effect (never stopped) and the happy path asserts the stop→copy→start order.
type t2rFakeProvider struct {
hdd string
mounts []string // F-S2: configurable compose-derived HDD mounts (drives appdata dir-name resolution)
stopped []string
started []string
order []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start"
}
func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false }
func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary { return nil }
func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return f.mounts }
func (f *t2rFakeProvider) GetStackHDDPath(string) string { return f.hdd }
func (f *t2rFakeProvider) GetDockerVolumes(string) []string { return nil }
func (f *t2rFakeProvider) StopStack(name string) error {
f.stopped = append(f.stopped, name)
f.order = append(f.order, "stop")
return nil
}
func (f *t2rFakeProvider) StartStack(name string) error {
f.started = append(f.started, name)
f.order = append(f.order, "start")
return nil
}
func (f *t2rFakeProvider) RefreshAndIsRunning(string) bool { return true }
func (f *t2rFakeProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) {
return RecoveryInfo{}, false
}
func (f *t2rFakeProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) { return nil, false }
func (f *t2rFakeProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
func (f *t2rFakeProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error {
return nil
}
func (f *t2rFakeProvider) StartStackServices(string, []string) error { return nil }
// newT2RManager builds a Manager with a RECORDED Tier-2 copy for "app": live drive + a populated
// <dest>/backups/secondary/app/appdata dir, and a CrossDriveBackup entry pointing at dest.
func newT2RManager(t *testing.T) (m *Manager, fake *t2rFakeProvider, liveDrive, destDrive string) {
t.Helper()
tmp := t.TempDir()
liveDrive = filepath.Join(tmp, "usb")
destDrive = filepath.Join(tmp, "flash")
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
for _, p := range []string{liveDrive, destDrive} {
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
t.Fatal(err)
}
}
if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, Method: "rsync", DestinationPath: destDrive,
LastRun: "2026-07-05T03:30:00Z", LastStatus: "ok",
}); err != nil {
t.Fatal(err)
}
// v2 layout: marker + an hdd/ leg with content.
destBase := filepath.Join(destDrive, "backups", "secondary", "app")
mustWrite(t, filepath.Join(destBase, "hdd", "appdata", "photos", "a.jpg"), "JPEGDATA")
mustWrite(t, filepath.Join(destBase, tier2LayoutMarker), tier2LayoutVersion)
fake = &t2rFakeProvider{hdd: liveDrive}
m = &Manager{
logger: log.New(io.Discard, "", 0),
settings: sett,
systemDataPath: filepath.Join(tmp, "sys"),
stackProvider: fake,
}
return m, fake, liveDrive, destDrive
}
// TestRestoreTier2Files_Orchestration proves the stop → copy → start order, the src/dst the copier
// receives (recorded Tier-2 layout → live AppDataDir), and the returned count.
func TestRestoreTier2Files_Orchestration(t *testing.T) {
m, fake, liveDrive, destDrive := newT2RManager(t)
var gotSrc, gotDst string
m.restoreFilesCopier = func(src, dst string) (int, error) {
gotSrc, gotDst = src, dst
fake.order = append(fake.order, "copy")
return 3, nil
}
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("restore: %v", err)
}
if n != 3 {
t.Errorf("filesRestored = %d, want 3", n)
}
// v2: the hdd/ subtree merges whole into the live namespace root (relpath-mirroring).
if want := filepath.Join(destDrive, "backups", "secondary", "app", "hdd"); gotSrc != want {
t.Errorf("copier src = %q, want the v2 hdd/ subtree %q", gotSrc, want)
}
if gotDst != liveDrive {
t.Errorf("copier dst = %q, want the live namespace root %q", gotDst, liveDrive)
}
if len(fake.order) != 3 || fake.order[0] != "stop" || fake.order[1] != "copy" || fake.order[2] != "start" {
t.Errorf("order = %v, want [stop copy start]", fake.order)
}
}
// TestRestoreTier2Files_Refusals proves every refusal happens BEFORE any side effect: the app is
// never stopped and the copier is never invoked (Scenario C1C5).
func TestRestoreTier2Files_Refusals(t *testing.T) {
assertNoEffect := func(t *testing.T, m *Manager, fake *t2rFakeProvider, copierCalled *bool, wantErr error) {
t.Helper()
n, err := m.RestoreTier2Files("app")
if err == nil {
t.Fatal("expected refusal, got nil")
}
if wantErr != nil && !errors.Is(err, wantErr) {
t.Errorf("err = %v, want %v", err, wantErr)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
if len(fake.stopped) != 0 {
t.Errorf("app was STOPPED on a refusal: %v", fake.stopped)
}
if *copierCalled {
t.Error("copier was invoked on a refusal")
}
}
wire := func(m *Manager) *bool {
called := false
m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil }
return &called
}
t.Run("C1 no Tier-2 record", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C1b LastRun empty", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, DestinationPath: destDrive, // no LastRun — never actually ran
}); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C2 copy dir absent on target", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := os.RemoveAll(filepath.Join(destDrive, "backups")); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C3 Tier-2 drive disconnected", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(destDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errTier2DriveGone)
})
t.Run("C4 live drive disconnected", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(liveDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveGone)
})
t.Run("C4b live drive decommissioned", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDecommissioned(liveDrive, ""); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveDecommed)
})
t.Run("C5 backup already running", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
m.mu.Lock()
m.running = true
m.mu.Unlock()
assertNoEffect(t, m, fake, called, nil) // acquireRunning's own error
m.mu.Lock()
m.running = false
m.mu.Unlock()
})
}
// TestRestoreTier2Files_NothingToRestore proves Scenario D: zero files copied is a SUCCESS
// (0, nil), not an error — the "everything is already in place" answer.
func TestRestoreTier2Files_NothingToRestore(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 0, nil }
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("zero-copy restore must succeed: %v", err)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
// The stop/start cycle still happened (acceptable per spec).
if len(fake.stopped) != 1 || len(fake.started) != 1 {
t.Errorf("stop/start: %v/%v", fake.stopped, fake.started)
}
}
// TestRestoreTier2Files_CopyErrorSurfaces proves no-silent-failure: a copier error is returned
// (flash_error path) and the app is still restarted.
func TestRestoreTier2Files_CopyErrorSurfaces(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 1, errors.New("disk full") }
if _, err := m.RestoreTier2Files("app"); err == nil {
t.Fatal("copier error must surface")
}
if len(fake.started) != 1 {
t.Error("app must still be restarted after a failed copy")
}
}
// TestCountRestoredFiles pins the itemize-changes parsing: only transferred regular files (">f")
// count — created dirs (cd), symlinks (cL) and untransferred lines do not.
func TestCountRestoredFiles(t *testing.T) {
out := ">f+++++++++ photos/a.jpg\n" +
"cd+++++++++ photos/\n" +
"cL+++++++++ link -> target\n" +
">f+++++++++ docs/b.pdf\n" +
".d..t...... ./\n" +
"\n"
if n := countRestoredFiles(out); n != 2 {
t.Errorf("count = %d, want 2", n)
}
if n := countRestoredFiles(""); n != 0 {
t.Errorf("empty output count = %d, want 0", n)
}
}
// TestRsyncRestoreMissing_Semantics is the FS-level proof of the additive-only contract (Scenarios
// A + B together), run against the REAL rsync (skipped where rsync is unavailable — it runs on the
// Linux build server and in the live validation):
// - a file missing live is restored byte-identical (and counted);
// - a live file with DIFFERENT content than the backup keeps its LIVE bytes;
// - a live-only file survives.
//
// §10 COMPANION: swapping the helper's flags for rsyncMirror's (-a --delete, no --ignore-existing)
// makes this fail TWICE — b.txt gets clobbered AND c.txt gets deleted.
func TestRsyncRestoreMissing_Semantics(t *testing.T) {
if _, err := exec.LookPath("rsync"); err != nil {
t.Skip("rsync not available on this machine — runs on the Linux build server")
}
tmp := t.TempDir()
src := filepath.Join(tmp, "tier2copy")
dst := filepath.Join(tmp, "live")
mustWrite(t, filepath.Join(src, "photos", "a.jpg"), "JPEG-FROM-BACKUP")
mustWrite(t, filepath.Join(src, "b.txt"), "BACKUP-VERSION")
mustWrite(t, filepath.Join(dst, "b.txt"), "LIVE-EDIT-AFTER-COPY")
mustWrite(t, filepath.Join(dst, "c.txt"), "LIVE-ONLY-NEW-FILE")
n, err := rsyncRestoreMissing(src, dst)
if err != nil {
t.Fatalf("rsyncRestoreMissing: %v", err)
}
if n != 1 {
t.Errorf("filesRestored = %d, want 1 (only the missing a.jpg)", n)
}
// Scenario A: the deleted file is back, byte-identical.
got, err := os.ReadFile(filepath.Join(dst, "photos", "a.jpg"))
if err != nil || !bytes.Equal(got, []byte("JPEG-FROM-BACKUP")) {
t.Errorf("a.jpg not restored byte-identical: %q %v", got, err)
}
// Scenario B core #1: the differing live file keeps its LIVE bytes (backup NOT restored).
if got := mustRead(t, filepath.Join(dst, "b.txt")); got != "LIVE-EDIT-AFTER-COPY" {
t.Errorf("live-edited b.txt was CLOBBERED: %q", got)
}
// Scenario B core #2: the live-only file was NOT deleted.
if got := mustRead(t, filepath.Join(dst, "c.txt")); got != "LIVE-ONLY-NEW-FILE" {
t.Errorf("live-only c.txt was harmed: %q", got)
}
}