feat(backup): C2 Part 1 — RestoreTier2Files: in-place, additive-only class-C file restore engine

Closes the engine half of drill finding F2: user files under appdata/<stack> had no customer recovery path (operator copy-back only). - RestoreTier2Files(stack): single-flight with backup/restore; ALL refusals before any stop (no Tier-2 record / LastRun empty / copy dir absent → "nincs másodlagos fájlmásolat"; Tier-2 drive disconnected; live drive disconnected/decommissioned — Hungarian, flash-ready); source is the RECORDED CrossDriveBackup.DestinationPath (never a fresh selectTier2Target); stop → copy → start → waitForHealthy; copy/restart errors surface (F17). - rsyncRestoreMissing: rsyncMirror's exec shape with the OPPOSITE-direction flags: -a --ignore-existing --itemize-changes — existing live files are never overwritten, nothing is ever deleted (the --delete trap this task exists to avoid). Count = ">f" itemize lines (pure countRestoredFiles). - restoreFilesCopier seam so orchestration tests never shell out; the one FS-level test of the real rsync is LookPath-guarded (runs on the Linux build server + live validation). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-05 13:17:40 +02:00
parent 0313ecda51
commit 30b11100e0
3 changed files with 452 additions and 0 deletions
@@ -0,0 +1,301 @@
+package backup
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// t2rFakeProvider records the lifecycle calls RestoreTier2Files makes — the refusal tests assert
+// the NON-effect (never stopped) and the happy path asserts the stop→copy→start order.
+type t2rFakeProvider struct {
+	hdd     string
+	stopped []string
+	started []string
+	order   []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start"
+}
+
+func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false }
+func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary        { return nil }
+func (f *t2rFakeProvider) GetStackHDDMounts(string) []string         { return nil }
+func (f *t2rFakeProvider) GetStackHDDPath(string) string             { return f.hdd }
+func (f *t2rFakeProvider) GetDockerVolumes(string) []string          { return nil }
+func (f *t2rFakeProvider) StopStack(name string) error {
+	f.stopped = append(f.stopped, name)
+	f.order = append(f.order, "stop")
+	return nil
+}
+func (f *t2rFakeProvider) StartStack(name string) error {
+	f.started = append(f.started, name)
+	f.order = append(f.order, "start")
+	return nil
+}
+func (f *t2rFakeProvider) RefreshAndIsRunning(string) bool { return true }
+func (f *t2rFakeProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) {
+	return RecoveryInfo{}, false
+}
+func (f *t2rFakeProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
+func (f *t2rFakeProvider) RecreateStackFromUnit(string, string, map[string]string) error {
+	return nil
+}
+
+// newT2RManager builds a Manager with a RECORDED Tier-2 copy for "app": live drive + a populated
+// <dest>/backups/secondary/app/appdata dir, and a CrossDriveBackup entry pointing at dest.
+func newT2RManager(t *testing.T) (m *Manager, fake *t2rFakeProvider, liveDrive, destDrive string) {
+	t.Helper()
+	tmp := t.TempDir()
+	liveDrive = filepath.Join(tmp, "usb")
+	destDrive = filepath.Join(tmp, "flash")
+	sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range []string{liveDrive, destDrive} {
+		if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
+		Enabled: true, Method: "rsync", DestinationPath: destDrive,
+		LastRun: "2026-07-05T03:30:00Z", LastStatus: "ok",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "appdata", "photos", "a.jpg"), "JPEGDATA")
+
+	fake = &t2rFakeProvider{hdd: liveDrive}
+	m = &Manager{
+		logger:         log.New(io.Discard, "", 0),
+		settings:       sett,
+		systemDataPath: filepath.Join(tmp, "sys"),
+		stackProvider:  fake,
+	}
+	return m, fake, liveDrive, destDrive
+}
+
+// TestRestoreTier2Files_Orchestration proves the stop → copy → start order, the src/dst the copier
+// receives (recorded Tier-2 layout → live AppDataDir), and the returned count.
+func TestRestoreTier2Files_Orchestration(t *testing.T) {
+	m, fake, liveDrive, destDrive := newT2RManager(t)
+	var gotSrc, gotDst string
+	m.restoreFilesCopier = func(src, dst string) (int, error) {
+		gotSrc, gotDst = src, dst
+		fake.order = append(fake.order, "copy")
+		return 3, nil
+	}
+
+	n, err := m.RestoreTier2Files("app")
+	if err != nil {
+		t.Fatalf("restore: %v", err)
+	}
+	if n != 3 {
+		t.Errorf("filesRestored = %d, want 3", n)
+	}
+	if want := filepath.Join(destDrive, "backups", "secondary", "app", "appdata"); gotSrc != want {
+		t.Errorf("copier src = %q, want the RECORDED Tier-2 copy %q", gotSrc, want)
+	}
+	if want := AppDataDir(liveDrive, "app"); gotDst != want {
+		t.Errorf("copier dst = %q, want live appdata %q", gotDst, want)
+	}
+	if len(fake.order) != 3 || fake.order[0] != "stop" || fake.order[1] != "copy" || fake.order[2] != "start" {
+		t.Errorf("order = %v, want [stop copy start]", fake.order)
+	}
+}
+
+// TestRestoreTier2Files_Refusals proves every refusal happens BEFORE any side effect: the app is
+// never stopped and the copier is never invoked (Scenario C1–C5).
+func TestRestoreTier2Files_Refusals(t *testing.T) {
+	assertNoEffect := func(t *testing.T, m *Manager, fake *t2rFakeProvider, copierCalled *bool, wantErr error) {
+		t.Helper()
+		n, err := m.RestoreTier2Files("app")
+		if err == nil {
+			t.Fatal("expected refusal, got nil")
+		}
+		if wantErr != nil && !errors.Is(err, wantErr) {
+			t.Errorf("err = %v, want %v", err, wantErr)
+		}
+		if n != 0 {
+			t.Errorf("filesRestored = %d, want 0", n)
+		}
+		if len(fake.stopped) != 0 {
+			t.Errorf("app was STOPPED on a refusal: %v", fake.stopped)
+		}
+		if *copierCalled {
+			t.Error("copier was invoked on a refusal")
+		}
+	}
+	wire := func(m *Manager) *bool {
+		called := false
+		m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil }
+		return &called
+	}
+
+	t.Run("C1 no Tier-2 record", func(t *testing.T) {
+		m, fake, _, _ := newT2RManager(t)
+		called := wire(m)
+		if err := m.settings.SetCrossDriveConfig("app", nil); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errNoTier2Copy)
+	})
+
+	t.Run("C1b LastRun empty", func(t *testing.T) {
+		m, fake, _, destDrive := newT2RManager(t)
+		called := wire(m)
+		if err := m.settings.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
+			Enabled: true, DestinationPath: destDrive, // no LastRun — never actually ran
+		}); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errNoTier2Copy)
+	})
+
+	t.Run("C2 copy dir absent on target", func(t *testing.T) {
+		m, fake, _, destDrive := newT2RManager(t)
+		called := wire(m)
+		if err := os.RemoveAll(filepath.Join(destDrive, "backups")); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errNoTier2Copy)
+	})
+
+	t.Run("C3 Tier-2 drive disconnected", func(t *testing.T) {
+		m, fake, _, destDrive := newT2RManager(t)
+		called := wire(m)
+		if err := m.settings.SetDisconnected(destDrive, true, nil); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errTier2DriveGone)
+	})
+
+	t.Run("C4 live drive disconnected", func(t *testing.T) {
+		m, fake, liveDrive, _ := newT2RManager(t)
+		called := wire(m)
+		if err := m.settings.SetDisconnected(liveDrive, true, nil); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errLiveDriveGone)
+	})
+
+	t.Run("C4b live drive decommissioned", func(t *testing.T) {
+		m, fake, liveDrive, _ := newT2RManager(t)
+		called := wire(m)
+		if err := m.settings.SetDecommissioned(liveDrive, ""); err != nil {
+			t.Fatal(err)
+		}
+		assertNoEffect(t, m, fake, called, errLiveDriveDecommed)
+	})
+
+	t.Run("C5 backup already running", func(t *testing.T) {
+		m, fake, _, _ := newT2RManager(t)
+		called := wire(m)
+		m.mu.Lock()
+		m.running = true
+		m.mu.Unlock()
+		assertNoEffect(t, m, fake, called, nil) // acquireRunning's own error
+		m.mu.Lock()
+		m.running = false
+		m.mu.Unlock()
+	})
+}
+
+// TestRestoreTier2Files_NothingToRestore proves Scenario D: zero files copied is a SUCCESS
+// (0, nil), not an error — the "everything is already in place" answer.
+func TestRestoreTier2Files_NothingToRestore(t *testing.T) {
+	m, fake, _, _ := newT2RManager(t)
+	m.restoreFilesCopier = func(string, string) (int, error) { return 0, nil }
+
+	n, err := m.RestoreTier2Files("app")
+	if err != nil {
+		t.Fatalf("zero-copy restore must succeed: %v", err)
+	}
+	if n != 0 {
+		t.Errorf("filesRestored = %d, want 0", n)
+	}
+	// The stop/start cycle still happened (acceptable per spec).
+	if len(fake.stopped) != 1 || len(fake.started) != 1 {
+		t.Errorf("stop/start: %v/%v", fake.stopped, fake.started)
+	}
+}
+
+// TestRestoreTier2Files_CopyErrorSurfaces proves no-silent-failure: a copier error is returned
+// (flash_error path) and the app is still restarted.
+func TestRestoreTier2Files_CopyErrorSurfaces(t *testing.T) {
+	m, fake, _, _ := newT2RManager(t)
+	m.restoreFilesCopier = func(string, string) (int, error) { return 1, errors.New("disk full") }
+
+	if _, err := m.RestoreTier2Files("app"); err == nil {
+		t.Fatal("copier error must surface")
+	}
+	if len(fake.started) != 1 {
+		t.Error("app must still be restarted after a failed copy")
+	}
+}
+
+// TestCountRestoredFiles pins the itemize-changes parsing: only transferred regular files (">f")
+// count — created dirs (cd), symlinks (cL) and untransferred lines do not.
+func TestCountRestoredFiles(t *testing.T) {
+	out := ">f+++++++++ photos/a.jpg\n" +
+		"cd+++++++++ photos/\n" +
+		"cL+++++++++ link -> target\n" +
+		">f+++++++++ docs/b.pdf\n" +
+		".d..t...... ./\n" +
+		"\n"
+	if n := countRestoredFiles(out); n != 2 {
+		t.Errorf("count = %d, want 2", n)
+	}
+	if n := countRestoredFiles(""); n != 0 {
+		t.Errorf("empty output count = %d, want 0", n)
+	}
+}
+
+// TestRsyncRestoreMissing_Semantics is the FS-level proof of the additive-only contract (Scenarios
+// A + B together), run against the REAL rsync (skipped where rsync is unavailable — it runs on the
+// Linux build server and in the live validation):
+//   - a file missing live is restored byte-identical (and counted);
+//   - a live file with DIFFERENT content than the backup keeps its LIVE bytes;
+//   - a live-only file survives.
+//
+// §10 COMPANION: swapping the helper's flags for rsyncMirror's (-a --delete, no --ignore-existing)
+// makes this fail TWICE — b.txt gets clobbered AND c.txt gets deleted.
+func TestRsyncRestoreMissing_Semantics(t *testing.T) {
+	if _, err := exec.LookPath("rsync"); err != nil {
+		t.Skip("rsync not available on this machine — runs on the Linux build server")
+	}
+	tmp := t.TempDir()
+	src := filepath.Join(tmp, "tier2copy")
+	dst := filepath.Join(tmp, "live")
+
+	mustWrite(t, filepath.Join(src, "photos", "a.jpg"), "JPEG-FROM-BACKUP")
+	mustWrite(t, filepath.Join(src, "b.txt"), "BACKUP-VERSION")
+	mustWrite(t, filepath.Join(dst, "b.txt"), "LIVE-EDIT-AFTER-COPY")
+	mustWrite(t, filepath.Join(dst, "c.txt"), "LIVE-ONLY-NEW-FILE")
+
+	n, err := rsyncRestoreMissing(src, dst)
+	if err != nil {
+		t.Fatalf("rsyncRestoreMissing: %v", err)
+	}
+	if n != 1 {
+		t.Errorf("filesRestored = %d, want 1 (only the missing a.jpg)", n)
+	}
+	// Scenario A: the deleted file is back, byte-identical.
+	got, err := os.ReadFile(filepath.Join(dst, "photos", "a.jpg"))
+	if err != nil || !bytes.Equal(got, []byte("JPEG-FROM-BACKUP")) {
+		t.Errorf("a.jpg not restored byte-identical: %q %v", got, err)
+	}
+	// Scenario B core #1: the differing live file keeps its LIVE bytes (backup NOT restored).
+	if got := mustRead(t, filepath.Join(dst, "b.txt")); got != "LIVE-EDIT-AFTER-COPY" {
+		t.Errorf("live-edited b.txt was CLOBBERED: %q", got)
+	}
+	// Scenario B core #2: the live-only file was NOT deleted.
+	if got := mustRead(t, filepath.Join(dst, "c.txt")); got != "LIVE-ONLY-NEW-FILE" {
+		t.Errorf("live-only c.txt was harmed: %q", got)
+	}
+}