Files
felhom-controller/controller/internal/backup/tier2_restore_test.go
T
admin 3603d1fc7f Tier-2 engine rework: class-driven legs, v2 layout, NAS-target exclusion (Task 3b, v0.135.0)
tier2_capture.go: classified apps get TierSecondary per-bind legs (paperless copy shrinks — export
drops); legacy apps keep the byte-identical resolver set. v2 relpath-mirroring layout
(backups/secondary/<stack>/{marker LAST, recovery-unit/, hdd/<rel>/, userdata/<rel>/}); N>1 native
(errTier2MultiDir/tier2AppDataName deleted). Migration=delete-and-rebuild + reconcile; all RemoveAll
via tier2SafeRemove (refuses outside backups/secondary/). SSD=state-only tier. selectTier2Target
never picks network storage (pinned+auto, F-6C-1). Restore reads v2 behind a marker gate.
Part 0: offbox_enlarge_blocked is a persisted one-time Load seed (opt-out sticks), not a getter
append. Part 0.5: offsite restore scratch prefers a local (non-network) path.
Full v2 test suite + all 10 §10 red-proofs verified. Destructive writes bounded to backups/secondary/.
2026-07-15 10:10:20 +02:00

308 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package backup
import (
"bytes"
"errors"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// t2rFakeProvider records the lifecycle calls RestoreTier2Files makes — the refusal tests assert
// the NON-effect (never stopped) and the happy path asserts the stop→copy→start order.
type t2rFakeProvider struct {
hdd string
mounts []string // F-S2: configurable compose-derived HDD mounts (drives appdata dir-name resolution)
stopped []string
started []string
order []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start"
}
func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false }
func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary { return nil }
func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return f.mounts }
func (f *t2rFakeProvider) GetStackHDDPath(string) string { return f.hdd }
func (f *t2rFakeProvider) GetDockerVolumes(string) []string { return nil }
func (f *t2rFakeProvider) StopStack(name string) error {
f.stopped = append(f.stopped, name)
f.order = append(f.order, "stop")
return nil
}
func (f *t2rFakeProvider) StartStack(name string) error {
f.started = append(f.started, name)
f.order = append(f.order, "start")
return nil
}
func (f *t2rFakeProvider) RefreshAndIsRunning(string) bool { return true }
func (f *t2rFakeProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) {
return RecoveryInfo{}, false
}
func (f *t2rFakeProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) { return nil, false }
func (f *t2rFakeProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
func (f *t2rFakeProvider) RecreateStackFromUnit(string, string, map[string]string) error {
return nil
}
// newT2RManager builds a Manager with a RECORDED Tier-2 copy for "app": live drive + a populated
// <dest>/backups/secondary/app/appdata dir, and a CrossDriveBackup entry pointing at dest.
func newT2RManager(t *testing.T) (m *Manager, fake *t2rFakeProvider, liveDrive, destDrive string) {
t.Helper()
tmp := t.TempDir()
liveDrive = filepath.Join(tmp, "usb")
destDrive = filepath.Join(tmp, "flash")
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
for _, p := range []string{liveDrive, destDrive} {
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
t.Fatal(err)
}
}
if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, Method: "rsync", DestinationPath: destDrive,
LastRun: "2026-07-05T03:30:00Z", LastStatus: "ok",
}); err != nil {
t.Fatal(err)
}
// v2 layout: marker + an hdd/ leg with content.
destBase := filepath.Join(destDrive, "backups", "secondary", "app")
mustWrite(t, filepath.Join(destBase, "hdd", "appdata", "photos", "a.jpg"), "JPEGDATA")
mustWrite(t, filepath.Join(destBase, tier2LayoutMarker), tier2LayoutVersion)
fake = &t2rFakeProvider{hdd: liveDrive}
m = &Manager{
logger: log.New(io.Discard, "", 0),
settings: sett,
systemDataPath: filepath.Join(tmp, "sys"),
stackProvider: fake,
}
return m, fake, liveDrive, destDrive
}
// TestRestoreTier2Files_Orchestration proves the stop → copy → start order, the src/dst the copier
// receives (recorded Tier-2 layout → live AppDataDir), and the returned count.
func TestRestoreTier2Files_Orchestration(t *testing.T) {
m, fake, liveDrive, destDrive := newT2RManager(t)
var gotSrc, gotDst string
m.restoreFilesCopier = func(src, dst string) (int, error) {
gotSrc, gotDst = src, dst
fake.order = append(fake.order, "copy")
return 3, nil
}
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("restore: %v", err)
}
if n != 3 {
t.Errorf("filesRestored = %d, want 3", n)
}
// v2: the hdd/ subtree merges whole into the live namespace root (relpath-mirroring).
if want := filepath.Join(destDrive, "backups", "secondary", "app", "hdd"); gotSrc != want {
t.Errorf("copier src = %q, want the v2 hdd/ subtree %q", gotSrc, want)
}
if gotDst != liveDrive {
t.Errorf("copier dst = %q, want the live namespace root %q", gotDst, liveDrive)
}
if len(fake.order) != 3 || fake.order[0] != "stop" || fake.order[1] != "copy" || fake.order[2] != "start" {
t.Errorf("order = %v, want [stop copy start]", fake.order)
}
}
// TestRestoreTier2Files_Refusals proves every refusal happens BEFORE any side effect: the app is
// never stopped and the copier is never invoked (Scenario C1C5).
func TestRestoreTier2Files_Refusals(t *testing.T) {
assertNoEffect := func(t *testing.T, m *Manager, fake *t2rFakeProvider, copierCalled *bool, wantErr error) {
t.Helper()
n, err := m.RestoreTier2Files("app")
if err == nil {
t.Fatal("expected refusal, got nil")
}
if wantErr != nil && !errors.Is(err, wantErr) {
t.Errorf("err = %v, want %v", err, wantErr)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
if len(fake.stopped) != 0 {
t.Errorf("app was STOPPED on a refusal: %v", fake.stopped)
}
if *copierCalled {
t.Error("copier was invoked on a refusal")
}
}
wire := func(m *Manager) *bool {
called := false
m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil }
return &called
}
t.Run("C1 no Tier-2 record", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C1b LastRun empty", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, DestinationPath: destDrive, // no LastRun — never actually ran
}); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C2 copy dir absent on target", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := os.RemoveAll(filepath.Join(destDrive, "backups")); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C3 Tier-2 drive disconnected", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(destDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errTier2DriveGone)
})
t.Run("C4 live drive disconnected", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(liveDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveGone)
})
t.Run("C4b live drive decommissioned", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDecommissioned(liveDrive, ""); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveDecommed)
})
t.Run("C5 backup already running", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
m.mu.Lock()
m.running = true
m.mu.Unlock()
assertNoEffect(t, m, fake, called, nil) // acquireRunning's own error
m.mu.Lock()
m.running = false
m.mu.Unlock()
})
}
// TestRestoreTier2Files_NothingToRestore proves Scenario D: zero files copied is a SUCCESS
// (0, nil), not an error — the "everything is already in place" answer.
func TestRestoreTier2Files_NothingToRestore(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 0, nil }
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("zero-copy restore must succeed: %v", err)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
// The stop/start cycle still happened (acceptable per spec).
if len(fake.stopped) != 1 || len(fake.started) != 1 {
t.Errorf("stop/start: %v/%v", fake.stopped, fake.started)
}
}
// TestRestoreTier2Files_CopyErrorSurfaces proves no-silent-failure: a copier error is returned
// (flash_error path) and the app is still restarted.
func TestRestoreTier2Files_CopyErrorSurfaces(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 1, errors.New("disk full") }
if _, err := m.RestoreTier2Files("app"); err == nil {
t.Fatal("copier error must surface")
}
if len(fake.started) != 1 {
t.Error("app must still be restarted after a failed copy")
}
}
// TestCountRestoredFiles pins the itemize-changes parsing: only transferred regular files (">f")
// count — created dirs (cd), symlinks (cL) and untransferred lines do not.
func TestCountRestoredFiles(t *testing.T) {
out := ">f+++++++++ photos/a.jpg\n" +
"cd+++++++++ photos/\n" +
"cL+++++++++ link -> target\n" +
">f+++++++++ docs/b.pdf\n" +
".d..t...... ./\n" +
"\n"
if n := countRestoredFiles(out); n != 2 {
t.Errorf("count = %d, want 2", n)
}
if n := countRestoredFiles(""); n != 0 {
t.Errorf("empty output count = %d, want 0", n)
}
}
// TestRsyncRestoreMissing_Semantics is the FS-level proof of the additive-only contract (Scenarios
// A + B together), run against the REAL rsync (skipped where rsync is unavailable — it runs on the
// Linux build server and in the live validation):
// - a file missing live is restored byte-identical (and counted);
// - a live file with DIFFERENT content than the backup keeps its LIVE bytes;
// - a live-only file survives.
//
// §10 COMPANION: swapping the helper's flags for rsyncMirror's (-a --delete, no --ignore-existing)
// makes this fail TWICE — b.txt gets clobbered AND c.txt gets deleted.
func TestRsyncRestoreMissing_Semantics(t *testing.T) {
if _, err := exec.LookPath("rsync"); err != nil {
t.Skip("rsync not available on this machine — runs on the Linux build server")
}
tmp := t.TempDir()
src := filepath.Join(tmp, "tier2copy")
dst := filepath.Join(tmp, "live")
mustWrite(t, filepath.Join(src, "photos", "a.jpg"), "JPEG-FROM-BACKUP")
mustWrite(t, filepath.Join(src, "b.txt"), "BACKUP-VERSION")
mustWrite(t, filepath.Join(dst, "b.txt"), "LIVE-EDIT-AFTER-COPY")
mustWrite(t, filepath.Join(dst, "c.txt"), "LIVE-ONLY-NEW-FILE")
n, err := rsyncRestoreMissing(src, dst)
if err != nil {
t.Fatalf("rsyncRestoreMissing: %v", err)
}
if n != 1 {
t.Errorf("filesRestored = %d, want 1 (only the missing a.jpg)", n)
}
// Scenario A: the deleted file is back, byte-identical.
got, err := os.ReadFile(filepath.Join(dst, "photos", "a.jpg"))
if err != nil || !bytes.Equal(got, []byte("JPEG-FROM-BACKUP")) {
t.Errorf("a.jpg not restored byte-identical: %q %v", got, err)
}
// Scenario B core #1: the differing live file keeps its LIVE bytes (backup NOT restored).
if got := mustRead(t, filepath.Join(dst, "b.txt")); got != "LIVE-EDIT-AFTER-COPY" {
t.Errorf("live-edited b.txt was CLOBBERED: %q", got)
}
// Scenario B core #2: the live-only file was NOT deleted.
if got := mustRead(t, filepath.Join(dst, "c.txt")); got != "LIVE-ONLY-NEW-FILE" {
t.Errorf("live-only c.txt was harmed: %q", got)
}
}