From 30b11100e04bee61c754f4728104bde8a60a7c68 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 5 Jul 2026 13:17:40 +0200 Subject: [PATCH] =?UTF-8?q?feat(backup):=20C2=20Part=201=20=E2=80=94=20Res?= =?UTF-8?q?toreTier2Files:=20in-place,=20additive-only=20class-C=20file=20?= =?UTF-8?q?restore=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the engine half of drill finding F2: user files under appdata/ had no customer recovery path (operator copy-back only). - RestoreTier2Files(stack): single-flight with backup/restore; ALL refusals before any stop (no Tier-2 record / LastRun empty / copy dir absent → "nincs másodlagos fájlmásolat"; Tier-2 drive disconnected; live drive disconnected/decommissioned — Hungarian, flash-ready); source is the RECORDED CrossDriveBackup.DestinationPath (never a fresh selectTier2Target); stop → copy → start → waitForHealthy; copy/restart errors surface (F17). - rsyncRestoreMissing: rsyncMirror's exec shape with the OPPOSITE-direction flags: -a --ignore-existing --itemize-changes — existing live files are never overwritten, nothing is ever deleted (the --delete trap this task exists to avoid). Count = ">f" itemize lines (pure countRestoredFiles). - restoreFilesCopier seam so orchestration tests never shell out; the one FS-level test of the real rsync is LookPath-guarded (runs on the Linux build server + live validation). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- controller/internal/backup/backup.go | 4 + controller/internal/backup/tier2_restore.go | 147 +++++++++ .../internal/backup/tier2_restore_test.go | 301 ++++++++++++++++++ 3 files changed, 452 insertions(+) create mode 100644 controller/internal/backup/tier2_restore.go create mode 100644 controller/internal/backup/tier2_restore_test.go diff --git a/controller/internal/backup/backup.go b/controller/internal/backup/backup.go index 59945e0..ed8edb0 100644 --- a/controller/internal/backup/backup.go +++ b/controller/internal/backup/backup.go @@ -50,6 +50,10 @@ type Manager struct { // NEVER consulted for data-keys — the fail-closed gate refuses those before generation runs. generateSecret func(stackName, envVar string) (string, bool) + // restoreFilesCopier (C2) — the Tier-2 in-place file-restore copy seam, overridable in tests so + // the orchestration never shells out. Nil → the real rsyncRestoreMissing (additive-only). + restoreFilesCopier func(src, dst string) (filesRestored int, err error) + // migrationRunning, if set, reports whether a data migration is in progress. The scheduled // backup paths skip when it returns true (Change 3 — backup ↔ migration mutual exclusion), so a // nightly dump/Tier-2 can't race a migration copy/cleanup on the same drive. diff --git a/controller/internal/backup/tier2_restore.go b/controller/internal/backup/tier2_restore.go new file mode 100644 index 0000000..38dcfbc --- /dev/null +++ b/controller/internal/backup/tier2_restore.go @@ -0,0 +1,147 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Tier-2 in-place file restore (TASK C2, closes drill finding F2): the customer-facing recovery for +// class-C data — HDD bind-mount user files under appdata/. Restores MISSING files from the +// recorded Tier-2 copy back into the live appdata dir, and touches NOTHING else: +// +// - a file that exists live is NEVER overwritten (a customer edit after the last Tier-2 run wins); +// - a live file absent from the backup is NEVER deleted (that is what rsyncMirror's --delete would +// do in this direction — the catastrophic trap this helper exists to avoid); +// - only files present in the copy and missing live are copied back (attrs preserved). +// +// This exactly serves the "I deleted my files" scenario. Corruption / point-in-time rollback stays +// with the offbox restore-to-verify + operator paths — deliberately out of scope. + +// Refusal reasons (customer-readable — they surface verbatim in the flash message). +var ( + errNoTier2Copy = errors.New("nincs másodlagos fájlmásolat ehhez az alkalmazáshoz") + errTier2DriveGone = errors.New("a másodlagos meghajtó nincs csatlakoztatva") + errLiveDriveGone = errors.New("az alkalmazás meghajtója nincs csatlakoztatva") + errLiveDriveDecommed = errors.New("az alkalmazás meghajtója le van szerelve") +) + +// RestoreTier2Files restores the app's MISSING user files in place from its recorded Tier-2 copy +// (additive-only; see the package comment above). Returns how many regular files were copied back. +// +// The source is the RECORDED Tier-2 destination (settings.CrossDriveBackup.DestinationPath) — never +// a fresh selectTier2Target, which could re-pick a different (empty) drive and "restore" nothing. +// All refusals happen BEFORE the app is stopped. Stop-first is the locked consistency policy: the +// app must not be reorganizing its data dir mid-copy. +func (m *Manager) RestoreTier2Files(stackName string) (filesRestored int, err error) { + if m.stackProvider == nil { + return 0, fmt.Errorf("stack provider not configured") + } + if err := m.acquireRunning(); err != nil { + return 0, err // shares the backup/restore single-flight — must not race a running backup + } + defer m.releaseRunning() + + // Live side: the app's drive must be present and in service. + drive := m.GetAppDrivePath(stackName) + if drive == "" || !filepath.IsAbs(drive) { + return 0, fmt.Errorf("cannot determine drive path for %s", stackName) + } + if m.settings != nil { + if m.settings.IsDisconnected(drive) { + return 0, fmt.Errorf("%w (%s)", errLiveDriveGone, drive) + } + if m.settings.IsDecommissioned(drive) { + return 0, fmt.Errorf("%w (%s)", errLiveDriveDecommed, drive) + } + } + liveDir := AppDataDir(m.namespaceRoot(drive), stackName) + + // Source side: the RECORDED Tier-2 copy must exist and its drive must be connected. + var srcDir string + if m.settings != nil { + if cfg := m.settings.GetCrossDriveConfig(stackName); cfg != nil && cfg.LastRun != "" && cfg.DestinationPath != "" { + if m.settings.IsDisconnected(cfg.DestinationPath) { + return 0, errTier2DriveGone + } + // Same layout literals as RunTier2's destBase + the appdata leg. + srcDir = filepath.Join(cfg.DestinationPath, "backups", "secondary", stackName, "appdata") + } + } + if srcDir == "" { + return 0, errNoTier2Copy + } + if _, statErr := os.Stat(srcDir); statErr != nil { + return 0, errNoTier2Copy // recorded but the copy dir is gone — same honest refusal + } + + m.logger.Printf("[INFO] [backup] Tier-2 file restore for %s: %s → %s (additive-only)", stackName, srcDir, liveDir) + + copier := m.restoreFilesCopier + if copier == nil { + copier = rsyncRestoreMissing + } + + // Stop → copy → start → health (the standard restore shape; F17: errors surface, never swallowed). + if stopErr := m.stackProvider.StopStack(stackName); stopErr != nil { + m.logger.Printf("[WARN] [backup] could not stop %s before Tier-2 file restore: %v (continuing)", stackName, stopErr) + } + start := time.Now() + filesRestored, copyErr := copier(srcDir, liveDir) + startErr := m.stackProvider.StartStack(stackName) + if startErr != nil { + m.logger.Printf("[ERROR] [backup] failed to restart %s after Tier-2 file restore: %v", stackName, startErr) + } + if healthErr := m.waitForHealthy(stackName, 90*time.Second); healthErr != nil { + m.logger.Printf("[WARN] [backup] %s Tier-2 file restore done but health check failed: %v", stackName, healthErr) + } + + if copyErr != nil { + return filesRestored, fmt.Errorf("fájlmásolás sikertelen: %w", copyErr) + } + if startErr != nil { + return filesRestored, fmt.Errorf("%d fájl visszaállítva, de az alkalmazás újraindítása sikertelen: %w", filesRestored, startErr) + } + // Privacy: count + duration only — customer file names never at INFO. + m.logger.Printf("[INFO] [backup] Tier-2 file restore completed for %s: %d file(s) restored (%s)", + stackName, filesRestored, time.Since(start).Round(time.Second)) + return filesRestored, nil +} + +// rsyncRestoreMissing copies the files MISSING from dst back from src, and nothing else: +// `rsync -a --ignore-existing` — existing dst files are never overwritten, and (unlike rsyncMirror, +// which carries --delete for the backup direction) nothing at dst is ever deleted. Returns the +// number of regular files transferred, counted from --itemize-changes output. +func rsyncRestoreMissing(src, dst string) (int, error) { + if err := os.MkdirAll(dst, 0755); err != nil { + return 0, fmt.Errorf("mkdir %s: %w", dst, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + // Trailing slashes: copy the CONTENTS of src into dst (same shape as rsyncMirror). + cmd := exec.CommandContext(ctx, "rsync", "-a", "--ignore-existing", "--itemize-changes", + strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/") + out, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) + } + return countRestoredFiles(string(out)), nil +} + +// countRestoredFiles counts the itemize-changes lines that mark a TRANSFERRED regular file (">f…"). +// Created dirs ("cd…") and symlinks ("cL…") are not counted — the flash reports files. Pure +// (unit-tested without rsync). +func countRestoredFiles(itemizedOut string) int { + n := 0 + for _, line := range strings.Split(itemizedOut, "\n") { + if strings.HasPrefix(line, ">f") { + n++ + } + } + return n +} diff --git a/controller/internal/backup/tier2_restore_test.go b/controller/internal/backup/tier2_restore_test.go new file mode 100644 index 0000000..bf29669 --- /dev/null +++ b/controller/internal/backup/tier2_restore_test.go @@ -0,0 +1,301 @@ +package backup + +import ( + "bytes" + "errors" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// t2rFakeProvider records the lifecycle calls RestoreTier2Files makes — the refusal tests assert +// the NON-effect (never stopped) and the happy path asserts the stop→copy→start order. +type t2rFakeProvider struct { + hdd string + stopped []string + started []string + order []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start" +} + +func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false } +func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary { return nil } +func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return nil } +func (f *t2rFakeProvider) GetStackHDDPath(string) string { return f.hdd } +func (f *t2rFakeProvider) GetDockerVolumes(string) []string { return nil } +func (f *t2rFakeProvider) StopStack(name string) error { + f.stopped = append(f.stopped, name) + f.order = append(f.order, "stop") + return nil +} +func (f *t2rFakeProvider) StartStack(name string) error { + f.started = append(f.started, name) + f.order = append(f.order, "start") + return nil +} +func (f *t2rFakeProvider) RefreshAndIsRunning(string) bool { return true } +func (f *t2rFakeProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) { + return RecoveryInfo{}, false +} +func (f *t2rFakeProvider) RecoverStackSecrets(string, []string) map[string]string { return nil } +func (f *t2rFakeProvider) RecreateStackFromUnit(string, string, map[string]string) error { + return nil +} + +// newT2RManager builds a Manager with a RECORDED Tier-2 copy for "app": live drive + a populated +// /backups/secondary/app/appdata dir, and a CrossDriveBackup entry pointing at dest. +func newT2RManager(t *testing.T) (m *Manager, fake *t2rFakeProvider, liveDrive, destDrive string) { + t.Helper() + tmp := t.TempDir() + liveDrive = filepath.Join(tmp, "usb") + destDrive = filepath.Join(tmp, "flash") + sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + for _, p := range []string{liveDrive, destDrive} { + if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil { + t.Fatal(err) + } + } + if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{ + Enabled: true, Method: "rsync", DestinationPath: destDrive, + LastRun: "2026-07-05T03:30:00Z", LastStatus: "ok", + }); err != nil { + t.Fatal(err) + } + mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "appdata", "photos", "a.jpg"), "JPEGDATA") + + fake = &t2rFakeProvider{hdd: liveDrive} + m = &Manager{ + logger: log.New(io.Discard, "", 0), + settings: sett, + systemDataPath: filepath.Join(tmp, "sys"), + stackProvider: fake, + } + return m, fake, liveDrive, destDrive +} + +// TestRestoreTier2Files_Orchestration proves the stop → copy → start order, the src/dst the copier +// receives (recorded Tier-2 layout → live AppDataDir), and the returned count. +func TestRestoreTier2Files_Orchestration(t *testing.T) { + m, fake, liveDrive, destDrive := newT2RManager(t) + var gotSrc, gotDst string + m.restoreFilesCopier = func(src, dst string) (int, error) { + gotSrc, gotDst = src, dst + fake.order = append(fake.order, "copy") + return 3, nil + } + + n, err := m.RestoreTier2Files("app") + if err != nil { + t.Fatalf("restore: %v", err) + } + if n != 3 { + t.Errorf("filesRestored = %d, want 3", n) + } + if want := filepath.Join(destDrive, "backups", "secondary", "app", "appdata"); gotSrc != want { + t.Errorf("copier src = %q, want the RECORDED Tier-2 copy %q", gotSrc, want) + } + if want := AppDataDir(liveDrive, "app"); gotDst != want { + t.Errorf("copier dst = %q, want live appdata %q", gotDst, want) + } + if len(fake.order) != 3 || fake.order[0] != "stop" || fake.order[1] != "copy" || fake.order[2] != "start" { + t.Errorf("order = %v, want [stop copy start]", fake.order) + } +} + +// TestRestoreTier2Files_Refusals proves every refusal happens BEFORE any side effect: the app is +// never stopped and the copier is never invoked (Scenario C1–C5). +func TestRestoreTier2Files_Refusals(t *testing.T) { + assertNoEffect := func(t *testing.T, m *Manager, fake *t2rFakeProvider, copierCalled *bool, wantErr error) { + t.Helper() + n, err := m.RestoreTier2Files("app") + if err == nil { + t.Fatal("expected refusal, got nil") + } + if wantErr != nil && !errors.Is(err, wantErr) { + t.Errorf("err = %v, want %v", err, wantErr) + } + if n != 0 { + t.Errorf("filesRestored = %d, want 0", n) + } + if len(fake.stopped) != 0 { + t.Errorf("app was STOPPED on a refusal: %v", fake.stopped) + } + if *copierCalled { + t.Error("copier was invoked on a refusal") + } + } + wire := func(m *Manager) *bool { + called := false + m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil } + return &called + } + + t.Run("C1 no Tier-2 record", func(t *testing.T) { + m, fake, _, _ := newT2RManager(t) + called := wire(m) + if err := m.settings.SetCrossDriveConfig("app", nil); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errNoTier2Copy) + }) + + t.Run("C1b LastRun empty", func(t *testing.T) { + m, fake, _, destDrive := newT2RManager(t) + called := wire(m) + if err := m.settings.SetCrossDriveConfig("app", &settings.CrossDriveBackup{ + Enabled: true, DestinationPath: destDrive, // no LastRun — never actually ran + }); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errNoTier2Copy) + }) + + t.Run("C2 copy dir absent on target", func(t *testing.T) { + m, fake, _, destDrive := newT2RManager(t) + called := wire(m) + if err := os.RemoveAll(filepath.Join(destDrive, "backups")); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errNoTier2Copy) + }) + + t.Run("C3 Tier-2 drive disconnected", func(t *testing.T) { + m, fake, _, destDrive := newT2RManager(t) + called := wire(m) + if err := m.settings.SetDisconnected(destDrive, true, nil); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errTier2DriveGone) + }) + + t.Run("C4 live drive disconnected", func(t *testing.T) { + m, fake, liveDrive, _ := newT2RManager(t) + called := wire(m) + if err := m.settings.SetDisconnected(liveDrive, true, nil); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errLiveDriveGone) + }) + + t.Run("C4b live drive decommissioned", func(t *testing.T) { + m, fake, liveDrive, _ := newT2RManager(t) + called := wire(m) + if err := m.settings.SetDecommissioned(liveDrive, ""); err != nil { + t.Fatal(err) + } + assertNoEffect(t, m, fake, called, errLiveDriveDecommed) + }) + + t.Run("C5 backup already running", func(t *testing.T) { + m, fake, _, _ := newT2RManager(t) + called := wire(m) + m.mu.Lock() + m.running = true + m.mu.Unlock() + assertNoEffect(t, m, fake, called, nil) // acquireRunning's own error + m.mu.Lock() + m.running = false + m.mu.Unlock() + }) +} + +// TestRestoreTier2Files_NothingToRestore proves Scenario D: zero files copied is a SUCCESS +// (0, nil), not an error — the "everything is already in place" answer. +func TestRestoreTier2Files_NothingToRestore(t *testing.T) { + m, fake, _, _ := newT2RManager(t) + m.restoreFilesCopier = func(string, string) (int, error) { return 0, nil } + + n, err := m.RestoreTier2Files("app") + if err != nil { + t.Fatalf("zero-copy restore must succeed: %v", err) + } + if n != 0 { + t.Errorf("filesRestored = %d, want 0", n) + } + // The stop/start cycle still happened (acceptable per spec). + if len(fake.stopped) != 1 || len(fake.started) != 1 { + t.Errorf("stop/start: %v/%v", fake.stopped, fake.started) + } +} + +// TestRestoreTier2Files_CopyErrorSurfaces proves no-silent-failure: a copier error is returned +// (flash_error path) and the app is still restarted. +func TestRestoreTier2Files_CopyErrorSurfaces(t *testing.T) { + m, fake, _, _ := newT2RManager(t) + m.restoreFilesCopier = func(string, string) (int, error) { return 1, errors.New("disk full") } + + if _, err := m.RestoreTier2Files("app"); err == nil { + t.Fatal("copier error must surface") + } + if len(fake.started) != 1 { + t.Error("app must still be restarted after a failed copy") + } +} + +// TestCountRestoredFiles pins the itemize-changes parsing: only transferred regular files (">f") +// count — created dirs (cd), symlinks (cL) and untransferred lines do not. +func TestCountRestoredFiles(t *testing.T) { + out := ">f+++++++++ photos/a.jpg\n" + + "cd+++++++++ photos/\n" + + "cL+++++++++ link -> target\n" + + ">f+++++++++ docs/b.pdf\n" + + ".d..t...... ./\n" + + "\n" + if n := countRestoredFiles(out); n != 2 { + t.Errorf("count = %d, want 2", n) + } + if n := countRestoredFiles(""); n != 0 { + t.Errorf("empty output count = %d, want 0", n) + } +} + +// TestRsyncRestoreMissing_Semantics is the FS-level proof of the additive-only contract (Scenarios +// A + B together), run against the REAL rsync (skipped where rsync is unavailable — it runs on the +// Linux build server and in the live validation): +// - a file missing live is restored byte-identical (and counted); +// - a live file with DIFFERENT content than the backup keeps its LIVE bytes; +// - a live-only file survives. +// +// §10 COMPANION: swapping the helper's flags for rsyncMirror's (-a --delete, no --ignore-existing) +// makes this fail TWICE — b.txt gets clobbered AND c.txt gets deleted. +func TestRsyncRestoreMissing_Semantics(t *testing.T) { + if _, err := exec.LookPath("rsync"); err != nil { + t.Skip("rsync not available on this machine — runs on the Linux build server") + } + tmp := t.TempDir() + src := filepath.Join(tmp, "tier2copy") + dst := filepath.Join(tmp, "live") + + mustWrite(t, filepath.Join(src, "photos", "a.jpg"), "JPEG-FROM-BACKUP") + mustWrite(t, filepath.Join(src, "b.txt"), "BACKUP-VERSION") + mustWrite(t, filepath.Join(dst, "b.txt"), "LIVE-EDIT-AFTER-COPY") + mustWrite(t, filepath.Join(dst, "c.txt"), "LIVE-ONLY-NEW-FILE") + + n, err := rsyncRestoreMissing(src, dst) + if err != nil { + t.Fatalf("rsyncRestoreMissing: %v", err) + } + if n != 1 { + t.Errorf("filesRestored = %d, want 1 (only the missing a.jpg)", n) + } + // Scenario A: the deleted file is back, byte-identical. + got, err := os.ReadFile(filepath.Join(dst, "photos", "a.jpg")) + if err != nil || !bytes.Equal(got, []byte("JPEG-FROM-BACKUP")) { + t.Errorf("a.jpg not restored byte-identical: %q %v", got, err) + } + // Scenario B core #1: the differing live file keeps its LIVE bytes (backup NOT restored). + if got := mustRead(t, filepath.Join(dst, "b.txt")); got != "LIVE-EDIT-AFTER-COPY" { + t.Errorf("live-edited b.txt was CLOBBERED: %q", got) + } + // Scenario B core #2: the live-only file was NOT deleted. + if got := mustRead(t, filepath.Join(dst, "c.txt")); got != "LIVE-ONLY-NEW-FILE" { + t.Errorf("live-only c.txt was harmed: %q", got) + } +}