feat(backup): C2 Part 1 — RestoreTier2Files: in-place, additive-only class-C file restore engine

Closes the engine half of drill finding F2: user files under appdata/<stack>
had no customer recovery path (operator copy-back only).

- RestoreTier2Files(stack): single-flight with backup/restore; ALL refusals
  before any stop (no Tier-2 record / LastRun empty / copy dir absent →
  "nincs másodlagos fájlmásolat"; Tier-2 drive disconnected; live drive
  disconnected/decommissioned — Hungarian, flash-ready); source is the
  RECORDED CrossDriveBackup.DestinationPath (never a fresh selectTier2Target);
  stop → copy → start → waitForHealthy; copy/restart errors surface (F17).
- rsyncRestoreMissing: rsyncMirror's exec shape with the OPPOSITE-direction
  flags: -a --ignore-existing --itemize-changes — existing live files are
  never overwritten, nothing is ever deleted (the --delete trap this task
  exists to avoid). Count = ">f" itemize lines (pure countRestoredFiles).
- restoreFilesCopier seam so orchestration tests never shell out; the one
  FS-level test of the real rsync is LookPath-guarded (runs on the Linux
  build server + live validation).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-05 13:17:40 +02:00
parent 0313ecda51
commit 30b11100e0
3 changed files with 452 additions and 0 deletions
+4
View File
@@ -50,6 +50,10 @@ type Manager struct {
// NEVER consulted for data-keys — the fail-closed gate refuses those before generation runs.
generateSecret func(stackName, envVar string) (string, bool)
// restoreFilesCopier (C2) — the Tier-2 in-place file-restore copy seam, overridable in tests so
// the orchestration never shells out. Nil → the real rsyncRestoreMissing (additive-only).
restoreFilesCopier func(src, dst string) (filesRestored int, err error)
// migrationRunning, if set, reports whether a data migration is in progress. The scheduled
// backup paths skip when it returns true (Change 3 — backup ↔ migration mutual exclusion), so a
// nightly dump/Tier-2 can't race a migration copy/cleanup on the same drive.
+147
View File
@@ -0,0 +1,147 @@
package backup
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// Tier-2 in-place file restore (TASK C2, closes drill finding F2): the customer-facing recovery for
// class-C data — HDD bind-mount user files under appdata/<stack>. Restores MISSING files from the
// recorded Tier-2 copy back into the live appdata dir, and touches NOTHING else:
//
// - a file that exists live is NEVER overwritten (a customer edit after the last Tier-2 run wins);
// - a live file absent from the backup is NEVER deleted (that is what rsyncMirror's --delete would
// do in this direction — the catastrophic trap this helper exists to avoid);
// - only files present in the copy and missing live are copied back (attrs preserved).
//
// This exactly serves the "I deleted my files" scenario. Corruption / point-in-time rollback stays
// with the offbox restore-to-verify + operator paths — deliberately out of scope.
// Refusal reasons (customer-readable — they surface verbatim in the flash message).
var (
errNoTier2Copy = errors.New("nincs másodlagos fájlmásolat ehhez az alkalmazáshoz")
errTier2DriveGone = errors.New("a másodlagos meghajtó nincs csatlakoztatva")
errLiveDriveGone = errors.New("az alkalmazás meghajtója nincs csatlakoztatva")
errLiveDriveDecommed = errors.New("az alkalmazás meghajtója le van szerelve")
)
// RestoreTier2Files restores the app's MISSING user files in place from its recorded Tier-2 copy
// (additive-only; see the package comment above). Returns how many regular files were copied back.
//
// The source is the RECORDED Tier-2 destination (settings.CrossDriveBackup.DestinationPath) — never
// a fresh selectTier2Target, which could re-pick a different (empty) drive and "restore" nothing.
// All refusals happen BEFORE the app is stopped. Stop-first is the locked consistency policy: the
// app must not be reorganizing its data dir mid-copy.
func (m *Manager) RestoreTier2Files(stackName string) (filesRestored int, err error) {
if m.stackProvider == nil {
return 0, fmt.Errorf("stack provider not configured")
}
if err := m.acquireRunning(); err != nil {
return 0, err // shares the backup/restore single-flight — must not race a running backup
}
defer m.releaseRunning()
// Live side: the app's drive must be present and in service.
drive := m.GetAppDrivePath(stackName)
if drive == "" || !filepath.IsAbs(drive) {
return 0, fmt.Errorf("cannot determine drive path for %s", stackName)
}
if m.settings != nil {
if m.settings.IsDisconnected(drive) {
return 0, fmt.Errorf("%w (%s)", errLiveDriveGone, drive)
}
if m.settings.IsDecommissioned(drive) {
return 0, fmt.Errorf("%w (%s)", errLiveDriveDecommed, drive)
}
}
liveDir := AppDataDir(m.namespaceRoot(drive), stackName)
// Source side: the RECORDED Tier-2 copy must exist and its drive must be connected.
var srcDir string
if m.settings != nil {
if cfg := m.settings.GetCrossDriveConfig(stackName); cfg != nil && cfg.LastRun != "" && cfg.DestinationPath != "" {
if m.settings.IsDisconnected(cfg.DestinationPath) {
return 0, errTier2DriveGone
}
// Same layout literals as RunTier2's destBase + the appdata leg.
srcDir = filepath.Join(cfg.DestinationPath, "backups", "secondary", stackName, "appdata")
}
}
if srcDir == "" {
return 0, errNoTier2Copy
}
if _, statErr := os.Stat(srcDir); statErr != nil {
return 0, errNoTier2Copy // recorded but the copy dir is gone — same honest refusal
}
m.logger.Printf("[INFO] [backup] Tier-2 file restore for %s: %s → %s (additive-only)", stackName, srcDir, liveDir)
copier := m.restoreFilesCopier
if copier == nil {
copier = rsyncRestoreMissing
}
// Stop → copy → start → health (the standard restore shape; F17: errors surface, never swallowed).
if stopErr := m.stackProvider.StopStack(stackName); stopErr != nil {
m.logger.Printf("[WARN] [backup] could not stop %s before Tier-2 file restore: %v (continuing)", stackName, stopErr)
}
start := time.Now()
filesRestored, copyErr := copier(srcDir, liveDir)
startErr := m.stackProvider.StartStack(stackName)
if startErr != nil {
m.logger.Printf("[ERROR] [backup] failed to restart %s after Tier-2 file restore: %v", stackName, startErr)
}
if healthErr := m.waitForHealthy(stackName, 90*time.Second); healthErr != nil {
m.logger.Printf("[WARN] [backup] %s Tier-2 file restore done but health check failed: %v", stackName, healthErr)
}
if copyErr != nil {
return filesRestored, fmt.Errorf("fájlmásolás sikertelen: %w", copyErr)
}
if startErr != nil {
return filesRestored, fmt.Errorf("%d fájl visszaállítva, de az alkalmazás újraindítása sikertelen: %w", filesRestored, startErr)
}
// Privacy: count + duration only — customer file names never at INFO.
m.logger.Printf("[INFO] [backup] Tier-2 file restore completed for %s: %d file(s) restored (%s)",
stackName, filesRestored, time.Since(start).Round(time.Second))
return filesRestored, nil
}
// rsyncRestoreMissing copies the files MISSING from dst back from src, and nothing else:
// `rsync -a --ignore-existing` — existing dst files are never overwritten, and (unlike rsyncMirror,
// which carries --delete for the backup direction) nothing at dst is ever deleted. Returns the
// number of regular files transferred, counted from --itemize-changes output.
func rsyncRestoreMissing(src, dst string) (int, error) {
if err := os.MkdirAll(dst, 0755); err != nil {
return 0, fmt.Errorf("mkdir %s: %w", dst, err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
defer cancel()
// Trailing slashes: copy the CONTENTS of src into dst (same shape as rsyncMirror).
cmd := exec.CommandContext(ctx, "rsync", "-a", "--ignore-existing", "--itemize-changes",
strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/")
out, err := cmd.CombinedOutput()
if err != nil {
return 0, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
}
return countRestoredFiles(string(out)), nil
}
// countRestoredFiles counts the itemize-changes lines that mark a TRANSFERRED regular file (">f…").
// Created dirs ("cd…") and symlinks ("cL…") are not counted — the flash reports files. Pure
// (unit-tested without rsync).
func countRestoredFiles(itemizedOut string) int {
n := 0
for _, line := range strings.Split(itemizedOut, "\n") {
if strings.HasPrefix(line, ">f") {
n++
}
}
return n
}
@@ -0,0 +1,301 @@
package backup
import (
"bytes"
"errors"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// t2rFakeProvider records the lifecycle calls RestoreTier2Files makes — the refusal tests assert
// the NON-effect (never stopped) and the happy path asserts the stop→copy→start order.
type t2rFakeProvider struct {
hdd string
stopped []string
started []string
order []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start"
}
func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false }
func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary { return nil }
func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return nil }
func (f *t2rFakeProvider) GetStackHDDPath(string) string { return f.hdd }
func (f *t2rFakeProvider) GetDockerVolumes(string) []string { return nil }
func (f *t2rFakeProvider) StopStack(name string) error {
f.stopped = append(f.stopped, name)
f.order = append(f.order, "stop")
return nil
}
func (f *t2rFakeProvider) StartStack(name string) error {
f.started = append(f.started, name)
f.order = append(f.order, "start")
return nil
}
func (f *t2rFakeProvider) RefreshAndIsRunning(string) bool { return true }
func (f *t2rFakeProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) {
return RecoveryInfo{}, false
}
func (f *t2rFakeProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
func (f *t2rFakeProvider) RecreateStackFromUnit(string, string, map[string]string) error {
return nil
}
// newT2RManager builds a Manager with a RECORDED Tier-2 copy for "app": live drive + a populated
// <dest>/backups/secondary/app/appdata dir, and a CrossDriveBackup entry pointing at dest.
func newT2RManager(t *testing.T) (m *Manager, fake *t2rFakeProvider, liveDrive, destDrive string) {
t.Helper()
tmp := t.TempDir()
liveDrive = filepath.Join(tmp, "usb")
destDrive = filepath.Join(tmp, "flash")
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
for _, p := range []string{liveDrive, destDrive} {
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
t.Fatal(err)
}
}
if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, Method: "rsync", DestinationPath: destDrive,
LastRun: "2026-07-05T03:30:00Z", LastStatus: "ok",
}); err != nil {
t.Fatal(err)
}
mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "appdata", "photos", "a.jpg"), "JPEGDATA")
fake = &t2rFakeProvider{hdd: liveDrive}
m = &Manager{
logger: log.New(io.Discard, "", 0),
settings: sett,
systemDataPath: filepath.Join(tmp, "sys"),
stackProvider: fake,
}
return m, fake, liveDrive, destDrive
}
// TestRestoreTier2Files_Orchestration proves the stop → copy → start order, the src/dst the copier
// receives (recorded Tier-2 layout → live AppDataDir), and the returned count.
func TestRestoreTier2Files_Orchestration(t *testing.T) {
m, fake, liveDrive, destDrive := newT2RManager(t)
var gotSrc, gotDst string
m.restoreFilesCopier = func(src, dst string) (int, error) {
gotSrc, gotDst = src, dst
fake.order = append(fake.order, "copy")
return 3, nil
}
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("restore: %v", err)
}
if n != 3 {
t.Errorf("filesRestored = %d, want 3", n)
}
if want := filepath.Join(destDrive, "backups", "secondary", "app", "appdata"); gotSrc != want {
t.Errorf("copier src = %q, want the RECORDED Tier-2 copy %q", gotSrc, want)
}
if want := AppDataDir(liveDrive, "app"); gotDst != want {
t.Errorf("copier dst = %q, want live appdata %q", gotDst, want)
}
if len(fake.order) != 3 || fake.order[0] != "stop" || fake.order[1] != "copy" || fake.order[2] != "start" {
t.Errorf("order = %v, want [stop copy start]", fake.order)
}
}
// TestRestoreTier2Files_Refusals proves every refusal happens BEFORE any side effect: the app is
// never stopped and the copier is never invoked (Scenario C1C5).
func TestRestoreTier2Files_Refusals(t *testing.T) {
assertNoEffect := func(t *testing.T, m *Manager, fake *t2rFakeProvider, copierCalled *bool, wantErr error) {
t.Helper()
n, err := m.RestoreTier2Files("app")
if err == nil {
t.Fatal("expected refusal, got nil")
}
if wantErr != nil && !errors.Is(err, wantErr) {
t.Errorf("err = %v, want %v", err, wantErr)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
if len(fake.stopped) != 0 {
t.Errorf("app was STOPPED on a refusal: %v", fake.stopped)
}
if *copierCalled {
t.Error("copier was invoked on a refusal")
}
}
wire := func(m *Manager) *bool {
called := false
m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil }
return &called
}
t.Run("C1 no Tier-2 record", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C1b LastRun empty", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
Enabled: true, DestinationPath: destDrive, // no LastRun — never actually ran
}); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C2 copy dir absent on target", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := os.RemoveAll(filepath.Join(destDrive, "backups")); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errNoTier2Copy)
})
t.Run("C3 Tier-2 drive disconnected", func(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(destDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errTier2DriveGone)
})
t.Run("C4 live drive disconnected", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDisconnected(liveDrive, true, nil); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveGone)
})
t.Run("C4b live drive decommissioned", func(t *testing.T) {
m, fake, liveDrive, _ := newT2RManager(t)
called := wire(m)
if err := m.settings.SetDecommissioned(liveDrive, ""); err != nil {
t.Fatal(err)
}
assertNoEffect(t, m, fake, called, errLiveDriveDecommed)
})
t.Run("C5 backup already running", func(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
called := wire(m)
m.mu.Lock()
m.running = true
m.mu.Unlock()
assertNoEffect(t, m, fake, called, nil) // acquireRunning's own error
m.mu.Lock()
m.running = false
m.mu.Unlock()
})
}
// TestRestoreTier2Files_NothingToRestore proves Scenario D: zero files copied is a SUCCESS
// (0, nil), not an error — the "everything is already in place" answer.
func TestRestoreTier2Files_NothingToRestore(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 0, nil }
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("zero-copy restore must succeed: %v", err)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
// The stop/start cycle still happened (acceptable per spec).
if len(fake.stopped) != 1 || len(fake.started) != 1 {
t.Errorf("stop/start: %v/%v", fake.stopped, fake.started)
}
}
// TestRestoreTier2Files_CopyErrorSurfaces proves no-silent-failure: a copier error is returned
// (flash_error path) and the app is still restarted.
func TestRestoreTier2Files_CopyErrorSurfaces(t *testing.T) {
m, fake, _, _ := newT2RManager(t)
m.restoreFilesCopier = func(string, string) (int, error) { return 1, errors.New("disk full") }
if _, err := m.RestoreTier2Files("app"); err == nil {
t.Fatal("copier error must surface")
}
if len(fake.started) != 1 {
t.Error("app must still be restarted after a failed copy")
}
}
// TestCountRestoredFiles pins the itemize-changes parsing: only transferred regular files (">f")
// count — created dirs (cd), symlinks (cL) and untransferred lines do not.
func TestCountRestoredFiles(t *testing.T) {
out := ">f+++++++++ photos/a.jpg\n" +
"cd+++++++++ photos/\n" +
"cL+++++++++ link -> target\n" +
">f+++++++++ docs/b.pdf\n" +
".d..t...... ./\n" +
"\n"
if n := countRestoredFiles(out); n != 2 {
t.Errorf("count = %d, want 2", n)
}
if n := countRestoredFiles(""); n != 0 {
t.Errorf("empty output count = %d, want 0", n)
}
}
// TestRsyncRestoreMissing_Semantics is the FS-level proof of the additive-only contract (Scenarios
// A + B together), run against the REAL rsync (skipped where rsync is unavailable — it runs on the
// Linux build server and in the live validation):
// - a file missing live is restored byte-identical (and counted);
// - a live file with DIFFERENT content than the backup keeps its LIVE bytes;
// - a live-only file survives.
//
// §10 COMPANION: swapping the helper's flags for rsyncMirror's (-a --delete, no --ignore-existing)
// makes this fail TWICE — b.txt gets clobbered AND c.txt gets deleted.
func TestRsyncRestoreMissing_Semantics(t *testing.T) {
if _, err := exec.LookPath("rsync"); err != nil {
t.Skip("rsync not available on this machine — runs on the Linux build server")
}
tmp := t.TempDir()
src := filepath.Join(tmp, "tier2copy")
dst := filepath.Join(tmp, "live")
mustWrite(t, filepath.Join(src, "photos", "a.jpg"), "JPEG-FROM-BACKUP")
mustWrite(t, filepath.Join(src, "b.txt"), "BACKUP-VERSION")
mustWrite(t, filepath.Join(dst, "b.txt"), "LIVE-EDIT-AFTER-COPY")
mustWrite(t, filepath.Join(dst, "c.txt"), "LIVE-ONLY-NEW-FILE")
n, err := rsyncRestoreMissing(src, dst)
if err != nil {
t.Fatalf("rsyncRestoreMissing: %v", err)
}
if n != 1 {
t.Errorf("filesRestored = %d, want 1 (only the missing a.jpg)", n)
}
// Scenario A: the deleted file is back, byte-identical.
got, err := os.ReadFile(filepath.Join(dst, "photos", "a.jpg"))
if err != nil || !bytes.Equal(got, []byte("JPEG-FROM-BACKUP")) {
t.Errorf("a.jpg not restored byte-identical: %q %v", got, err)
}
// Scenario B core #1: the differing live file keeps its LIVE bytes (backup NOT restored).
if got := mustRead(t, filepath.Join(dst, "b.txt")); got != "LIVE-EDIT-AFTER-COPY" {
t.Errorf("live-edited b.txt was CLOBBERED: %q", got)
}
// Scenario B core #2: the live-only file was NOT deleted.
if got := mustRead(t, filepath.Join(dst, "c.txt")); got != "LIVE-ONLY-NEW-FILE" {
t.Errorf("live-only c.txt was harmed: %q", got)
}
}