controller: F7 atomic volume dumps + F6 no-single-copy + F5 stale-primary sweep (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
2026-07-12 09:00:16 +02:00
parent b389e73fff
commit 68b3a3932e
13 changed files with 569 additions and 57 deletions
@@ -13,6 +13,7 @@ import (

 	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
 )

 // Manager orchestrates app-data backups: database dumps and Docker-volume tars.
@@ -44,6 +45,16 @@ type Manager struct {
 	// disconnected) can be unit-tested without Docker. Nil → the real DumpAppVolumesSafe.
 	dumpVolumesSafe func(stackName string) error

+	// F7 tar seam — the ONE docker exec inside DumpAppVolumes, overridable so the atomic-write
+	// behaviour (tmp+fsync+rename; the last good `.tar` survives a mid-write failure) is unit-testable
+	// without Docker. It must write the tar to `<dumpDir>/<volName>.tar.tmp` and return the combined
+	// output + error. Nil → the real `docker run … alpine tar cf …tar.tmp`.
+	tarVolume func(volName, dumpDir string) ([]byte, error)
+
+	// F6 per-app tier-2 seam — overridable so RunAllTier2's app SELECTION (now incl. volume-only apps)
+	// is unit-testable without rsync/du. Nil → the real RunTier2.
+	perAppTier2 func(stackName string) error
+
 	// generateSecret (O4), if set, produces a replacement value for a RESETTABLE secret that could
 	// not be recovered during restore-from-unit (wired to stacks.Manager.GenerateSecretForField in
 	// main.go). Nil / ok=false → the secret stays absent and the restore proceeds with a loud WARN.
@@ -95,8 +106,21 @@ type FullBackupStatus struct {
 	// Flash messages (set by handlers, passed through redirect)
 	FlashSuccess string
 	FlashError   string
+
+	// SingleCopyWarning (F6, CAMPAIGN-3) is a non-empty honest Hungarian notice when the box has NO
+	// off-drive target at all — tier-1 is then the ONLY local copy and 3-2-1 needs a 2nd drive or
+	// offsite. Empty when an off-drive (tier-2) target exists. Never a fake 3-2-1 guarantee.
+	SingleCopyWarning string
 }

+// systemDriveLabel is the human label for the internal SSD / system drive (F6 — a sys_drive app's
+// backup used to render with a blank drive label). Matches the tier-2 UI wording.
+const systemDriveLabel = "Belső SSD (rendszer)"
+
+// singleCopyNotice is the honest single-drive signal (F6): shown when no off-drive tier-2 target
+// exists, instead of silently implying a 3-2-1 guarantee the box cannot provide.
+const singleCopyNotice = "Csak egy másolat készül (nincs második meghajtó) — a 3-2-1 mentéshez csatlakoztasson egy második meghajtót vagy offsite tárolót."
+
 // DBDumpStatus holds the last DB dump result.
 type DBDumpStatus struct {
 	LastRun  time.Time
@@ -336,6 +360,11 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
 	// Phase 2: refresh each deployed app's self-contained recovery unit (compose + manifest).
 	m.captureAllRecoveryUnits()

+	// F5 (CAMPAIGN-3): after the units are fresh on the CURRENT drives, prune any orphaned
+	// backups/primary/<app> dir an app left on an OLD drive when its HDD_PATH moved — pure disk
+	// residue, invisible in the snapshot list. Guarded (deployed + different current drive only).
+	m.pruneStalePrimaryDirs()
+
 	// No silent partials: a DB-dump or volume-dump failure fails the whole run.
 	if !allOK {
 		return fmt.Errorf("some backup steps failed: %s", strings.Join(failedSummaryLines(summary), "; "))
@@ -436,22 +465,35 @@ func (m *Manager) DumpAppVolumes(stackName string) error {
 	var dumpErrors []string
 	for _, volName := range volumes {
 		tarPath := filepath.Join(dumpDir, volName+".tar")
+		// F7 (CAMPAIGN-3, HIGH): write the tar to a `.tar.tmp` sibling and only atomically rename it
+		// over the restore point on success — the same crash-safe pattern the DB-dump path uses
+		// (appbackup/dbdump.go DumpOne). Before this, tar wrote the `.tar` IN PLACE, so a mid-write NFS
+		// cut left a 0-byte tar REPLACING the last good dump (tier-1 restore is replace-semantics → an
+		// empty volume). Now a failed/interrupted write only ever touches the `.tmp`; the last good
+		// `.tar` is untouched. The `.tmp` name (ends `.tmp`, not `.tar`) is invisible to the
+		// restore-point/stale scans, so it is never mistaken for a restore point.
+		tmpPath := tarPath + ".tmp"
 		if m.isDebug() {
 			m.logger.Printf("[DEBUG] [backup] Dumping volume %s for %s", volName, stackName)
 		}

-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-		cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
-			"-v", volName+":/vol:ro",
-			"-v", dumpDir+":/out",
-			"alpine", "tar", "cf", "/out/"+volName+".tar", "-C", "/vol", ".")
-		out, err := cmd.CombinedOutput()
-		cancel()
+		out, err := m.tarVolumeOrDefault(volName, dumpDir)

 		if err != nil {
-			m.logger.Printf("[WARN] [backup] Volume dump failed for %s/%s: %s — %v",
+			// Any tar error or context timeout (incl. a dead NFS target → EIO): remove ONLY the tmp,
+			// leave the existing `.tar` restore point byte-untouched, WARN, continue.
+			m.logger.Printf("[WARN] [backup] Volume dump failed for %s/%s (last good dump preserved): %s — %v",
 				stackName, volName, strings.TrimSpace(string(out)), err)
-			os.Remove(tarPath)
+			os.Remove(tmpPath)
+			dumpErrors = append(dumpErrors, volName)
+			continue
+		}
+
+		// fsync the tmp file (flush the tar to disk) then atomically rename over the restore point.
+		if err := atomicPromoteTar(tmpPath, tarPath); err != nil {
+			m.logger.Printf("[WARN] [backup] Volume dump promote failed for %s/%s (last good dump preserved): %v",
+				stackName, volName, err)
+			os.Remove(tmpPath)
 			dumpErrors = append(dumpErrors, volName)
 			continue
 		}
@@ -461,17 +503,24 @@ func (m *Manager) DumpAppVolumes(stackName string) error {
 		}
 	}

-	// Clean up tars for volumes that no longer exist
+	// Clean up tars (and any orphan `.tar.tmp` from a killed run) for volumes that no longer exist.
 	entries, _ := os.ReadDir(dumpDir)
 	activeVols := make(map[string]bool)
 	for _, v := range volumes {
 		activeVols[v+".tar"] = true
 	}
 	for _, e := range entries {
-		if !activeVols[e.Name()] && strings.HasSuffix(e.Name(), ".tar") {
-			os.Remove(filepath.Join(dumpDir, e.Name()))
+		name := e.Name()
+		// A leftover `.tar.tmp` is never a restore point — always safe to remove (its `.tar` sibling,
+		// if any, is the real restore point and is handled by the `.tar` branch).
+		if strings.HasSuffix(name, ".tar.tmp") {
+			os.Remove(filepath.Join(dumpDir, name))
+			continue
+		}
+		if !activeVols[name] && strings.HasSuffix(name, ".tar") {
+			os.Remove(filepath.Join(dumpDir, name))
 			if m.isDebug() {
-				m.logger.Printf("[DEBUG] [backup] Removed stale volume dump: %s/%s", stackName, e.Name())
+				m.logger.Printf("[DEBUG] [backup] Removed stale volume dump: %s/%s", stackName, name)
 			}
 		}
 	}
@@ -482,6 +531,51 @@ func (m *Manager) DumpAppVolumes(stackName string) error {
 	return nil
 }

+// tarVolumeOrDefault runs the F7 tar seam (m.tarVolume) or, when unset, the real docker tar into the
+// `<volName>.tar.tmp` sibling under dumpDir. The 10-minute bound matches the original.
+func (m *Manager) tarVolumeOrDefault(volName, dumpDir string) ([]byte, error) {
+	if m.tarVolume != nil {
+		return m.tarVolume(volName, dumpDir)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+		"-v", volName+":/vol:ro",
+		"-v", dumpDir+":/out",
+		"alpine", "tar", "cf", "/out/"+volName+".tar.tmp", "-C", "/vol", ".")
+	return cmd.CombinedOutput()
+}
+
+// atomicPromoteTar fsyncs a completed `.tar.tmp` then atomically renames it over the final `.tar`
+// (same-dir rename = atomic on the target fs). It mirrors DumpOne's crash-safety (F7) and EXCEEDS it
+// by also best-effort fsync'ing the directory entry, so the rename itself survives a power loss — the
+// DB-dump path fsyncs the file but not the dir; a follow-up could add the dir fsync there too. On any
+// error the tmp is left for the caller to remove; the final `.tar` is never touched here except by a
+// successful rename.
+func atomicPromoteTar(tmpPath, finalPath string) error {
+	f, err := os.Open(tmpPath)
+	if err != nil {
+		return fmt.Errorf("opening tmp dump: %w", err)
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return fmt.Errorf("syncing tmp dump: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("closing tmp dump: %w", err)
+	}
+	if err := os.Rename(tmpPath, finalPath); err != nil {
+		return fmt.Errorf("renaming tmp dump: %w", err)
+	}
+	// Best-effort: fsync the directory so the rename is durable (ignore errors — the rename already
+	// made the new content visible; this only hardens against a power loss immediately after).
+	if dir, derr := os.Open(filepath.Dir(finalPath)); derr == nil {
+		_ = dir.Sync()
+		_ = dir.Close()
+	}
+	return nil
+}
+
 // DumpAppVolumesSafe stops the stack before dumping volumes and restarts after.
 // Prevents inconsistent tars of live database volumes (e.g. PostgreSQL).
 // Protected stacks that reject StopStack will return an error — callers handle as warning.
@@ -749,6 +843,7 @@ func (m *Manager) GetFullStatus(nextDBDump time.Time) *FullBackupStatus {
 		// Update dynamic fields that don't need subprocess calls
 		status.Running = m.running
 		status.NextDBDump = nextDBDump
+		status.SingleCopyWarning = m.singleCopyWarning() // F6: honest single-drive signal
 		// Deep-copy lastDBDump so callers cannot mutate shared state.
 		if m.lastDBDump != nil {
 			copyDump := *m.lastDBDump
@@ -786,10 +881,11 @@ func (m *Manager) GetFullStatus(nextDBDump time.Time) *FullBackupStatus {

 	// No cache yet — return a minimal status (first page load before cache is populated)
 	status := &FullBackupStatus{
-		Enabled:        m.cfg.Backup.Enabled,
-		Running:        m.running,
-		DBDumpSchedule: m.cfg.Backup.DBDumpSchedule,
-		NextDBDump:     nextDBDump,
+		Enabled:           m.cfg.Backup.Enabled,
+		Running:           m.running,
+		DBDumpSchedule:    m.cfg.Backup.DBDumpSchedule,
+		NextDBDump:        nextDBDump,
+		SingleCopyWarning: m.singleCopyWarning(), // F6
 	}
 	if m.lastDBDump != nil {
 		copyDump := *m.lastDBDump
@@ -802,6 +898,114 @@ func (m *Manager) GetFullStatus(nextDBDump time.Time) *FullBackupStatus {
 	return status
 }

+// hasOffDriveTarget reports whether any registered, schedulable storage path lives on a physical disk
+// OTHER than the system drive — i.e. whether a genuine off-drive (tier-2) copy is possible at all.
+// When false the box is single-drive: tier-1 is the ONLY local copy and 3-2-1 needs a 2nd drive or
+// offsite (F6 — surfaced honestly via SingleCopyWarning, never a faked guarantee).
+func (m *Manager) hasOffDriveTarget() bool {
+	if m.settings == nil || m.systemDataPath == "" {
+		return false
+	}
+	for _, sp := range m.settings.GetSchedulableStoragePaths() {
+		if sp.Path == m.systemDataPath || system.SamePhysicalDevice(m.systemDataPath, sp.Path) {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+// singleCopyWarning returns the honest single-drive notice, or "" when an off-drive target exists.
+func (m *Manager) singleCopyWarning() string {
+	if m.hasOffDriveTarget() {
+		return ""
+	}
+	return singleCopyNotice
+}
+
+// sysDriveLabelFor returns the drive label for a stack's tier-1 restore point — the clear
+// system-drive label for a sys_drive (volume-only) app (F6: never blank), else the enrolled drive's
+// storage label.
+func (m *Manager) sysDriveLabelFor(stackName string) string {
+	drive := m.GetAppDrivePath(stackName)
+	if drive == "" {
+		return ""
+	}
+	if drive == m.systemDataPath {
+		return systemDriveLabel
+	}
+	if m.settings != nil {
+		return m.settings.GetStorageLabel(drive)
+	}
+	return ""
+}
+
+// pruneStalePrimaryDirs removes orphaned `backups/primary/<app>` dirs left on a drive after an app's
+// HDD_PATH moved to another drive (F5, CAMPAIGN-3 — pure disk residue, invisible in the snapshot
+// list). LOAD-BEARING GUARDS: a dir is removed ONLY when <app> is currently deployed AND its current
+// namespace root differs from this dir's drive. It NEVER removes the dir on the app's CURRENT drive
+// (that IS the live restore point), and NEVER removes a dir for an app NOT in the deployed set (an
+// undeployed app's last backup is still its restore point — orphaned-app cleanup is a separate,
+// user-driven concern). Only operates strictly under a `backups/primary/` prefix.
+func (m *Manager) pruneStalePrimaryDirs() {
+	if m.stackProvider == nil {
+		return
+	}
+	// Current namespace root per DEPLOYED app.
+	current := map[string]string{}
+	for _, s := range m.stackProvider.ListDeployedStacks() {
+		if drive := m.GetAppDrivePath(s.Name); drive != "" {
+			current[s.Name] = filepath.Clean(m.namespaceRoot(drive))
+		}
+	}
+	// Candidate drives to scan: the system drive + every registered storage path.
+	var nsRoots []string
+	if m.systemDataPath != "" {
+		nsRoots = append(nsRoots, filepath.Clean(m.namespaceRoot(m.systemDataPath)))
+	}
+	if m.settings != nil {
+		for _, sp := range m.settings.GetStoragePaths() {
+			nsRoots = append(nsRoots, filepath.Clean(NamespaceRoot(sp.Path, true)))
+		}
+	}
+	seen := map[string]bool{}
+	for _, nsRoot := range nsRoots {
+		if seen[nsRoot] {
+			continue
+		}
+		seen[nsRoot] = true
+		primaryDir := PrimaryBackupPath(nsRoot)
+		entries, err := os.ReadDir(primaryDir)
+		if err != nil {
+			continue // absent/unreadable (e.g. a disconnected drive) — nothing to prune here
+		}
+		for _, e := range entries {
+			if !e.IsDir() {
+				continue
+			}
+			app := e.Name()
+			cur, deployed := current[app]
+			if !deployed {
+				continue // GUARD: an undeployed app's last backup is still its restore point
+			}
+			if cur == nsRoot {
+				continue // GUARD: this IS the app's current drive — the live restore point
+			}
+			stalePath := RecoveryUnitPath(nsRoot, app)
+			// Prefix safety: only ever remove strictly inside `backups/primary/` (no surprise user data).
+			if !strings.HasPrefix(filepath.Clean(stalePath)+string(filepath.Separator),
+				filepath.Clean(primaryDir)+string(filepath.Separator)) {
+				continue
+			}
+			if err := os.RemoveAll(stalePath); err != nil {
+				m.logger.Printf("[WARN] [backup] F5: could not remove stale primary dir for %s on old drive: %v", app, err)
+			} else {
+				m.logger.Printf("[INFO] [backup] F5: removed stale primary backup dir for %s on an old drive (app now on %s)", app, cur)
+			}
+		}
+	}
+}
+
 // isDebug returns true if logging level is "debug".
 func (m *Manager) isDebug() bool {
 	return m.cfg != nil && m.cfg.Logging.Level == "debug"