felhom-controller/controller/internal/backup/backup.go

package backup

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
)

// Manager orchestrates app-data backups: database dumps and Docker-volume tars.
//
// Disk-tier backup (restic, cross-drive, drive-recovery, infra-backup) has been
// moved out of the controller into the host agent (slice 8C). This Manager now
// only owns the app-data domain.
type Manager struct {
	cfg            *config.Config
	logger         *log.Logger
	settings       *settings.Settings
	stackProvider  StackDataProvider
	systemDataPath string // fallback drive for SSD-only apps
	version        string // controller version, stamped into recovery-unit manifests

	// tier2Notify, if set, is called after each Tier 2 copy (success: err==nil) for notifications.
	tier2Notify func(stackName, destLabel string, dur time.Duration, err error)

	// offbox (Part B): the restic-SFTP exec seam (nil → real restic) + the failure→operator-alert hook.
	offboxRunner offboxRunner
	offboxNotify func(dur time.Duration, snapshots int, err error)
	// offboxStreamRunner + offboxProgress: the MANUAL run's live progress (v0.147.0, 4c). The stream
	// seam scans restic's `--json` stdout line-by-line; the state is what the page polls. Both are
	// inert on the nightly path — the sink is installed only for the duration of a manual run.
	offboxStreamRunner offboxStreamRunner
	offboxProgress     offboxProgressState
	// offboxOrphanEvent (v0.142.0), if set, pushes a hub event on offsite-repo continuity transitions
	// ("offbox_repo_orphaned" / "offbox_repo_reset"); renamedTo names the move-aside path (reset only).
	// Wired in main.go to the notifier. Nil-safe.
	offboxOrphanEvent func(eventType, renamedTo string)
	// offboxSSH (v0.142.0) is the raw-ssh exec seam for the orphaned-repo move-aside (restic has no
	// rename); tests inject a fake. Nil → the real ssh invocation (defaultOffboxSSH).
	offboxSSH func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)

	// offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable
	// in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb).
	offboxSizer func(path string) int64
	// offboxEnlargeBlockedNotify (3a), if set, is called ONCE per app that NEWLY enters the
	// quota-blocked (enlargement-refused) state — edge-triggered against the persisted EnlargedBlocked
	// set so a nightly schedule can't re-notify a persistently-blocked app (the hub owns cooldown; the
	// controller must not add a timer). Wired in cmd/controller/main.go.
	offboxEnlargeBlockedNotify func(stack string, estBytes int64, usedGB, quotaGB int)
	// offboxPlaceCopier (3a) — the place-to-live missing-only merge seam (nil → rsyncRestoreMissing,
	// the `-a --ignore-existing` additive copy). Never rsyncMirror (--delete trap).
	offboxPlaceCopier func(src, dst string) (int, error)
	// offboxFullPlaceCopier (R-43, v0.148.0) — the FULL-restore overwrite seam (nil →
	// rsyncRestoreOverwrite: `-a` with NO --ignore-existing and NO --delete). Distinct from
	// offboxPlaceCopier on purpose: the two have opposite semantics for an existing file.
	offboxFullPlaceCopier func(src, dst string) (int, error)
	// safetyDumpFn (R-43) — the pre-restore safety-dump seam (nil → the real DumpOne), so the
	// "never replay without an undo on disk" refusal is unit-testable without Docker.
	safetyDumpFn func(ctx context.Context, db DiscoveredDB, dumpDir string) DumpResult
	// offsitePreDumpFn (R-44) — the offsite dump pre-phase seam (nil → runDBDumpsInternal), so the
	// dumps-strictly-before-capture ordering is observable in a test without Docker or restic.
	offsitePreDumpFn func(ctx context.Context) error
	// offboxFreeFn (3a) — the free-space probe for the restore headroom gate, overridable in tests (the
	// Windows `go test` host has no `df`). Nil → the real diskFreeBytes (df --output=avail).
	offboxFreeFn func(path string) int64

	// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
	// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
	discoverDBs  func(ctx context.Context) ([]DiscoveredDB, error)
	importDBDump func(ctx context.Context, db DiscoveredDB, dumpPath string) error

	// F3 volume-dump seam — overridable in tests so runVolumeDumps' gating (protected / volume-less /
	// disconnected) can be unit-tested without Docker. Nil → the real DumpAppVolumesSafe.
	dumpVolumesSafe func(stackName string) error

	// F7 tar seam — the ONE docker exec inside DumpAppVolumes, overridable so the atomic-write
	// behaviour (tmp+fsync+rename; the last good `.tar` survives a mid-write failure) is unit-testable
	// without Docker. It must write the tar to `<dumpDir>/<volName>.tar.tmp` and return the combined
	// output + error. Nil → the real `docker run … alpine tar cf …tar.tmp`.
	tarVolume func(volName, dumpDir string) ([]byte, error)

	// F6 per-app tier-2 seam — overridable so RunAllTier2's app SELECTION (now incl. volume-only apps)
	// is unit-testable without rsync/du. Nil → the real RunTier2.
	perAppTier2 func(stackName string) error

	// generateSecret (O4), if set, produces a replacement value for a RESETTABLE secret that could
	// not be recovered during restore-from-unit (wired to stacks.Manager.GenerateSecretForField in
	// main.go). Nil / ok=false → the secret stays absent and the restore proceeds with a loud WARN.
	// NEVER consulted for data-keys — the fail-closed gate refuses those before generation runs.
	generateSecret func(stackName, envVar string) (string, bool)

	// restoreFilesCopier (C2) — the Tier-2 in-place file-restore copy seam, overridable in tests so
	// the orchestration never shells out. Nil → the real rsyncRestoreMissing (additive-only).
	restoreFilesCopier func(src, dst string) (filesRestored int, err error)

	// tier2Mirror (F-S2) — the Tier-2 backup mirror seam (both rsync legs in RunTier2), overridable
	// so the resolve→mirror→record flow is unit-testable without rsync. Nil → the real rsyncMirror
	// (`-a --delete`, contents-of-src semantics).
	tier2Mirror func(src, dst string) error

	// sharesPassdbCapture (R-7b) — the samba passdb capture seam (a `docker exec … tar cf -`),
	// overridable so the shares payload builder is unit-testable without docker. Nil → the real
	// defaultSharesPassdbCapture. Best-effort by contract: an error yields a manifest-only payload.
	sharesPassdbCapture func() ([]byte, error)

	// sharesPassdbRestore (R-7b) — the mirror seam for putting a captured passdb archive BACK into the
	// samba named volume (`docker exec -i … tar xf -`). Nil → the real defaultSharesPassdbRestore.
	sharesPassdbRestore func(tar []byte) error

	// sharesReconcile (R-7b), if set, re-renders and applies the samba stack after a shares restore
	// re-adds definitions to the registry (wired in main.go to stacks.Manager.ReconcileSamba). It is a
	// SEAM rather than a direct call because the backup package must not depend on the stacks package.
	// Nil → the registry is updated and a WARN says smb.conf will catch up on the next health tick.
	sharesReconcile func() error

	// tier2SSDFits (3b) — the SSD-headroom predicate seam, overridable in tests (system.GetDiskUsage is
	// Linux-only → nil on the Windows test host, which would always refuse the SSD branch). Nil → the
	// real tier2FitsSystemDrive.
	tier2SSDFits func(sys string, sizeBytes int64) bool

	// samePhysicalDevice — the off-drive identity predicate behind every Tier-2 "is this really a
	// SECOND disk?" guard, overridable in tests. The real check is `st_dev` equality, so on a host
	// where every `t.TempDir()` lands on one filesystem the fixture's "two drives" are indistinguishable
	// and Tier-2 correctly refuses them — which makes the off-drive tests unrunnable rather than wrong.
	// Nil → the real system.SamePhysicalDevice (production always takes this path).
	samePhysicalDevice func(a, b string) bool

	// migrationRunning, if set, reports whether a data migration is in progress. The scheduled
	// backup paths skip when it returns true (Change 3 — backup ↔ migration mutual exclusion), so a
	// nightly dump/Tier-2 can't race a migration copy/cleanup on the same drive.
	migrationRunning func() bool

	mu         sync.Mutex
	lastDBDump *DBDumpStatus
	running    bool

	// R-43/R-44 (v0.148.0) — the coherence stamp of the offsite run in flight, read by
	// CaptureRecoveryUnit so each unit records WHICH run took the dumps sitting beside its files.
	// Set for the duration of the dump pre-phase + capture, cleared after; "" means "no offsite run
	// is establishing coherence right now" (the periodic refresh and the local 02:30 dump leg).
	offsiteRunID     string
	offsiteRunDumpAt string

	// Restore op-status (Part B, opstatus.go) — display-only async-restore progress, under `mu`.
	opRunning   bool
	opName      string
	opStack     string
	opStartedAt time.Time
	opLast      *RestoreOpResult

	// Cached status for page rendering (refreshed periodically)
	cachedStatus *FullBackupStatus
	cacheTime    time.Time
}

// FullBackupStatus contains everything the backup page needs.
type FullBackupStatus struct {
	Enabled bool
	Running bool

	// DB Dumps
	LastDBDump    *DBDumpStatus
	DumpFiles     []DumpFileInfo
	DiscoveredDBs []DiscoveredDB

	// Schedule
	DBDumpSchedule string
	NextDBDump     time.Time

	// App data backup
	AppDataInfo []AppBackupInfo

	// Flash messages (set by handlers, passed through redirect)
	FlashSuccess string
	FlashError   string

	// SingleCopyWarning (F6, CAMPAIGN-3) is a non-empty honest Hungarian notice when the box has NO
	// off-drive target at all — tier-1 is then the ONLY local copy and 3-2-1 needs a 2nd drive or
	// offsite. Empty when an off-drive (tier-2) target exists. Never a fake 3-2-1 guarantee.
	SingleCopyWarning string
}

// systemDriveLabel is the human label for the internal SSD / system drive (F6 — a sys_drive app's
// backup used to render with a blank drive label). Matches the tier-2 UI wording.
const systemDriveLabel = "Belső SSD (rendszer)"

// singleCopyNotice is the honest single-drive signal (F6): shown when no off-drive tier-2 target
// exists, instead of silently implying a 3-2-1 guarantee the box cannot provide.
const singleCopyNotice = "Csak egy másolat készül (nincs második meghajtó) — a 3-2-1 mentéshez csatlakoztasson egy második meghajtót vagy offsite tárolót."

// DBDumpStatus holds the last DB dump result.
type DBDumpStatus struct {
	LastRun  time.Time
	Results  []DumpResult
	Success  bool
	Duration time.Duration
}

// NewManager creates a new backup manager.
func NewManager(cfg *config.Config, sett *settings.Settings, logger *log.Logger) *Manager {
	if cfg.Paths.SystemDataPath == "" {
		logger.Printf("[WARN] [backup] SystemDataPath is empty in config — SSD-only apps will not have correct backup paths")
	}
	m := &Manager{
		cfg:            cfg,
		logger:         logger,
		settings:       sett,
		systemDataPath: cfg.Paths.SystemDataPath,
	}
	m.reconcileCrashedRun()
	return m
}

// reconcileCrashedRun makes the persisted offbox status truthful after a crash (campaign C1): a controller
// that died mid-run left LastStatus="running" on disk (the in-memory single-flight mutex is gone with the
// process, but the persisted status keeps lying "running" forever). Flip it to error with a Hungarian
// "interrupted run" message; the next successful run overwrites it. No-op unless a run was actually in
// flight at the crash.
func (m *Manager) reconcileCrashedRun() {
	if m.settings == nil {
		return
	}
	t := m.settings.GetOffboxTarget()
	if t == nil || t.LastStatus != "running" {
		return
	}
	_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
		o.LastStatus = "error"
		o.LastError = "megszakadt futás (a vezérlő újraindult futás közben)"
	})
	m.logger.Printf("[WARN] [offbox] previous run was interrupted by a controller restart — marking the last status as failed (self-corrects on the next run)")
}

// GetAppDrivePath returns the drive path for an app.
// Uses HDD_PATH from app.yaml if set, otherwise falls back to system data path.
func (m *Manager) GetAppDrivePath(stackName string) string {
	if m.stackProvider != nil {
		if hddPath := m.stackProvider.GetStackHDDPath(stackName); hddPath != "" {
			return hddPath
		}
	}
	if m.systemDataPath == "" {
		m.logger.Printf("[ERROR] [backup] systemDataPath is empty — cannot determine drive for %s", stackName)
	}
	return m.systemDataPath
}

// namespaceRoot maps an app's drive path to its felhom-data namespace ROOT (the dir that directly
// holds backups/ and appdata/). A drive-resident app's in-guest mount IS the namespace already
// (Model A, slice 10 — the agent binds <drive>/felhom-data onto the guest mountpoint), so it is used
// as-is; only the SSD-only system-data fallback gets the felhom-data subdir appended. This is what
// keeps a drive-resident app's backups single-nested instead of .../felhom-data/felhom-data/... .
func (m *Manager) namespaceRoot(drivePath string) string {
	return NamespaceRoot(drivePath, drivePath != m.systemDataPath)
}

// AppNamespaceRoot returns the felhom-data namespace root for a stack's keep-side backups, resolving
// HDD-vs-system provenance internally. For callers outside this package that only know the stack
// name (e.g. the API router) so they don't double-nest the felhom-data segment.
func (m *Manager) AppNamespaceRoot(stackName string) string {
	drivePath := m.GetAppDrivePath(stackName)
	if drivePath == "" {
		return ""
	}
	return m.namespaceRoot(drivePath)
}

// knownStackNames returns the names of all deployed stacks, for M19 DB-container→stack attribution.
// Empty when no provider is wired (DiscoverDatabases then falls back to legacy suffix-strip).
func (m *Manager) knownStackNames() []string {
	if m.stackProvider == nil {
		return nil
	}
	stacks := m.stackProvider.ListDeployedStacks()
	names := make([]string, 0, len(stacks))
	for _, s := range stacks {
		names = append(names, s.Name)
	}
	return names
}

// groupStacksByDrive groups deployed stacks by their home drive path.
func (m *Manager) groupStacksByDrive() map[string][]StackSummary {
	if m.stackProvider == nil {
		return nil
	}
	result := make(map[string][]StackSummary)
	for _, stack := range m.stackProvider.ListDeployedStacks() {
		drive := m.GetAppDrivePath(stack.Name)
		result[drive] = append(result[drive], stack)
	}
	if m.isDebug() {
		for drive, stacks := range result {
			names := make([]string, len(stacks))
			for i, s := range stacks {
				names[i] = s.Name
			}
			m.logger.Printf("[DEBUG] groupStacksByDrive: %s → [%s]", drive, strings.Join(names, ", "))
		}
	}
	return result
}

// SetMigrationRunningCheck wires the mutual-exclusion guard (Change 3): when fn() reports a
// migration is active, the scheduled backup paths skip rather than race it.
func (m *Manager) SetMigrationRunningCheck(fn func() bool) {
	m.migrationRunning = fn
}

// migrationActive reports whether a migration is in progress (false when no check is wired).
func (m *Manager) migrationActive() bool {
	return m.migrationRunning != nil && m.migrationRunning()
}

// RunDBDumps discovers and dumps all databases to per-drive, per-app paths.
func (m *Manager) RunDBDumps(ctx context.Context) error {
	if m.migrationActive() {
		m.logger.Printf("[INFO] [backup] DB dump kihagyva: migráció folyamatban")
		return nil
	}
	if err := m.acquireRunning(); err != nil {
		return err
	}
	defer m.releaseRunning()
	return m.runDBDumpsInternal(ctx)
}

// offsiteRunStamp returns the in-flight offsite run's coherence stamp ("" when none).
func (m *Manager) offsiteRunStamp() (runID, dumpsAt string) {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.offsiteRunID, m.offsiteRunDumpAt
}

// beginOffsiteRunStamp marks the start of an offsite run's coherence window and returns the cleanup.
// The stamp is what CaptureRecoveryUnit writes into each unit manifest, so it must be live across
// BOTH the dump leg and the unit capture that follows it — those two together are the pair.
func (m *Manager) beginOffsiteRunStamp(runID string) func() {
	m.mu.Lock()
	m.offsiteRunID = runID
	m.offsiteRunDumpAt = time.Now().UTC().Format(time.RFC3339)
	m.mu.Unlock()
	return func() {
		m.mu.Lock()
		m.offsiteRunID, m.offsiteRunDumpAt = "", ""
		m.mu.Unlock()
	}
}

// runDBDumpsInternal is the implementation of RunDBDumps. Caller must hold the running flag.
func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
	start := time.Now()
	m.logger.Printf("[INFO] [backup] Starting database dump run")

	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames())
	if err != nil {
		m.logger.Printf("[ERROR] [backup] Database discovery failed: %v", err)
		return err
	}

	// F3: no early return on zero DBs — volume-bearing apps without a database still need their
	// class-B volume dump + recovery-unit refresh below (the DB loop simply has no iterations).
	if len(dbs) == 0 {
		m.logger.Printf("[INFO] [backup] No database containers found")
	} else {
		m.logger.Printf("[INFO] [backup] Discovered %d database(s): %s", len(dbs), dbNames(dbs))
	}

	// Dump each DB to its app's drive path
	var results []DumpResult
	allOK := true
	var summary []string
	var totalSize int64

	for _, db := range dbs {
		drivePath := m.GetAppDrivePath(db.StackName)

		// Skip if drive is disconnected or decommissioned
		if m.settings != nil && m.settings.IsDisconnected(drivePath) {
			m.logger.Printf("[WARN] [backup] Skipping DB dump for %s — drive disconnected: %s", db.StackName, drivePath)
			summary = append(summary, fmt.Sprintf("SKIP %s (drive disconnected)", db.ContainerName))
			continue
		}
		if m.settings != nil && m.settings.IsDecommissioned(drivePath) {
			m.logger.Printf("[WARN] [backup] Skipping DB dump for %s — drive decommissioned: %s", db.StackName, drivePath)
			summary = append(summary, fmt.Sprintf("SKIP %s (drive decommissioned)", db.ContainerName))
			continue
		}

		dumpDir := AppDBDumpPath(m.namespaceRoot(drivePath), db.StackName)

		result := DumpOne(ctx, db, dumpDir, m.logger, m.isDebug())
		results = append(results, result)

		if result.Error != nil {
			allOK = false
			summary = append(summary, fmt.Sprintf("FAIL %s: %v", result.DB.ContainerName, result.Error))
			m.logger.Printf("[ERROR] [backup] DB dump failed for %s: %v", result.DB.ContainerName, result.Error)
		} else {
			totalSize += result.Size
			summary = append(summary, fmt.Sprintf("OK %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size)))

			// Persist validation result to settings.json
			if m.settings != nil && result.FilePath != "" {
				filename := filepath.Base(result.FilePath)
				cache := settings.DBValidationCache{
					ValidatedAt: time.Now().Format(time.RFC3339),
					TableCount:  result.Validation.TableCount,
					HasHeader:   result.Validation.Valid,
					Size:        result.Validation.FileSize,
					ModTime:     result.Validation.ModTime.UTC().Format(time.RFC3339),
				}
				if !result.Validation.Valid {
					cache.Error = result.Validation.Error
				}
				if err := m.settings.SetDBValidation(filename, cache); err != nil {
					m.logger.Printf("[WARN] [backup] Failed to cache validation for %s: %v", filename, err)
				}
			}
		}
	}

	// F3: class-B leg — dump each app's named-volume data (stop → tar → restart). MUST run before
	// captureAllRecoveryUnits so the manifests enumerate the fresh tars into VolumeDumps.
	dbOK := allOK
	volSummary, volDumped, volOK := m.runVolumeDumps()
	summary = append(summary, volSummary...)
	allOK = dbOK && volOK

	duration := time.Since(start)
	m.mu.Lock()
	m.lastDBDump = &DBDumpStatus{
		LastRun:  time.Now(),
		Results:  results,
		Success:  allOK,
		Duration: duration,
	}
	m.mu.Unlock()

	if allOK {
		m.logger.Printf("[INFO] [backup] App-data backup completed: %d databases (%s total), %d volume dump(s) (%s)",
			len(results), humanizeBytes(totalSize), volDumped, duration.Round(time.Millisecond))
	} else {
		// Still refresh recovery units below — a partial failure shouldn't leave units stale.
		m.logger.Printf("[WARN] [backup] some backup steps failed (%s); refreshing recovery units anyway",
			strings.Join(failedSummaryLines(summary), "; "))
	}

	// Phase 2: refresh each deployed app's self-contained recovery unit (compose + manifest).
	m.captureAllRecoveryUnits()

	// F5 (CAMPAIGN-3): after the units are fresh on the CURRENT drives, prune any orphaned
	// backups/primary/<app> dir an app left on an OLD drive when its HDD_PATH moved — pure disk
	// residue, invisible in the snapshot list. Guarded (deployed + different current drive only).
	m.pruneStalePrimaryDirs()

	// No silent partials: a DB-dump or volume-dump failure fails the whole run.
	if !allOK {
		return fmt.Errorf("some backup steps failed: %s", strings.Join(failedSummaryLines(summary), "; "))
	}
	return nil
}

// failedSummaryLines filters a run summary down to its FAIL entries (for logs/errors).
func failedSummaryLines(summary []string) []string {
	var failed []string
	for _, s := range summary {
		if strings.HasPrefix(s, "FAIL ") {
			failed = append(failed, s)
		}
	}
	return failed
}

// runVolumeDumps exports the Docker named-volume data of every deployed, unprotected stack whose
// drive is writable — the class-B leg of the nightly app-data backup. (F3: DumpAppVolumesSafe
// previously had NO production caller, so volume-dumps/ was never produced and the granular
// restore had nothing to restore for named-volume apps.) Caller must hold the running flag.
//
// Gate ORDER is load-bearing: the volume check precedes DumpAppVolumesSafe, because the Safe
// variant stops the stack before its own volume check — calling it unconditionally would bounce
// every volume-less app on every nightly run. Per-stack isolation mirrors the DB loop: one app's
// failure is recorded and does not abort the others.
func (m *Manager) runVolumeDumps() (summary []string, dumped int, allOK bool) {
	allOK = true
	if m.stackProvider == nil {
		return nil, 0, true
	}
	dump := m.dumpVolumesSafe
	if dump == nil {
		dump = m.DumpAppVolumesSafe
	}

	for _, stack := range m.stackProvider.ListDeployedStacks() {
		// Never stop/dump infra stacks (felhom-controller, traefik, cloudflared).
		if m.cfg != nil && m.cfg.IsProtectedStack(stack.Name) {
			continue
		}
		// Volume check FIRST — a volume-less stack must not be stopped at all (see gate-order note).
		if len(m.stackProvider.GetDockerVolumes(stack.Name)) == 0 {
			if m.isDebug() {
				m.logger.Printf("[DEBUG] [backup] %s has no named volumes — volume dump skipped", stack.Name)
			}
			continue
		}
		// Same drive-state skip guards as the DB-dump loop.
		drivePath := m.GetAppDrivePath(stack.Name)
		if m.settings != nil && m.settings.IsDisconnected(drivePath) {
			m.logger.Printf("[WARN] [backup] Skipping volume dump for %s — drive disconnected: %s", stack.Name, drivePath)
			summary = append(summary, fmt.Sprintf("SKIP %s volumes (drive disconnected)", stack.Name))
			continue
		}
		if m.settings != nil && m.settings.IsDecommissioned(drivePath) {
			m.logger.Printf("[WARN] [backup] Skipping volume dump for %s — drive decommissioned: %s", stack.Name, drivePath)
			summary = append(summary, fmt.Sprintf("SKIP %s volumes (drive decommissioned)", stack.Name))
			continue
		}

		if err := dump(stack.Name); err != nil {
			allOK = false
			summary = append(summary, fmt.Sprintf("FAIL %s volumes: %v", stack.Name, err))
			m.logger.Printf("[ERROR] [backup] Volume dump failed for %s: %v", stack.Name, err)
			continue
		}
		dumped++
		summary = append(summary, fmt.Sprintf("OK %s volumes", stack.Name))
	}
	return summary, dumped, allOK
}

// DumpAppVolumes exports Docker named volumes to tar files for the given stack.
// Tars are written to AppVolumeDumpPath(drivePath, stackName)/.
// Uses "docker run alpine tar" (same pattern as appexport).
func (m *Manager) DumpAppVolumes(stackName string) error {
	if m.stackProvider == nil {
		return nil
	}

	volumes := m.stackProvider.GetDockerVolumes(stackName)
	if len(volumes) == 0 {
		return nil
	}

	drivePath := m.GetAppDrivePath(stackName)
	if drivePath == "" {
		return fmt.Errorf("cannot determine drive path for %s", stackName)
	}

	dumpDir := AppVolumeDumpPath(m.namespaceRoot(drivePath), stackName)
	if err := os.MkdirAll(dumpDir, 0755); err != nil {
		return fmt.Errorf("creating volume dump dir: %w", err)
	}

	var dumpErrors []string
	for _, volName := range volumes {
		tarPath := filepath.Join(dumpDir, volName+".tar")
		// F7 (CAMPAIGN-3, HIGH): write the tar to a `.tar.tmp` sibling and only atomically rename it
		// over the restore point on success — the same crash-safe pattern the DB-dump path uses
		// (appbackup/dbdump.go DumpOne). Before this, tar wrote the `.tar` IN PLACE, so a mid-write NFS
		// cut left a 0-byte tar REPLACING the last good dump (tier-1 restore is replace-semantics → an
		// empty volume). Now a failed/interrupted write only ever touches the `.tmp`; the last good
		// `.tar` is untouched. The `.tmp` name (ends `.tmp`, not `.tar`) is invisible to the
		// restore-point/stale scans, so it is never mistaken for a restore point.
		tmpPath := tarPath + ".tmp"
		if m.isDebug() {
			m.logger.Printf("[DEBUG] [backup] Dumping volume %s for %s", volName, stackName)
		}

		out, err := m.tarVolumeOrDefault(volName, dumpDir)

		if err != nil {
			// Any tar error or context timeout (incl. a dead NFS target → EIO): remove ONLY the tmp,
			// leave the existing `.tar` restore point byte-untouched, WARN, continue.
			m.logger.Printf("[WARN] [backup] Volume dump failed for %s/%s (last good dump preserved): %s — %v",
				stackName, volName, strings.TrimSpace(string(out)), err)
			os.Remove(tmpPath)
			dumpErrors = append(dumpErrors, volName)
			continue
		}

		// fsync the tmp file (flush the tar to disk) then atomically rename over the restore point.
		if err := atomicPromoteTar(tmpPath, tarPath); err != nil {
			m.logger.Printf("[WARN] [backup] Volume dump promote failed for %s/%s (last good dump preserved): %v",
				stackName, volName, err)
			os.Remove(tmpPath)
			dumpErrors = append(dumpErrors, volName)
			continue
		}

		if info, _ := os.Stat(tarPath); info != nil {
			m.logger.Printf("[INFO] [backup] Volume dump: %s/%s → %s", stackName, volName, humanizeBytes(info.Size()))
		}
	}

	// Clean up tars (and any orphan `.tar.tmp` from a killed run) for volumes that no longer exist.
	entries, _ := os.ReadDir(dumpDir)
	activeVols := make(map[string]bool)
	for _, v := range volumes {
		activeVols[v+".tar"] = true
	}
	for _, e := range entries {
		name := e.Name()
		// A leftover `.tar.tmp` is never a restore point — always safe to remove (its `.tar` sibling,
		// if any, is the real restore point and is handled by the `.tar` branch).
		if strings.HasSuffix(name, ".tar.tmp") {
			os.Remove(filepath.Join(dumpDir, name))
			continue
		}
		if !activeVols[name] && strings.HasSuffix(name, ".tar") {
			os.Remove(filepath.Join(dumpDir, name))
			if m.isDebug() {
				m.logger.Printf("[DEBUG] [backup] Removed stale volume dump: %s/%s", stackName, name)
			}
		}
	}

	if len(dumpErrors) > 0 {
		return fmt.Errorf("volume dump failed for: %s", strings.Join(dumpErrors, ", "))
	}
	return nil
}

// tarVolumeOrDefault runs the F7 tar seam (m.tarVolume) or, when unset, the real docker tar into the
// `<volName>.tar.tmp` sibling under dumpDir. The 10-minute bound matches the original.
func (m *Manager) tarVolumeOrDefault(volName, dumpDir string) ([]byte, error) {
	if m.tarVolume != nil {
		return m.tarVolume(volName, dumpDir)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()
	cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
		"-v", volName+":/vol:ro",
		"-v", dumpDir+":/out",
		"alpine", "tar", "cf", "/out/"+volName+".tar.tmp", "-C", "/vol", ".")
	return cmd.CombinedOutput()
}

// atomicPromoteTar fsyncs a completed `.tar.tmp` then atomically renames it over the final `.tar`
// (same-dir rename = atomic on the target fs). It mirrors DumpOne's crash-safety (F7) and EXCEEDS it
// by also best-effort fsync'ing the directory entry, so the rename itself survives a power loss — the
// DB-dump path fsyncs the file but not the dir; a follow-up could add the dir fsync there too. On any
// error the tmp is left for the caller to remove; the final `.tar` is never touched here except by a
// successful rename.
func atomicPromoteTar(tmpPath, finalPath string) error {
	// O_RDWR, not os.Open: fsync on a read-only handle is refused on Windows (dev-box test runs),
	// while a writable handle syncs on every platform. Content is not modified.
	f, err := os.OpenFile(tmpPath, os.O_RDWR, 0)
	if err != nil {
		return fmt.Errorf("opening tmp dump: %w", err)
	}
	if err := f.Sync(); err != nil {
		f.Close()
		return fmt.Errorf("syncing tmp dump: %w", err)
	}
	if err := f.Close(); err != nil {
		return fmt.Errorf("closing tmp dump: %w", err)
	}
	if err := os.Rename(tmpPath, finalPath); err != nil {
		return fmt.Errorf("renaming tmp dump: %w", err)
	}
	// Best-effort: fsync the directory so the rename is durable (ignore errors — the rename already
	// made the new content visible; this only hardens against a power loss immediately after).
	if dir, derr := os.Open(filepath.Dir(finalPath)); derr == nil {
		_ = dir.Sync()
		_ = dir.Close()
	}
	return nil
}

// DumpAppVolumesSafe stops the stack before dumping volumes and restarts after.
// Prevents inconsistent tars of live database volumes (e.g. PostgreSQL).
// Protected stacks that reject StopStack will return an error — callers handle as warning.
func (m *Manager) DumpAppVolumesSafe(stackName string) error {
	if m.stackProvider == nil {
		return fmt.Errorf("no stack provider")
	}

	m.logger.Printf("[INFO] [backup] Stopping %s for safe volume dump", stackName)
	if err := m.stackProvider.StopStack(stackName); err != nil {
		return fmt.Errorf("could not stop %s for volume dump: %w", stackName, err)
	}

	dumpErr := m.DumpAppVolumes(stackName)

	m.logger.Printf("[INFO] [backup] Restarting %s after volume dump", stackName)
	startErr := m.stackProvider.StartStack(stackName)
	if startErr != nil {
		m.logger.Printf("[ERROR] [backup] Failed to restart %s after volume dump: %v", stackName, startErr)
	}

	// Surface both errors — callers must know if the app is left stopped
	if dumpErr != nil && startErr != nil {
		return fmt.Errorf("volume dump failed for %s: %v; restart also failed: %v", stackName, dumpErr, startErr)
	}
	if startErr != nil {
		return fmt.Errorf("volume dump OK but restart failed for %s: %w", stackName, startErr)
	}
	return dumpErr
}

// GetStatus returns the current DB-dump status.
func (m *Manager) GetStatus() *DBDumpStatus {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.lastDBDump
}

// IsRunning returns whether a backup or restore is currently in progress.
func (m *Manager) IsRunning() bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.running
}

// acquireRunning atomically sets the running flag. Returns error if already running.
func (m *Manager) acquireRunning() error {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.running {
		return fmt.Errorf("backup already in progress")
	}
	m.running = true
	return nil
}

// releaseRunning clears the running flag.
func (m *Manager) releaseRunning() {
	m.mu.Lock()
	m.running = false
	m.mu.Unlock()
}

// SetSecretGenerator wires the O4 resettable-secret generator used by RestoreFromRecoveryUnit
// (init-only, same contract as SetStackProvider: call once during single-threaded startup).
func (m *Manager) SetSecretGenerator(fn func(stackName, envVar string) (string, bool)) {
	m.generateSecret = fn
}

// SetStackProvider sets the stack data provider for app data discovery.
//
// M2: this MUST be called exactly once during single-threaded startup (main.go),
// before the scheduler / HTTP server / any backup goroutine starts. That write
// then happens-before all the (unlocked) reads of m.stackProvider, so no data
// race exists. The earlier mutex on this write was misleading — it implied
// runtime concurrency the reads don't honour; removed to make the init-only
// contract explicit. Do NOT call this after startup.
func (m *Manager) SetStackProvider(provider StackDataProvider) {
	m.stackProvider = provider
}

// GetStackHDDMounts returns HDD mount paths for the named stack via the stack provider.
func (m *Manager) GetStackHDDMounts(name string) []string {
	if m.stackProvider == nil {
		return nil
	}
	return m.stackProvider.GetStackHDDMounts(name)
}

// DumpStackDB runs a database dump for containers belonging to a specific stack.
// Dumps to the stack's home drive: <drive>/backups/primary/<stack>/db-dumps/.
func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames())
	if err != nil {
		return fmt.Errorf("database discovery failed: %w", err)
	}

	var stackDBs []DiscoveredDB
	for _, db := range dbs {
		if db.StackName == stackName {
			stackDBs = append(stackDBs, db)
		}
	}
	if len(stackDBs) == 0 {
		m.logger.Printf("[DEBUG] No databases found for stack %s — skipping pre-backup dump", stackName)
		return nil
	}

	drivePath := m.GetAppDrivePath(stackName)
	if drivePath == "" || !filepath.IsAbs(drivePath) {
		return fmt.Errorf("cannot determine absolute drive path for %s (systemDataPath not configured?)", stackName)
	}
	dumpDir := AppDBDumpPath(m.namespaceRoot(drivePath), stackName)

	m.logger.Printf("[INFO] [backup] Running pre-backup DB dump for %s (%d database(s)) → %s", stackName, len(stackDBs), dumpDir)

	for _, db := range stackDBs {
		result := DumpOne(ctx, db, dumpDir, m.logger, m.isDebug())
		if result.Error != nil {
			return fmt.Errorf("DB dump failed for %s: %w", result.DB.ContainerName, result.Error)
		}
		m.logger.Printf("[INFO] [backup] Pre-backup DB dump OK: %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size))

		// Persist validation to settings
		if m.settings != nil && result.FilePath != "" {
			filename := filepath.Base(result.FilePath)
			cache := settings.DBValidationCache{
				ValidatedAt: time.Now().Format(time.RFC3339),
				TableCount:  result.Validation.TableCount,
				HasHeader:   result.Validation.Valid,
				Size:        result.Validation.FileSize,
				ModTime:     result.Validation.ModTime.UTC().Format(time.RFC3339),
			}
			if !result.Validation.Valid {
				cache.Error = result.Validation.Error
			}
			_ = m.settings.SetDBValidation(filename, cache)
		}
	}
	return nil
}

// listAllDumpFiles scans per-drive per-stack DB dump directories.
//
// M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is
// passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min
// cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime —
// so the write-back (and its settings.json disk write) only happens when a dump actually changed.
func (m *Manager) listAllDumpFiles() []DumpFileInfo {
	modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) }

	var cacheSnapshot map[string]settings.DBValidationCache
	if m.settings != nil {
		cacheSnapshot = m.settings.GetDBValidations()
	}
	lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
		c, ok := cacheSnapshot[name]
		if !ok || c.Size != size || c.ModTime != modKey(mod) {
			return DumpValidation{}, false // cache miss / changed → validate
		}
		return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true
	}

	var allFiles []DumpFileInfo
	for drive, stacks := range m.groupStacksByDrive() {
		for _, stack := range stacks {
			dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name)
			files, err := ListDumpFiles(dumpDir, lookup)
			if err != nil {
				continue
			}
			for _, f := range files {
				// Write back only fresh validations (cache miss against the snapshot), so unchanged
				// dumps cause neither a re-validation nor a settings.json write each cycle.
				if m.settings != nil {
					if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) {
						_ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{
							ValidatedAt: time.Now().Format(time.RFC3339),
							TableCount:  f.Validation.TableCount,
							HasHeader:   f.Validation.Valid,
							Error:       f.Validation.Error,
							Size:        f.Size,
							ModTime:     modKey(f.ModTime),
						})
					}
				}
				allFiles = append(allFiles, f)
			}
		}
	}
	m.logger.Printf("[INFO] [backup] Found %d DB dump files across drives", len(allFiles))
	return allFiles
}

// RefreshCache updates the cached full status. Called by scheduler every 5 minutes.
func (m *Manager) RefreshCache(nextDBDump time.Time) {
	status := &FullBackupStatus{
		Enabled:        m.cfg.Backup.Enabled,
		DBDumpSchedule: m.cfg.Backup.DBDumpSchedule,
		NextDBDump:     nextDBDump,
	}

	// Scan dump files from per-drive per-stack paths
	files := m.listAllDumpFiles()
	status.DumpFiles = files

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	if dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()); err == nil {
		status.DiscoveredDBs = dbs
	}

	// Discover app data — all deployed stacks, backup is mandatory
	if m.stackProvider != nil {
		status.AppDataInfo = DiscoverAppData(m.stackProvider, status.DiscoveredDBs)
		// Phase 2: keep each app's recovery unit current with its definition. Idempotent
		// (checksum-skip), so this periodic refresh only writes when the config actually changed,
		// and ensures units exist shortly after startup without waiting for the daily DB dump.
		m.captureAllRecoveryUnits()
	}

	// Fill in dynamic fields under lock.
	m.mu.Lock()
	status.Running = m.running
	status.LastDBDump = m.lastDBDump

	// Cross-check lastDBDump results inside lock to prevent torn writes.
	if m.lastDBDump != nil && len(files) > 0 {
		fileValidation := make(map[string]DumpValidation) // keyed by filename
		for _, f := range files {
			fileValidation[f.FileName] = f.Validation
		}
		for i, r := range m.lastDBDump.Results {
			if !r.Validation.Valid && r.Validation.Error == "" && r.FilePath != "" {
				filename := filepath.Base(r.FilePath)
				if fv, ok := fileValidation[filename]; ok {
					m.lastDBDump.Results[i].Validation = fv
					m.logger.Printf("[INFO] [backup] Re-validated %s from disk: valid=%v tables=%d",
						filename, fv.Valid, fv.TableCount)
				}
			}
		}
	}

	m.cachedStatus = status
	m.cacheTime = time.Now()
	m.mu.Unlock()

	m.logger.Printf("[INFO] [backup] Backup status cache refreshed")
}

// GetFullStatus returns the cached backup status for page rendering.
// Returns instantly — no subprocess calls.
// Returns a deep copy so callers can safely append to slice fields without
// polluting the cache.
func (m *Manager) GetFullStatus(nextDBDump time.Time) *FullBackupStatus {
	m.mu.Lock()
	defer m.mu.Unlock()

	if m.cachedStatus != nil {
		status := *m.cachedStatus
		status.AppDataInfo = make([]AppBackupInfo, len(m.cachedStatus.AppDataInfo))
		copy(status.AppDataInfo, m.cachedStatus.AppDataInfo)

		// Update dynamic fields that don't need subprocess calls
		status.Running = m.running
		status.NextDBDump = nextDBDump
		status.SingleCopyWarning = m.singleCopyWarning() // F6: honest single-drive signal
		// Deep-copy lastDBDump so callers cannot mutate shared state.
		if m.lastDBDump != nil {
			copyDump := *m.lastDBDump
			if len(m.lastDBDump.Results) > 0 {
				copyDump.Results = make([]DumpResult, len(m.lastDBDump.Results))
				copy(copyDump.Results, m.lastDBDump.Results)
			}
			status.LastDBDump = &copyDump
		}

		// Synthesize LastDBDump from DumpFiles on disk if not in memory
		if status.LastDBDump == nil && len(status.DumpFiles) > 0 {
			var results []DumpResult
			var latestTime time.Time
			for _, f := range status.DumpFiles {
				results = append(results, DumpResult{
					DB:         DiscoveredDB{StackName: f.StackName, DBType: f.DBType, ContainerName: f.StackName},
					FilePath:   f.FileName,
					Size:       f.Size,
					Validation: f.Validation,
				})
				if f.ModTime.After(latestTime) {
					latestTime = f.ModTime
				}
			}
			status.LastDBDump = &DBDumpStatus{
				LastRun: latestTime,
				Results: results,
				Success: true,
			}
		}

		return &status
	}

	// No cache yet — return a minimal status (first page load before cache is populated)
	status := &FullBackupStatus{
		Enabled:           m.cfg.Backup.Enabled,
		Running:           m.running,
		DBDumpSchedule:    m.cfg.Backup.DBDumpSchedule,
		NextDBDump:        nextDBDump,
		SingleCopyWarning: m.singleCopyWarning(), // F6
	}
	if m.lastDBDump != nil {
		copyDump := *m.lastDBDump
		if len(m.lastDBDump.Results) > 0 {
			copyDump.Results = make([]DumpResult, len(m.lastDBDump.Results))
			copy(copyDump.Results, m.lastDBDump.Results)
		}
		status.LastDBDump = &copyDump
	}
	return status
}

// sameDevice reports whether two paths sit on the same physical device, through the test seam when
// one is installed. Nil seam → system.SamePhysicalDevice, i.e. byte-for-byte the previous behaviour.
func (m *Manager) sameDevice(a, b string) bool {
	if m.samePhysicalDevice != nil {
		return m.samePhysicalDevice(a, b)
	}
	return system.SamePhysicalDevice(a, b)
}

// hasOffDriveTarget reports whether any registered, schedulable storage path lives on a physical disk
// OTHER than the system drive — i.e. whether a genuine off-drive (tier-2) copy is possible at all.
// When false the box is single-drive: tier-1 is the ONLY local copy and 3-2-1 needs a 2nd drive or
// offsite (F6 — surfaced honestly via SingleCopyWarning, never a faked guarantee).
func (m *Manager) hasOffDriveTarget() bool {
	if m.settings == nil || m.systemDataPath == "" {
		return false
	}
	for _, sp := range m.settings.GetSchedulableStoragePaths() {
		if sp.Path == m.systemDataPath || m.sameDevice(m.systemDataPath, sp.Path) {
			continue
		}
		return true
	}
	return false
}

// singleCopyWarning returns the honest single-drive notice, or "" when an off-drive target exists.
func (m *Manager) singleCopyWarning() string {
	if m.hasOffDriveTarget() {
		return ""
	}
	return singleCopyNotice
}

// sysDriveLabelFor returns the drive label for a stack's tier-1 restore point — the clear
// system-drive label for a sys_drive (volume-only) app (F6: never blank), else the enrolled drive's
// storage label.
func (m *Manager) sysDriveLabelFor(stackName string) string {
	drive := m.GetAppDrivePath(stackName)
	if drive == "" {
		return ""
	}
	if drive == m.systemDataPath {
		return systemDriveLabel
	}
	if m.settings != nil {
		return m.settings.GetStorageLabel(drive)
	}
	return ""
}

// pruneStalePrimaryDirs removes orphaned `backups/primary/<app>` dirs left on a drive after an app's
// HDD_PATH moved to another drive (F5, CAMPAIGN-3 — pure disk residue, invisible in the snapshot
// list). LOAD-BEARING GUARDS: a dir is removed ONLY when <app> is currently deployed AND its current
// namespace root differs from this dir's drive. It NEVER removes the dir on the app's CURRENT drive
// (that IS the live restore point), and NEVER removes a dir for an app NOT in the deployed set (an
// undeployed app's last backup is still its restore point — orphaned-app cleanup is a separate,
// user-driven concern). Only operates strictly under a `backups/primary/` prefix.
func (m *Manager) pruneStalePrimaryDirs() {
	if m.stackProvider == nil {
		return
	}
	// Current namespace root per DEPLOYED app.
	current := map[string]string{}
	for _, s := range m.stackProvider.ListDeployedStacks() {
		if drive := m.GetAppDrivePath(s.Name); drive != "" {
			current[s.Name] = filepath.Clean(m.namespaceRoot(drive))
		}
	}
	// Candidate drives to scan: the system drive + every registered storage path.
	var nsRoots []string
	if m.systemDataPath != "" {
		nsRoots = append(nsRoots, filepath.Clean(m.namespaceRoot(m.systemDataPath)))
	}
	if m.settings != nil {
		for _, sp := range m.settings.GetStoragePaths() {
			nsRoots = append(nsRoots, filepath.Clean(NamespaceRoot(sp.Path, true)))
		}
	}
	seen := map[string]bool{}
	for _, nsRoot := range nsRoots {
		if seen[nsRoot] {
			continue
		}
		seen[nsRoot] = true
		primaryDir := PrimaryBackupPath(nsRoot)
		entries, err := os.ReadDir(primaryDir)
		if err != nil {
			continue // absent/unreadable (e.g. a disconnected drive) — nothing to prune here
		}
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}
			app := e.Name()
			cur, deployed := current[app]
			if !deployed {
				continue // GUARD: an undeployed app's last backup is still its restore point
			}
			if cur == nsRoot {
				continue // GUARD: this IS the app's current drive — the live restore point
			}
			stalePath := RecoveryUnitPath(nsRoot, app)
			// Prefix safety: only ever remove strictly inside `backups/primary/` (no surprise user data).
			if !strings.HasPrefix(filepath.Clean(stalePath)+string(filepath.Separator),
				filepath.Clean(primaryDir)+string(filepath.Separator)) {
				continue
			}
			if err := os.RemoveAll(stalePath); err != nil {
				m.logger.Printf("[WARN] [backup] F5: could not remove stale primary dir for %s on old drive: %v", app, err)
			} else {
				m.logger.Printf("[INFO] [backup] F5: removed stale primary backup dir for %s on an old drive (app now on %s)", app, cur)
			}
		}
	}
}

// isDebug returns true if logging level is "debug".
func (m *Manager) isDebug() bool {
	return m.cfg != nil && m.cfg.Logging.Level == "debug"
}

func dbNames(dbs []DiscoveredDB) string {
	var names []string
	for _, db := range dbs {
		names = append(names, fmt.Sprintf("%s(%s)", db.ContainerName, db.DBType))
	}
	return strings.Join(names, ", ")
}