package backup import ( "context" "fmt" "log" "os" "os/exec" "path/filepath" "strings" "sync" "sync/atomic" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/config" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" "gitea.dooplex.hu/admin/felhom-controller/internal/system" ) // Manager orchestrates app-data backups: database dumps and Docker-volume tars. // // Disk-tier backup (restic, cross-drive, drive-recovery, infra-backup) has been // moved out of the controller into the host agent (slice 8C). This Manager now // only owns the app-data domain. type Manager struct { cfg *config.Config logger *log.Logger settings *settings.Settings stackProvider StackDataProvider systemDataPath string // fallback drive for SSD-only apps version string // controller version, stamped into recovery-unit manifests // tier2Notify, if set, is called after each Tier 2 copy (success: err==nil) for notifications. tier2Notify func(stackName, destLabel string, dur time.Duration, err error) // unitNotify (R-158 / R-167), if set, is called ONCE PER APP whose Tier-1 recovery-unit capture // FAILED, and the capture loop continues to the next app. Wired in cmd/controller/main.go. // // WHY IT EXISTS. `/backups/apps` is the page a person opens to ask whether ONE app is backed up, // and until now it was the one page that never said: a per-app capture failure was a `[WARN]` // line and went no further. The manager had three notify seams and none for the unit capture — // the FIFTH instance in this project of a mechanism built and left disconnected. // // IT CARRIES THE SPACE FIGURES DELIBERATELY. The overwhelmingly likely cause is a full // filesystem, and an operator who has the used/free bytes at the moment of failure can act // without logging in. It is the same pair of numbers the customer-facing fill warning reports, // which is why the two ship together. // // OPERATOR-TIER. Routed to a hub event type that is in `notify.operatorOnlyEvents` — a customer // can take no action on a capture failure. Deliberately NOT `backup_failed`, which is // customer-enabled by default and would email them in Hungarian about it (D-c). // // NO CONTROLLER-SIDE COOLDOWN — the hub owns cooldown, per the offboxEnlargeBlockedNotify // precedent. unitNotify func(stackName string, err error, usage *UnitSpace) // unitSpaceFn (R-165 / B2), if set, replaces the real statfs behind the capture floor so a test // can state a filesystem's occupancy as an input. Nil in production → `unitTargetSpace`. unitSpaceFn func(stackName string) *UnitSpace // admission (R-181) is the per-RUN memo of the reserve's per-app verdict, guarded by admissionMu. // Non-nil only for the duration of a backup run (beginAdmissionRun → its closer). One verdict per // app covers all THREE write legs — DB dump, volume dump, unit capture — because all three write // under one per-app root; see admission.go for why it is decided lazily and never re-decided. admissionMu sync.Mutex admission *admissionSet // summary (R-182) is the per-RUN digest collector, guarded by summaryMu. Same lifetime as // `admission` and for the same reason: an absent collector means "no run in flight", never a // stale answer from last night. runSummaryNotify is the operator digest seam, wired in main.go. summaryMu sync.Mutex summary *runSummary runSummaryNotify func(RunSummary) // manualRun tags the NEXT run as operator-triggered (cleared as the run starts), so the digest // can say which kind it was and the hub can decline to collapse a manual run into a nightly one. manualRun atomic.Bool // appStop (R-166) is the crash marker for operations that stop an app, work on its data, and // start it again. Written BEFORE the stop and cleared AFTER the restart, so a SIGKILL or a power // cut in that window leaves a durable record that Recover honours at the next startup. Built in // NewManager from cfg.Paths.DataDir — see appstop_marker.go for why it is not quiesce's file. appStop *AppStopGuard // offbox (Part B): the restic-SFTP exec seam (nil → real restic) + the failure→operator-alert hook. offboxRunner offboxRunner offboxNotify func(dur time.Duration, snapshots int, err error) // offboxStreamRunner + offboxProgress: the MANUAL run's live progress (v0.147.0, 4c). The stream // seam scans restic's `--json` stdout line-by-line; the state is what the page polls. Both are // inert on the nightly path — the sink is installed only for the duration of a manual run. offboxStreamRunner offboxStreamRunner offboxProgress offboxProgressState // offboxOrphanEvent (v0.142.0), if set, pushes a hub event on offsite-repo continuity transitions // ("offbox_repo_orphaned" / "offbox_repo_reset"); renamedTo names the move-aside path (reset only). // Wired in main.go to the notifier. Nil-safe. offboxOrphanEvent func(eventType, renamedTo string) // offboxGapNotify (R-203) fires when a COMPLETED offsite run could not capture a directory an // app declares MANDATORY — a coverage gap, not a failed run. nil → no signal. offboxGapNotify func(gaps map[string][]string) // offboxSSH (v0.142.0) is the raw-ssh exec seam for the orphaned-repo move-aside (restic has no // rename); tests inject a fake. Nil → the real ssh invocation (defaultOffboxSSH). offboxSSH func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) // offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable // in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb). offboxSizer func(path string) int64 // offboxEnlargeBlockedNotify (3a), if set, is called ONCE per app that NEWLY enters the // quota-blocked (enlargement-refused) state — edge-triggered against the persisted EnlargedBlocked // set so a nightly schedule can't re-notify a persistently-blocked app (the hub owns cooldown; the // controller must not add a timer). Wired in cmd/controller/main.go. offboxEnlargeBlockedNotify func(stack string, estBytes int64, usedGB, quotaGB int) // offboxPlaceCopier (3a) — the place-to-live missing-only merge seam (nil → rsyncRestoreMissing, // the `-a --ignore-existing` additive copy). Never rsyncMirror (--delete trap). offboxPlaceCopier func(src, dst string) (int, error) // offboxFullPlaceCopier (R-43, v0.148.0) — the FULL-restore overwrite seam (nil → // rsyncRestoreOverwrite: `-a` with NO --ignore-existing and NO --delete). Distinct from // offboxPlaceCopier on purpose: the two have opposite semantics for an existing file. offboxFullPlaceCopier func(src, dst string) (int, error) // safetyDumpFn (R-43) — the pre-restore safety-dump seam (nil → the real DumpOne), so the // "never replay without an undo on disk" refusal is unit-testable without Docker. safetyDumpFn func(ctx context.Context, db DiscoveredDB, dumpDir string) DumpResult // offsitePreDumpFn (R-44) — the offsite dump pre-phase seam (nil → runDBDumpsInternal), so the // dumps-strictly-before-capture ordering is observable in a test without Docker or restic. offsitePreDumpFn func(ctx context.Context) error // offboxFreeFn (3a) — the free-space probe for the restore headroom gate, overridable in tests (the // Windows `go test` host has no `df`). Nil → the real diskFreeBytes (df --output=avail). offboxFreeFn func(path string) int64 // F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested // without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps). discoverDBs func(ctx context.Context) ([]DiscoveredDB, error) importDBDump func(ctx context.Context, db DiscoveredDB, dumpPath string) error // F3 volume-dump seam — overridable in tests so runVolumeDumps' gating (protected / volume-less / // disconnected) can be unit-tested without Docker. Nil → the real DumpAppVolumesSafe. dumpVolumesSafe func(stackName string) error // F7 tar seam — the ONE docker exec inside DumpAppVolumes, overridable so the atomic-write // behaviour (tmp+fsync+rename; the last good `.tar` survives a mid-write failure) is unit-testable // without Docker. It must write the tar to `/.tar.tmp` and return the combined // output + error. Nil → the real `docker run … alpine tar cf …tar.tmp`. tarVolume func(volName, dumpDir string) ([]byte, error) // F6 per-app tier-2 seam — overridable so RunAllTier2's app SELECTION (now incl. volume-only apps) // is unit-testable without rsync/du. Nil → the real RunTier2. perAppTier2 func(stackName string) error // generateSecret (O4), if set, produces a replacement value for a RESETTABLE secret that could // not be recovered during restore-from-unit (wired to stacks.Manager.GenerateSecretForField in // main.go). Nil / ok=false → the secret stays absent and the restore proceeds with a loud WARN. // NEVER consulted for data-keys — the fail-closed gate refuses those before generation runs. generateSecret func(stackName, envVar string) (string, bool) // restoreFilesCopier (C2) — the Tier-2 in-place file-restore copy seam, overridable in tests so // the orchestration never shells out. Nil → the real rsyncRestoreMissing (additive-only). restoreFilesCopier func(src, dst string) (filesRestored int, err error) // tier2Mirror (F-S2) — the Tier-2 backup mirror seam (both rsync legs in RunTier2), overridable // so the resolve→mirror→record flow is unit-testable without rsync. Nil → the real rsyncMirror // (`-a --delete`, contents-of-src semantics). tier2Mirror func(src, dst string) error // sharesPassdbCapture (R-7b) — the samba passdb capture seam (a `docker exec … tar cf -`), // overridable so the shares payload builder is unit-testable without docker. Nil → the real // defaultSharesPassdbCapture. Best-effort by contract: an error yields a manifest-only payload. sharesPassdbCapture func() ([]byte, error) // sharesPassdbRestore (R-7b) — the mirror seam for putting a captured passdb archive BACK into the // samba named volume (`docker exec -i … tar xf -`). Nil → the real defaultSharesPassdbRestore. sharesPassdbRestore func(tar []byte) error // sharesReconcile (R-7b), if set, re-renders and applies the samba stack after a shares restore // re-adds definitions to the registry (wired in main.go to stacks.Manager.ReconcileSamba). It is a // SEAM rather than a direct call because the backup package must not depend on the stacks package. // Nil → the registry is updated and a WARN says smb.conf will catch up on the next health tick. sharesReconcile func() error // tier2SSDFits (3b) — the SSD-headroom predicate seam, overridable in tests (system.GetDiskUsage is // Linux-only → nil on the Windows test host, which would always refuse the SSD branch). Nil → the // real tier2FitsSystemDrive. tier2SSDFits func(sys string, sizeBytes int64) bool // samePhysicalDevice — the off-drive identity predicate behind every Tier-2 "is this really a // SECOND disk?" guard, overridable in tests. The real check is `st_dev` equality, so on a host // where every `t.TempDir()` lands on one filesystem the fixture's "two drives" are indistinguishable // and Tier-2 correctly refuses them — which makes the off-drive tests unrunnable rather than wrong. // Nil → the real system.SamePhysicalDevice (production always takes this path). samePhysicalDevice func(a, b string) bool // migrationRunning, if set, reports whether a data migration is in progress. The scheduled // backup paths skip when it returns true (Change 3 — backup ↔ migration mutual exclusion), so a // nightly dump/Tier-2 can't race a migration copy/cleanup on the same drive. migrationRunning func() bool mu sync.Mutex lastDBDump *DBDumpStatus running bool // R-43/R-44 (v0.148.0) — the coherence stamp of the offsite run in flight, read by // CaptureRecoveryUnit so each unit records WHICH run took the dumps sitting beside its files. // Set for the duration of the dump pre-phase + capture, cleared after; "" means "no offsite run // is establishing coherence right now" (the periodic refresh and the local 02:30 dump leg). offsiteRunID string offsiteRunDumpAt string // Restore op-status (Part B, opstatus.go) — display-only async-restore progress, under `mu`. opRunning bool opName string opStack string opStartedAt time.Time opLast *RestoreOpResult // Cached status for page rendering (refreshed periodically) cachedStatus *FullBackupStatus cacheTime time.Time } // FullBackupStatus contains everything the backup page needs. type FullBackupStatus struct { Enabled bool Running bool // DB Dumps LastDBDump *DBDumpStatus DumpFiles []DumpFileInfo DiscoveredDBs []DiscoveredDB // Schedule DBDumpSchedule string NextDBDump time.Time // App data backup AppDataInfo []AppBackupInfo // Flash messages (set by handlers, passed through redirect) FlashSuccess string FlashError string // SingleCopyWarning (F6, CAMPAIGN-3) is a non-empty honest Hungarian notice when the box has NO // off-drive target at all — tier-1 is then the ONLY local copy and 3-2-1 needs a 2nd drive or // offsite. Empty when an off-drive (tier-2) target exists. Never a fake 3-2-1 guarantee. SingleCopyWarning string } // systemDriveLabel is the human label for the internal SSD / system drive (F6 — a sys_drive app's // backup used to render with a blank drive label). Matches the tier-2 UI wording. const systemDriveLabel = "Belső SSD (rendszer)" // singleCopyNotice is the honest single-drive signal (F6): shown when no off-drive tier-2 target // exists, instead of silently implying a 3-2-1 guarantee the box cannot provide. const singleCopyNotice = "Csak egy másolat készül (nincs második meghajtó) — a 3-2-1 mentéshez csatlakoztasson egy második meghajtót vagy offsite tárolót." // DBDumpStatus holds the last DB dump result. type DBDumpStatus struct { LastRun time.Time Results []DumpResult Success bool Duration time.Duration } // NewManager creates a new backup manager. func NewManager(cfg *config.Config, sett *settings.Settings, logger *log.Logger) *Manager { if cfg.Paths.SystemDataPath == "" { logger.Printf("[WARN] [backup] SystemDataPath is empty in config — SSD-only apps will not have correct backup paths") } m := &Manager{ cfg: cfg, logger: logger, settings: sett, systemDataPath: cfg.Paths.SystemDataPath, } // R-166: its OWN file next to quiesce-state.json, never inside it — one file, one writer. m.appStop = NewAppStopGuard(filepath.Join(cfg.Paths.DataDir, "appstop-state.json"), logger) m.reconcileCrashedRun() return m } // AppStopGuard exposes the app-stop crash marker so the exporter (a different package with the same // stop-work-start shape) can share the one marker file rather than opening a second one. func (m *Manager) AppStopGuard() *AppStopGuard { return m.appStop } // SetAppStopGuard injects the guard instead of using the one NewManager built. INIT-ONLY — call once // during single-threaded startup, before any backup runs. // // It exists because of a startup ORDERING constraint, not for testing: the guard's Recover must // complete before the boot reconciler is launched (main.go:~236) and this manager is not constructed // until ~line 272. So main.go builds the guard early, recovers, and hands the SAME object here — // rather than a second guard over the same file, which would be one file with two owners, the exact // shape this marker was kept out of quiesce's file to avoid. func (m *Manager) SetAppStopGuard(g *AppStopGuard) { if g != nil { m.appStop = g } } // reconcileCrashedRun makes the persisted offbox status truthful after a crash (campaign C1): a controller // that died mid-run left LastStatus="running" on disk (the in-memory single-flight mutex is gone with the // process, but the persisted status keeps lying "running" forever). Flip it to error with a Hungarian // "interrupted run" message; the next successful run overwrites it. No-op unless a run was actually in // flight at the crash. func (m *Manager) reconcileCrashedRun() { if m.settings == nil { return } t := m.settings.GetOffboxTarget() if t == nil || t.LastStatus != "running" { return } _ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "error" o.LastError = "megszakadt futás (a vezérlő újraindult futás közben)" }) m.logger.Printf("[WARN] [offbox] previous run was interrupted by a controller restart — marking the last status as failed (self-corrects on the next run)") } // GetAppDrivePath returns the drive path for an app. // Uses HDD_PATH from app.yaml if set, otherwise falls back to system data path. func (m *Manager) GetAppDrivePath(stackName string) string { if m.stackProvider != nil { if hddPath := m.stackProvider.GetStackHDDPath(stackName); hddPath != "" { return hddPath } } if m.systemDataPath == "" { m.logger.Printf("[ERROR] [backup] systemDataPath is empty — cannot determine drive for %s", stackName) } return m.systemDataPath } // namespaceRoot maps an app's drive path to its felhom-data namespace ROOT (the dir that directly // holds backups/ and appdata/). A drive-resident app's in-guest mount IS the namespace already // (Model A, slice 10 — the agent binds /felhom-data onto the guest mountpoint), so it is used // as-is; only the SSD-only system-data fallback gets the felhom-data subdir appended. This is what // keeps a drive-resident app's backups single-nested instead of .../felhom-data/felhom-data/... . func (m *Manager) namespaceRoot(drivePath string) string { // R-203: delegates to the ONE expression of the rule (appbackup.NamespaceRootFor). This used to // hold its own copy — `drivePath != m.systemDataPath`, without Clean on either side — while // stacks.Manager.inGuest held a second copy WITH Clean. Two copies that already differed. return NamespaceRootFor(drivePath, m.systemDataPath) } // AppNamespaceRoot returns the felhom-data namespace root for a stack's keep-side backups, resolving // HDD-vs-system provenance internally. For callers outside this package that only know the stack // name (e.g. the API router) so they don't double-nest the felhom-data segment. func (m *Manager) AppNamespaceRoot(stackName string) string { drivePath := m.GetAppDrivePath(stackName) if drivePath == "" { return "" } return m.namespaceRoot(drivePath) } // knownStackNames returns the names of all deployed stacks, for M19 DB-container→stack attribution. // Empty when no provider is wired (DiscoverDatabases then falls back to legacy suffix-strip). func (m *Manager) knownStackNames() []string { if m.stackProvider == nil { return nil } stacks := m.stackProvider.ListDeployedStacks() names := make([]string, 0, len(stacks)) for _, s := range stacks { names = append(names, s.Name) } return names } // groupStacksByDrive groups deployed stacks by their home drive path. func (m *Manager) groupStacksByDrive() map[string][]StackSummary { if m.stackProvider == nil { return nil } result := make(map[string][]StackSummary) for _, stack := range m.stackProvider.ListDeployedStacks() { drive := m.GetAppDrivePath(stack.Name) result[drive] = append(result[drive], stack) } if m.isDebug() { for drive, stacks := range result { names := make([]string, len(stacks)) for i, s := range stacks { names[i] = s.Name } m.logger.Printf("[DEBUG] groupStacksByDrive: %s → [%s]", drive, strings.Join(names, ", ")) } } return result } // SetMigrationRunningCheck wires the mutual-exclusion guard (Change 3): when fn() reports a // migration is active, the scheduled backup paths skip rather than race it. func (m *Manager) SetMigrationRunningCheck(fn func() bool) { m.migrationRunning = fn } // migrationActive reports whether a migration is in progress (false when no check is wired). func (m *Manager) migrationActive() bool { return m.migrationRunning != nil && m.migrationRunning() } // RunDBDumps discovers and dumps all databases to per-drive, per-app paths. func (m *Manager) RunDBDumps(ctx context.Context) error { if m.migrationActive() { m.logger.Printf("[INFO] [backup] DB dump kihagyva: migráció folyamatban") return nil } if err := m.acquireRunning(); err != nil { return err } defer m.releaseRunning() return m.runDBDumpsInternal(ctx) } // offsiteRunStamp returns the in-flight offsite run's coherence stamp ("" when none). func (m *Manager) offsiteRunStamp() (runID, dumpsAt string) { m.mu.Lock() defer m.mu.Unlock() return m.offsiteRunID, m.offsiteRunDumpAt } // beginOffsiteRunStamp marks the start of an offsite run's coherence window and returns the cleanup. // The stamp is what CaptureRecoveryUnit writes into each unit manifest, so it must be live across // BOTH the dump leg and the unit capture that follows it — those two together are the pair. func (m *Manager) beginOffsiteRunStamp(runID string) func() { m.mu.Lock() m.offsiteRunID = runID m.offsiteRunDumpAt = time.Now().UTC().Format(time.RFC3339) m.mu.Unlock() return func() { m.mu.Lock() m.offsiteRunID, m.offsiteRunDumpAt = "", "" m.mu.Unlock() } } // runDBDumpsInternal is the implementation of RunDBDumps. Caller must hold the running flag. func (m *Manager) runDBDumpsInternal(ctx context.Context) error { start := time.Now() m.logger.Printf("[INFO] [backup] Starting database dump run") // R-181: open the per-run admission scope HERE, because this function is the single orchestrator // of all three write legs. Each app's reserve verdict is taken at its first write of this run and // then reused by the other two legs, so a refused app writes nothing at all and is alerted once. // The scope is closed on every exit path — a set that outlived its run would answer tonight's // question with last night's disk. defer m.beginAdmissionRun()() // R-182: the digest scope has the same lifetime. `emitRunSummary` runs BEFORE the closer (defers // unwind last-in-first-out), so the summary is still populated when it is sent, and it sends // nothing at all when the run was clean. kind := m.runKindFor() m.manualRun.Store(false) // tags exactly ONE run; a stale flag would mislabel every later nightly defer m.beginRunSummary(kind, newRunID())() defer m.emitRunSummary() dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()) if err != nil { m.logger.Printf("[ERROR] [backup] Database discovery failed: %v", err) return err } // F3: no early return on zero DBs — volume-bearing apps without a database still need their // class-B volume dump + recovery-unit refresh below (the DB loop simply has no iterations). if len(dbs) == 0 { m.logger.Printf("[INFO] [backup] No database containers found") } else { m.logger.Printf("[INFO] [backup] Discovered %d database(s): %s", len(dbs), dbNames(dbs)) } // Dump each DB to its app's drive path var results []DumpResult allOK := true var summary []string var totalSize int64 for _, db := range dbs { drivePath := m.GetAppDrivePath(db.StackName) // Skip if drive is disconnected or decommissioned if m.settings != nil && m.settings.IsDisconnected(drivePath) { m.logger.Printf("[WARN] [backup] Skipping DB dump for %s — drive disconnected: %s", db.StackName, drivePath) summary = append(summary, fmt.Sprintf("SKIP %s (drive disconnected)", db.ContainerName)) continue } if m.settings != nil && m.settings.IsDecommissioned(drivePath) { m.logger.Printf("[WARN] [backup] Skipping DB dump for %s — drive decommissioned: %s", db.StackName, drivePath) summary = append(summary, fmt.Sprintf("SKIP %s (drive decommissioned)", db.ContainerName)) continue } // R-181: the reserve, BEFORE the first byte of this app's backup is written. This is usually // where an app's verdict is taken, because the DB leg runs first; the volume leg and the // capture then read the same memo. SKIP, not FAIL — a deliberate hold is not a broken dump, // and the operator alert (fired once, inside admitApp) is the signal that it happened. m.noteAttempted(db.StackName) if !m.admitApp(db.StackName) { summary = append(summary, fmt.Sprintf("SKIP %s (reserve — app backup refused)", db.ContainerName)) continue } dumpDir := AppDBDumpPath(m.namespaceRoot(drivePath), db.StackName) result := DumpOne(ctx, db, dumpDir, m.logger, m.isDebug()) results = append(results, result) if result.Error != nil { allOK = false summary = append(summary, fmt.Sprintf("FAIL %s: %v", result.DB.ContainerName, result.Error)) m.noteFailure(db.StackName, "database dump", result.Error.Error()) m.logger.Printf("[ERROR] [backup] DB dump failed for %s: %v", result.DB.ContainerName, result.Error) } else { totalSize += result.Size summary = append(summary, fmt.Sprintf("OK %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size))) // Persist validation result to settings.json if m.settings != nil && result.FilePath != "" { filename := filepath.Base(result.FilePath) cache := settings.DBValidationCache{ ValidatedAt: time.Now().Format(time.RFC3339), TableCount: result.Validation.TableCount, HasHeader: result.Validation.Valid, Size: result.Validation.FileSize, ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339), } if !result.Validation.Valid { cache.Error = result.Validation.Error } if err := m.settings.SetDBValidation(filename, cache); err != nil { m.logger.Printf("[WARN] [backup] Failed to cache validation for %s: %v", filename, err) } } } } // F3: class-B leg — dump each app's named-volume data (stop → tar → restart). MUST run before // captureAllRecoveryUnits so the manifests enumerate the fresh tars into VolumeDumps. dbOK := allOK volSummary, volDumped, volOK := m.runVolumeDumps() summary = append(summary, volSummary...) allOK = dbOK && volOK duration := time.Since(start) m.mu.Lock() m.lastDBDump = &DBDumpStatus{ LastRun: time.Now(), Results: results, Success: allOK, Duration: duration, } m.mu.Unlock() if allOK { m.logger.Printf("[INFO] [backup] App-data backup completed: %d databases (%s total), %d volume dump(s) (%s)", len(results), humanizeBytes(totalSize), volDumped, duration.Round(time.Millisecond)) } else { // Still refresh recovery units below — a partial failure shouldn't leave units stale. m.logger.Printf("[WARN] [backup] some backup steps failed (%s); refreshing recovery units anyway", strings.Join(failedSummaryLines(summary), "; ")) } // Phase 2: refresh each deployed app's self-contained recovery unit (compose + manifest). m.captureAllRecoveryUnits() // F5 (CAMPAIGN-3): after the units are fresh on the CURRENT drives, prune any orphaned // backups/primary/ dir an app left on an OLD drive when its HDD_PATH moved — pure disk // residue, invisible in the snapshot list. Guarded (deployed + different current drive only). m.pruneStalePrimaryDirs() // No silent partials: a DB-dump or volume-dump failure fails the whole run. if !allOK { return fmt.Errorf("some backup steps failed: %s", strings.Join(failedSummaryLines(summary), "; ")) } return nil } // failedSummaryLines filters a run summary down to its FAIL entries (for logs/errors). func failedSummaryLines(summary []string) []string { var failed []string for _, s := range summary { if strings.HasPrefix(s, "FAIL ") { failed = append(failed, s) } } return failed } // runVolumeDumps exports the Docker named-volume data of every deployed, unprotected stack whose // drive is writable — the class-B leg of the nightly app-data backup. (F3: DumpAppVolumesSafe // previously had NO production caller, so volume-dumps/ was never produced and the granular // restore had nothing to restore for named-volume apps.) Caller must hold the running flag. // // Gate ORDER is load-bearing: the volume check precedes DumpAppVolumesSafe, because the Safe // variant stops the stack before its own volume check — calling it unconditionally would bounce // every volume-less app on every nightly run. Per-stack isolation mirrors the DB loop: one app's // failure is recorded and does not abort the others. // // R-181 adds the reserve to that order, and for the SAME reason: it sits ahead of DumpAppVolumesSafe, // so a refused app is never stopped. A refusal decided inside the Safe variant would already have // bounced the app it was refusing to back up. It sits AFTER the volume-less check because an app with // no named volumes writes nothing in this leg — there is no first write here to gate, and consulting // the reserve for it would only decide a verdict early on a stale reading. func (m *Manager) runVolumeDumps() (summary []string, dumped int, allOK bool) { allOK = true if m.stackProvider == nil { return nil, 0, true } dump := m.dumpVolumesSafe if dump == nil { dump = m.DumpAppVolumesSafe } for _, stack := range m.stackProvider.ListDeployedStacks() { // Never stop/dump infra stacks (felhom-controller, traefik, cloudflared). if m.cfg != nil && m.cfg.IsProtectedStack(stack.Name) { continue } // Volume check FIRST — a volume-less stack must not be stopped at all (see gate-order note). if len(m.stackProvider.GetDockerVolumes(stack.Name)) == 0 { if m.isDebug() { m.logger.Printf("[DEBUG] [backup] %s has no named volumes — volume dump skipped", stack.Name) } continue } // Same drive-state skip guards as the DB-dump loop. drivePath := m.GetAppDrivePath(stack.Name) if m.settings != nil && m.settings.IsDisconnected(drivePath) { m.logger.Printf("[WARN] [backup] Skipping volume dump for %s — drive disconnected: %s", stack.Name, drivePath) summary = append(summary, fmt.Sprintf("SKIP %s volumes (drive disconnected)", stack.Name)) continue } if m.settings != nil && m.settings.IsDecommissioned(drivePath) { m.logger.Printf("[WARN] [backup] Skipping volume dump for %s — drive decommissioned: %s", stack.Name, drivePath) summary = append(summary, fmt.Sprintf("SKIP %s volumes (drive decommissioned)", stack.Name)) continue } // R-181: the reserve, ahead of DumpAppVolumesSafe so a refused app is NOT stopped. For an app // that already has a DB this is a memo lookup taken before its DB dump; for a volume-only app // this is where its verdict is taken, still before its first byte. m.noteAttempted(stack.Name) if !m.admitApp(stack.Name) { summary = append(summary, fmt.Sprintf("SKIP %s volumes (reserve — app backup refused)", stack.Name)) continue } if err := dump(stack.Name); err != nil { allOK = false summary = append(summary, fmt.Sprintf("FAIL %s volumes: %v", stack.Name, err)) m.noteFailure(stack.Name, "volume dump", err.Error()) m.logger.Printf("[ERROR] [backup] Volume dump failed for %s: %v", stack.Name, err) continue } dumped++ summary = append(summary, fmt.Sprintf("OK %s volumes", stack.Name)) } return summary, dumped, allOK } // DumpAppVolumes exports Docker named volumes to tar files for the given stack. // Tars are written to AppVolumeDumpPath(drivePath, stackName)/. // Uses "docker run alpine tar" (same pattern as appexport). func (m *Manager) DumpAppVolumes(stackName string) error { if m.stackProvider == nil { return nil } volumes := m.stackProvider.GetDockerVolumes(stackName) if len(volumes) == 0 { return nil } drivePath := m.GetAppDrivePath(stackName) if drivePath == "" { return fmt.Errorf("cannot determine drive path for %s", stackName) } dumpDir := AppVolumeDumpPath(m.namespaceRoot(drivePath), stackName) if err := os.MkdirAll(dumpDir, 0755); err != nil { return fmt.Errorf("creating volume dump dir: %w", err) } var dumpErrors []string for _, volName := range volumes { tarPath := filepath.Join(dumpDir, volName+".tar") // F7 (CAMPAIGN-3, HIGH): write the tar to a `.tar.tmp` sibling and only atomically rename it // over the restore point on success — the same crash-safe pattern the DB-dump path uses // (appbackup/dbdump.go DumpOne). Before this, tar wrote the `.tar` IN PLACE, so a mid-write NFS // cut left a 0-byte tar REPLACING the last good dump (tier-1 restore is replace-semantics → an // empty volume). Now a failed/interrupted write only ever touches the `.tmp`; the last good // `.tar` is untouched. The `.tmp` name (ends `.tmp`, not `.tar`) is invisible to the // restore-point/stale scans, so it is never mistaken for a restore point. tmpPath := tarPath + ".tmp" if m.isDebug() { m.logger.Printf("[DEBUG] [backup] Dumping volume %s for %s", volName, stackName) } out, err := m.tarVolumeOrDefault(volName, dumpDir) if err != nil { // Any tar error or context timeout (incl. a dead NFS target → EIO): remove ONLY the tmp, // leave the existing `.tar` restore point byte-untouched, WARN, continue. m.logger.Printf("[WARN] [backup] Volume dump failed for %s/%s (last good dump preserved): %s — %v", stackName, volName, strings.TrimSpace(string(out)), err) os.Remove(tmpPath) dumpErrors = append(dumpErrors, volName) continue } // fsync the tmp file (flush the tar to disk) then atomically rename over the restore point. if err := atomicPromoteTar(tmpPath, tarPath); err != nil { m.logger.Printf("[WARN] [backup] Volume dump promote failed for %s/%s (last good dump preserved): %v", stackName, volName, err) os.Remove(tmpPath) dumpErrors = append(dumpErrors, volName) continue } if info, _ := os.Stat(tarPath); info != nil { m.logger.Printf("[INFO] [backup] Volume dump: %s/%s → %s", stackName, volName, humanizeBytes(info.Size())) } } // Clean up tars (and any orphan `.tar.tmp` from a killed run) for volumes that no longer exist. entries, _ := os.ReadDir(dumpDir) activeVols := make(map[string]bool) for _, v := range volumes { activeVols[v+".tar"] = true } for _, e := range entries { name := e.Name() // A leftover `.tar.tmp` is never a restore point — always safe to remove (its `.tar` sibling, // if any, is the real restore point and is handled by the `.tar` branch). if strings.HasSuffix(name, ".tar.tmp") { os.Remove(filepath.Join(dumpDir, name)) continue } if !activeVols[name] && strings.HasSuffix(name, ".tar") { os.Remove(filepath.Join(dumpDir, name)) if m.isDebug() { m.logger.Printf("[DEBUG] [backup] Removed stale volume dump: %s/%s", stackName, name) } } } if len(dumpErrors) > 0 { return fmt.Errorf("volume dump failed for: %s", strings.Join(dumpErrors, ", ")) } return nil } // tarVolumeOrDefault runs the F7 tar seam (m.tarVolume) or, when unset, the real docker tar into the // `.tar.tmp` sibling under dumpDir. The 10-minute bound matches the original. func (m *Manager) tarVolumeOrDefault(volName, dumpDir string) ([]byte, error) { if m.tarVolume != nil { return m.tarVolume(volName, dumpDir) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, "docker", "run", "--rm", "-v", volName+":/vol:ro", "-v", dumpDir+":/out", "alpine", "tar", "cf", "/out/"+volName+".tar.tmp", "-C", "/vol", ".") return cmd.CombinedOutput() } // atomicPromoteTar fsyncs a completed `.tar.tmp` then atomically renames it over the final `.tar` // (same-dir rename = atomic on the target fs). It mirrors DumpOne's crash-safety (F7) and EXCEEDS it // by also best-effort fsync'ing the directory entry, so the rename itself survives a power loss — the // DB-dump path fsyncs the file but not the dir; a follow-up could add the dir fsync there too. On any // error the tmp is left for the caller to remove; the final `.tar` is never touched here except by a // successful rename. func atomicPromoteTar(tmpPath, finalPath string) error { // O_RDWR, not os.Open: fsync on a read-only handle is refused on Windows (dev-box test runs), // while a writable handle syncs on every platform. Content is not modified. f, err := os.OpenFile(tmpPath, os.O_RDWR, 0) if err != nil { return fmt.Errorf("opening tmp dump: %w", err) } if err := f.Sync(); err != nil { f.Close() return fmt.Errorf("syncing tmp dump: %w", err) } if err := f.Close(); err != nil { return fmt.Errorf("closing tmp dump: %w", err) } if err := os.Rename(tmpPath, finalPath); err != nil { return fmt.Errorf("renaming tmp dump: %w", err) } // Best-effort: fsync the directory so the rename is durable (ignore errors — the rename already // made the new content visible; this only hardens against a power loss immediately after). if dir, derr := os.Open(filepath.Dir(finalPath)); derr == nil { _ = dir.Sync() _ = dir.Close() } return nil } // DumpAppVolumesSafe stops the stack before dumping volumes and restarts after. // Prevents inconsistent tars of live database volumes (e.g. PostgreSQL). // Protected stacks that reject StopStack will return an error — callers handle as warning. // // R-166: the stop→dump→start window is marked. Before this, a controller killed between the stop // and the start left the app down with NOTHING on disk saying why or that it was owed a restart — // and a stopped app has zero containers, which the boot reconciler then read as a deliberate // customer stop and left alone. The marker is the mechanism, not the restart call below: a SIGKILL // runs no deferred function (Campaign 8 fault 10, on live hardware), so only something already // written to disk can survive it. func (m *Manager) DumpAppVolumesSafe(stackName string) error { if m.stackProvider == nil { return fmt.Errorf("no stack provider") } // Intent before the act: refuse to stop an app we cannot promise to restart. if err := m.appStop.Begin("volume-dump:"+stackName, ReasonVolumeDump, []string{stackName}); err != nil { return fmt.Errorf("could not record the app-stop marker for %s (refusing to stop it unprotected): %w", stackName, err) } m.logger.Printf("[INFO] [backup] Stopping %s for safe volume dump", stackName) if err := m.stackProvider.StopStack(stackName); err != nil { // Nothing was stopped, so nothing is owed a restart — clear rather than strand a marker that // would cost a spurious (if harmless) restart at the next startup. m.appStop.End() return fmt.Errorf("could not stop %s for volume dump: %w", stackName, err) } dumpErr := m.DumpAppVolumes(stackName) m.logger.Printf("[INFO] [backup] Restarting %s after volume dump", stackName) startErr := m.stackProvider.StartStack(stackName) if startErr != nil { m.logger.Printf("[ERROR] [backup] Failed to restart %s after volume dump: %v", stackName, startErr) } else { // Cleared ONLY on a restart that succeeded. A failed restart keeps the marker so the next // startup retries — the app really is still owed one. m.appStop.End() } // Surface both errors — callers must know if the app is left stopped if dumpErr != nil && startErr != nil { return fmt.Errorf("volume dump failed for %s: %v; restart also failed: %v", stackName, dumpErr, startErr) } if startErr != nil { return fmt.Errorf("volume dump OK but restart failed for %s: %w", stackName, startErr) } return dumpErr } // GetStatus returns the current DB-dump status. func (m *Manager) GetStatus() *DBDumpStatus { m.mu.Lock() defer m.mu.Unlock() return m.lastDBDump } // IsRunning returns whether a backup or restore is currently in progress. func (m *Manager) IsRunning() bool { m.mu.Lock() defer m.mu.Unlock() return m.running } // acquireRunning atomically sets the running flag. Returns error if already running. func (m *Manager) acquireRunning() error { m.mu.Lock() defer m.mu.Unlock() if m.running { return fmt.Errorf("backup already in progress") } m.running = true return nil } // releaseRunning clears the running flag. func (m *Manager) releaseRunning() { m.mu.Lock() m.running = false m.mu.Unlock() } // SetSecretGenerator wires the O4 resettable-secret generator used by RestoreFromRecoveryUnit // (init-only, same contract as SetStackProvider: call once during single-threaded startup). func (m *Manager) SetSecretGenerator(fn func(stackName, envVar string) (string, bool)) { m.generateSecret = fn } // SetStackProvider sets the stack data provider for app data discovery. // // M2: this MUST be called exactly once during single-threaded startup (main.go), // before the scheduler / HTTP server / any backup goroutine starts. That write // then happens-before all the (unlocked) reads of m.stackProvider, so no data // race exists. The earlier mutex on this write was misleading — it implied // runtime concurrency the reads don't honour; removed to make the init-only // contract explicit. Do NOT call this after startup. func (m *Manager) SetStackProvider(provider StackDataProvider) { m.stackProvider = provider } // GetStackHDDMounts returns HDD mount paths for the named stack via the stack provider. func (m *Manager) GetStackHDDMounts(name string) []string { if m.stackProvider == nil { return nil } return m.stackProvider.GetStackHDDMounts(name) } // DumpStackDB runs a database dump for containers belonging to a specific stack. // Dumps to the stack's home drive: /backups/primary//db-dumps/. func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error { dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()) if err != nil { return fmt.Errorf("database discovery failed: %w", err) } var stackDBs []DiscoveredDB for _, db := range dbs { if db.StackName == stackName { stackDBs = append(stackDBs, db) } } if len(stackDBs) == 0 { m.logger.Printf("[DEBUG] No databases found for stack %s — skipping pre-backup dump", stackName) return nil } drivePath := m.GetAppDrivePath(stackName) if drivePath == "" || !filepath.IsAbs(drivePath) { return fmt.Errorf("cannot determine absolute drive path for %s (systemDataPath not configured?)", stackName) } dumpDir := AppDBDumpPath(m.namespaceRoot(drivePath), stackName) m.logger.Printf("[INFO] [backup] Running pre-backup DB dump for %s (%d database(s)) → %s", stackName, len(stackDBs), dumpDir) for _, db := range stackDBs { result := DumpOne(ctx, db, dumpDir, m.logger, m.isDebug()) if result.Error != nil { return fmt.Errorf("DB dump failed for %s: %w", result.DB.ContainerName, result.Error) } m.logger.Printf("[INFO] [backup] Pre-backup DB dump OK: %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size)) // Persist validation to settings if m.settings != nil && result.FilePath != "" { filename := filepath.Base(result.FilePath) cache := settings.DBValidationCache{ ValidatedAt: time.Now().Format(time.RFC3339), TableCount: result.Validation.TableCount, HasHeader: result.Validation.Valid, Size: result.Validation.FileSize, ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339), } if !result.Validation.Valid { cache.Error = result.Validation.Error } _ = m.settings.SetDBValidation(filename, cache) } } return nil } // listAllDumpFiles scans per-drive per-stack DB dump directories. // // M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is // passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min // cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime — // so the write-back (and its settings.json disk write) only happens when a dump actually changed. func (m *Manager) listAllDumpFiles() []DumpFileInfo { modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) } var cacheSnapshot map[string]settings.DBValidationCache if m.settings != nil { cacheSnapshot = m.settings.GetDBValidations() } lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) { c, ok := cacheSnapshot[name] if !ok || c.Size != size || c.ModTime != modKey(mod) { return DumpValidation{}, false // cache miss / changed → validate } return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true } var allFiles []DumpFileInfo for drive, stacks := range m.groupStacksByDrive() { for _, stack := range stacks { dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name) files, err := ListDumpFiles(dumpDir, lookup) if err != nil { continue } for _, f := range files { // Write back only fresh validations (cache miss against the snapshot), so unchanged // dumps cause neither a re-validation nor a settings.json write each cycle. if m.settings != nil { if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) { _ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{ ValidatedAt: time.Now().Format(time.RFC3339), TableCount: f.Validation.TableCount, HasHeader: f.Validation.Valid, Error: f.Validation.Error, Size: f.Size, ModTime: modKey(f.ModTime), }) } } allFiles = append(allFiles, f) } } } m.logger.Printf("[INFO] [backup] Found %d DB dump files across drives", len(allFiles)) return allFiles } // RefreshCache updates the cached full status. Called by scheduler every 5 minutes. func (m *Manager) RefreshCache(nextDBDump time.Time) { status := &FullBackupStatus{ Enabled: m.cfg.Backup.Enabled, DBDumpSchedule: m.cfg.Backup.DBDumpSchedule, NextDBDump: nextDBDump, } // Scan dump files from per-drive per-stack paths files := m.listAllDumpFiles() status.DumpFiles = files ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()); err == nil { status.DiscoveredDBs = dbs } // Discover app data — all deployed stacks, backup is mandatory if m.stackProvider != nil { status.AppDataInfo = DiscoverAppData(m.stackProvider, status.DiscoveredDBs) // Phase 2: keep each app's recovery unit current with its definition. Idempotent // (checksum-skip), so this periodic refresh only writes when the config actually changed, // and ensures units exist shortly after startup without waiting for the daily DB dump. // // R-182: this sweep gets its OWN digest scope. It has to, and the reason is the whole // balance of this change. The per-app event is now record-only, so without a digest here a // capture failure detected between runs would be recorded and NEVER notified — a new // silence introduced while closing one. But this path can fire on every status poll, so its // digest deliberately carries NO run id: the hub's ordinary 1-hour operator cooldown then // applies, which caps it at one mail an hour exactly as before, while the mail now lists // EVERY failing app instead of whichever one happened to be first. func() { defer m.beginRunSummary(runKindRefresh, "")() defer m.emitRunSummary() m.captureAllRecoveryUnits() }() } // Fill in dynamic fields under lock. m.mu.Lock() status.Running = m.running status.LastDBDump = m.lastDBDump // Cross-check lastDBDump results inside lock to prevent torn writes. if m.lastDBDump != nil && len(files) > 0 { fileValidation := make(map[string]DumpValidation) // keyed by filename for _, f := range files { fileValidation[f.FileName] = f.Validation } for i, r := range m.lastDBDump.Results { if !r.Validation.Valid && r.Validation.Error == "" && r.FilePath != "" { filename := filepath.Base(r.FilePath) if fv, ok := fileValidation[filename]; ok { m.lastDBDump.Results[i].Validation = fv m.logger.Printf("[INFO] [backup] Re-validated %s from disk: valid=%v tables=%d", filename, fv.Valid, fv.TableCount) } } } } m.cachedStatus = status m.cacheTime = time.Now() m.mu.Unlock() m.logger.Printf("[INFO] [backup] Backup status cache refreshed") } // GetFullStatus returns the cached backup status for page rendering. // Returns instantly — no subprocess calls. // Returns a deep copy so callers can safely append to slice fields without // polluting the cache. func (m *Manager) GetFullStatus(nextDBDump time.Time) *FullBackupStatus { m.mu.Lock() defer m.mu.Unlock() if m.cachedStatus != nil { status := *m.cachedStatus status.AppDataInfo = make([]AppBackupInfo, len(m.cachedStatus.AppDataInfo)) copy(status.AppDataInfo, m.cachedStatus.AppDataInfo) // Update dynamic fields that don't need subprocess calls status.Running = m.running status.NextDBDump = nextDBDump status.SingleCopyWarning = m.singleCopyWarning() // F6: honest single-drive signal // Deep-copy lastDBDump so callers cannot mutate shared state. if m.lastDBDump != nil { copyDump := *m.lastDBDump if len(m.lastDBDump.Results) > 0 { copyDump.Results = make([]DumpResult, len(m.lastDBDump.Results)) copy(copyDump.Results, m.lastDBDump.Results) } status.LastDBDump = ©Dump } // Synthesize LastDBDump from DumpFiles on disk if not in memory if status.LastDBDump == nil && len(status.DumpFiles) > 0 { var results []DumpResult var latestTime time.Time for _, f := range status.DumpFiles { results = append(results, DumpResult{ DB: DiscoveredDB{StackName: f.StackName, DBType: f.DBType, ContainerName: f.StackName}, FilePath: f.FileName, Size: f.Size, Validation: f.Validation, }) if f.ModTime.After(latestTime) { latestTime = f.ModTime } } status.LastDBDump = &DBDumpStatus{ LastRun: latestTime, Results: results, Success: true, } } return &status } // No cache yet — return a minimal status (first page load before cache is populated) status := &FullBackupStatus{ Enabled: m.cfg.Backup.Enabled, Running: m.running, DBDumpSchedule: m.cfg.Backup.DBDumpSchedule, NextDBDump: nextDBDump, SingleCopyWarning: m.singleCopyWarning(), // F6 } if m.lastDBDump != nil { copyDump := *m.lastDBDump if len(m.lastDBDump.Results) > 0 { copyDump.Results = make([]DumpResult, len(m.lastDBDump.Results)) copy(copyDump.Results, m.lastDBDump.Results) } status.LastDBDump = ©Dump } return status } // sameDevice reports whether two paths sit on the same physical device, through the test seam when // one is installed. Nil seam → system.SamePhysicalDevice, i.e. byte-for-byte the previous behaviour. func (m *Manager) sameDevice(a, b string) bool { if m.samePhysicalDevice != nil { return m.samePhysicalDevice(a, b) } return system.SamePhysicalDevice(a, b) } // hasOffDriveTarget reports whether any registered, schedulable storage path lives on a physical disk // OTHER than the system drive — i.e. whether a genuine off-drive (tier-2) copy is possible at all. // When false the box is single-drive: tier-1 is the ONLY local copy and 3-2-1 needs a 2nd drive or // offsite (F6 — surfaced honestly via SingleCopyWarning, never a faked guarantee). func (m *Manager) hasOffDriveTarget() bool { if m.settings == nil || m.systemDataPath == "" { return false } for _, sp := range m.settings.GetSchedulableStoragePaths() { if sp.Path == m.systemDataPath || m.sameDevice(m.systemDataPath, sp.Path) { continue } return true } return false } // singleCopyWarning returns the honest single-drive notice, or "" when an off-drive target exists. func (m *Manager) singleCopyWarning() string { if m.hasOffDriveTarget() { return "" } return singleCopyNotice } // sysDriveLabelFor returns the drive label for a stack's tier-1 restore point — the clear // system-drive label for a sys_drive (volume-only) app (F6: never blank), else the enrolled drive's // storage label. func (m *Manager) sysDriveLabelFor(stackName string) string { drive := m.GetAppDrivePath(stackName) if drive == "" { return "" } if drive == m.systemDataPath { return systemDriveLabel } if m.settings != nil { return m.settings.GetStorageLabel(drive) } return "" } // pruneStalePrimaryDirs removes orphaned `backups/primary/` dirs left on a drive after an app's // HDD_PATH moved to another drive (F5, CAMPAIGN-3 — pure disk residue, invisible in the snapshot // list). LOAD-BEARING GUARDS: a dir is removed ONLY when is currently deployed AND its current // namespace root differs from this dir's drive. It NEVER removes the dir on the app's CURRENT drive // (that IS the live restore point), and NEVER removes a dir for an app NOT in the deployed set (an // undeployed app's last backup is still its restore point — orphaned-app cleanup is a separate, // user-driven concern). Only operates strictly under a `backups/primary/` prefix. func (m *Manager) pruneStalePrimaryDirs() { if m.stackProvider == nil { return } // Current namespace root per DEPLOYED app. current := map[string]string{} for _, s := range m.stackProvider.ListDeployedStacks() { if drive := m.GetAppDrivePath(s.Name); drive != "" { current[s.Name] = filepath.Clean(m.namespaceRoot(drive)) } } // Candidate drives to scan: the system drive + every registered storage path. var nsRoots []string if m.systemDataPath != "" { nsRoots = append(nsRoots, filepath.Clean(m.namespaceRoot(m.systemDataPath))) } if m.settings != nil { for _, sp := range m.settings.GetStoragePaths() { nsRoots = append(nsRoots, filepath.Clean(NamespaceRoot(sp.Path, true))) } } seen := map[string]bool{} for _, nsRoot := range nsRoots { if seen[nsRoot] { continue } seen[nsRoot] = true primaryDir := PrimaryBackupPath(nsRoot) entries, err := os.ReadDir(primaryDir) if err != nil { continue // absent/unreadable (e.g. a disconnected drive) — nothing to prune here } for _, e := range entries { if !e.IsDir() { continue } app := e.Name() cur, deployed := current[app] if !deployed { continue // GUARD: an undeployed app's last backup is still its restore point } if cur == nsRoot { continue // GUARD: this IS the app's current drive — the live restore point } stalePath := RecoveryUnitPath(nsRoot, app) // Prefix safety: only ever remove strictly inside `backups/primary/` (no surprise user data). if !strings.HasPrefix(filepath.Clean(stalePath)+string(filepath.Separator), filepath.Clean(primaryDir)+string(filepath.Separator)) { continue } if err := os.RemoveAll(stalePath); err != nil { m.logger.Printf("[WARN] [backup] F5: could not remove stale primary dir for %s on old drive: %v", app, err) } else { m.logger.Printf("[INFO] [backup] F5: removed stale primary backup dir for %s on an old drive (app now on %s)", app, cur) } } } } // isDebug returns true if logging level is "debug". func (m *Manager) isDebug() bool { return m.cfg != nil && m.cfg.Logging.Level == "debug" } func dbNames(dbs []DiscoveredDB) string { var names []string for _, db := range dbs { names = append(names, fmt.Sprintf("%s(%s)", db.ContainerName, db.DBType)) } return strings.Join(names, ", ") }