package backup import ( "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" "gitea.dooplex.hu/admin/felhom-controller/internal/system" ) // Tier 2 = an off-drive (different physical disk) copy of an app's recovery unit + its CLASS-DRIVEN // capture legs (Task 3b, architecture §2/§8). For a classified app the legs are the Task-3-core // TierSecondary set — per-bind mandatory + optional HDD/userdata paths (so a classified app's copy // tracks exactly what the catalog classed non-excluded; paperless's copy legitimately shrinks as // `export` drops out). For a legacy app the legs are the resolver appdata dir(s), byte-identical to // v0.134.x (the SQ5 footprint promise), mapped into the same layout. The dest is the v2 // relpath-mirroring layout (hdd/, userdata/, recovery-unit/, .felhom-tier2-layout marker) — // N>1 dirs + nested binds represented natively (the old flat-appdata N>1 refusal is gone), migration // is delete-and-rebuild of the derived copy, and a reconcile step prunes dest dirs a bind no longer // covers. Target auto-pick: another registered NON-NETWORK user-data drive (holds the full set), else // the internal SSD for the STATE-ONLY set (unit + mandatory), rootfs-headroom-guarded. NETWORK (NAS) // storage is never a target (F-6C-1: rsync -og under root_squash → silently-wrong-owner restore). No // off-drive target ⇒ an honest recorded status, not a silent no-op. const gibibyte = 1024 * 1024 * 1024 var ( errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)") errSSDNoHeadroom = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup") // errTier2NetworkOnly (F-6C-1): the only off-disk candidate is a NAS network share, which can never // be a tier-2 target — rsync -og chowns under a root_squash export and a wrong-owner restore is // silently broken. The message IS the customer-facing reason. errTier2NetworkOnly = errors.New("Hálózati tároló nem lehet a 2. mentés célja — a fájltulajdonos-adatok megőrzése ott nem garantálható.") ) // appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD // binds, via the stack provider (nil provider → legacy [stackName] fallback). See // appbackup.AppDataDirNames. func (m *Manager) appDataDirNames(stackName, hddPath string) []string { var mounts []string if m.stackProvider != nil { mounts = m.stackProvider.GetStackHDDMounts(stackName) } return AppDataDirNames(hddPath, stackName, mounts) } // tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the // WARN-on-missing-declared-dir rule; nil provider → false). func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool { if m.stackProvider == nil { return false } return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName)) } // Tier2Target is a resolved off-drive destination for an app's Tier 2 copy. type Tier2Target struct { NamespaceRoot string // felhom-data namespace root on the target drive Label string // human label (UI) IsSystemDrive bool // target is the internal SSD/system drive (DB/config only) StateOnly bool // §2.2: this target carries unit + MANDATORY only (optional legs skipped) — the SSD tier Reason string // why this target (Hungarian, for UI/logs) } // tier2FitsHeadroom reports whether a unit of unitGB fits on a system/rootfs drive while leaving a // reserve free. Reserve = max(2 GB, 20% of total) — this is what protects the small (~8 GB) guest // rootfs from being filled by a Tier 2 copy. Pure function (unit-tested). func tier2FitsHeadroom(availGB, totalGB, unitGB float64) bool { reserve := totalGB * 0.20 if reserve < 2.0 { reserve = 2.0 } return (availGB - unitGB) >= reserve } // selectTier2Target picks the off-drive destination for an app's Tier 2 copy. A customer-pinned // target wins when still valid; otherwise it auto-picks: another user-data drive (holds the FULL set), // else the internal SSD for the STATE-ONLY set (unit + mandatory), headroom-guarded. NETWORK storage // (NAS) is never a valid target at any step — pinned or auto (F-6C-1: rsync -og under root_squash → // silently-wrong-owner restore). fullSize sizes the real-drive path; stateOnlySize sizes the SSD path. func (m *Manager) selectTier2Target(stackName string, fullSize, stateOnlySize int64) (*Tier2Target, error) { sourceDrive := m.GetAppDrivePath(stackName) if sourceDrive == "" { return nil, fmt.Errorf("no source drive for %s", stackName) } sawNetworkCandidate := false // an off-disk candidate existed but was network-only → better reason // 0. Honor a customer-pinned target if it is still valid (registered, schedulable, off-disk, // NON-network). An invalid pin (gone / same disk / network) silently falls through to auto. if m.settings != nil { if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.PreferredTarget != "" { for _, sp := range m.settings.GetSchedulableStoragePaths() { if sp.Path != cd.PreferredTarget { continue } if sp.IsNetwork() { m.logger.Printf("[INFO] [backup] Tier 2 %s: pinned target %s is network storage — invalid (F-6C-1); falling through to auto", stackName, sp.Path) sawNetworkCandidate = true break } if sp.Path == sourceDrive || system.SamePhysicalDevice(sourceDrive, sp.Path) { break // pinned target is on the same physical disk — not off-drive; fall through } label := sp.Label if label == "" { label = filepath.Base(sp.Path) } return &Tier2Target{ NamespaceRoot: NamespaceRoot(sp.Path, true), Label: label, Reason: "kézi választás", }, nil } } } // 1. Prefer another registered user-data drive on a DIFFERENT physical disk, NON-network. if m.settings != nil { for _, sp := range m.settings.GetSchedulableStoragePaths() { if sp.Path == sourceDrive || system.SamePhysicalDevice(sourceDrive, sp.Path) { continue } if sp.IsNetwork() { sawNetworkCandidate = true // F-6C-1: never a tier-2 target continue } label := sp.Label if label == "" { label = filepath.Base(sp.Path) } return &Tier2Target{ NamespaceRoot: NamespaceRoot(sp.Path, true), // Model A: in-guest mount IS the namespace root Label: label, Reason: "másik adatmeghajtó", }, nil } } // 2. Fall back to the internal SSD (system data path) — STATE-ONLY set only. sys := m.systemDataPath if sys == "" || system.SamePhysicalDevice(sourceDrive, sys) { if sawNetworkCandidate { return nil, errTier2NetworkOnly // the only off-disk candidate was a NAS } return nil, errNoOffDiskTarget // single drive / app already on the system disk } fits := m.tier2FitsSystemDrive if m.tier2SSDFits != nil { fits = m.tier2SSDFits } if !fits(sys, stateOnlySize) { return nil, errSSDNoHeadroom // would fill the ~8 GB rootfs — refuse, don't fill } return &Tier2Target{ NamespaceRoot: NamespaceRoot(sys, false), // system path is a real root → felhom-data appended Label: "belső SSD (rendszer)", IsSystemDrive: true, StateOnly: true, Reason: "nincs 2. adatmeghajtó — csak az adatbázis/konfiguráció fér a belső SSD-re; a nagy fájlokhoz 2. meghajtó kell", }, nil } // tier2FitsSystemDrive checks the size-aware rootfs-headroom guard for the SSD target. func (m *Manager) tier2FitsSystemDrive(sys string, unitSizeBytes int64) bool { di := system.GetDiskUsage(sys) if di == nil { return false // can't determine free space → refuse (fail-closed for the rootfs) } return tier2FitsHeadroom(di.AvailGB, di.TotalGB, float64(unitSizeBytes)/gibibyte) } // Tier-2 v2 layout (Task 3b, architecture §8): backups/secondary// holds // .felhom-tier2-layout (marker file, content "2" — written LAST, after all legs + reconcile) // recovery-unit/ (the unit leg, layout-identical to v1) // hdd// (per-bind HDD legs, relpath-mirroring) // userdata// (per-bind USERDATA legs) // Relpath-mirroring represents N>1 dirs + nested binds natively (the v1 flat-appdata N>1 refusal is // lifted structurally) and makes restore position-derivable (dest relpath → live path under the app's // current HDD_PATH). The whole tree is DERIVED from live data — migration is delete-and-rebuild. const ( tier2LayoutMarker = ".felhom-tier2-layout" tier2LayoutVersion = "2" ) // tier2SafeRemove os.RemoveAll's target ONLY if it is strictly WITHIN a backups/secondary/<...> // destBase (§9.2 destructive-write boundary — defense in depth against a mapping bug ever pointing a // removal at live data; red-proofed). Never removes destBase itself. func tier2SafeRemove(destBase, target string) error { cb := filepath.Clean(destBase) ct := filepath.Clean(target) if !strings.Contains(filepath.ToSlash(cb), "/backups/secondary/") { return fmt.Errorf("refusing tier-2 removal: destBase %q is not under backups/secondary/", cb) } if !strings.HasPrefix(ct, cb+string(filepath.Separator)) { return fmt.Errorf("refusing tier-2 removal: %q is not strictly within %q", ct, cb) } return os.RemoveAll(ct) } // tier2RelClass classifies a dest dir (relpath dirRel, slash-form) against the current leg relpaths: // keepInside = the dir is a leg or inside one (keep, don't descend); keepAncestor = on the path to a // leg (keep, descend to find stale deeper); stale = neither (remove the topmost). type tier2RelClass int const ( tier2Stale tier2RelClass = iota tier2KeepInside tier2KeepAncestor ) func classifyTier2Rel(dirRel string, legRels []string) tier2RelClass { for _, lr := range legRels { if dirRel == lr || strings.HasPrefix(dirRel, lr+"/") { return tier2KeepInside // exact leg or descendant content } } for _, lr := range legRels { if strings.HasPrefix(lr, dirRel+"/") { return tier2KeepAncestor // on the path to a deeper leg } } return tier2Stale } // tier2Reconcile removes dest subdirs under hdd/ and userdata/ that are neither an ancestor nor a // descendant of any current leg relpath (§7-D — the deferred-pruning answer: a bind removed/re-classed // stops occupying the secondary drive within one run). Runs strictly inside destBase. func (m *Manager) tier2Reconcile(destBase string, legRels []string) { var walk func(dirAbs, dirRel string) walk = func(dirAbs, dirRel string) { entries, err := os.ReadDir(dirAbs) if err != nil { return } for _, e := range entries { if !e.IsDir() { continue } childRel := dirRel + "/" + e.Name() childAbs := filepath.Join(dirAbs, e.Name()) switch classifyTier2Rel(childRel, legRels) { case tier2KeepInside: // mirrored leg content — keep, no descent case tier2KeepAncestor: walk(childAbs, childRel) default: if err := tier2SafeRemove(destBase, childAbs); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 reconcile: %v", err) } else { m.logger.Printf("[INFO] [backup] Tier 2 reconcile: removed stale dest dir %s", childRel) } } } } for _, root := range []string{"hdd", "userdata"} { walk(filepath.Join(destBase, root), root) } } // RunTier2 makes/refreshes the off-drive v2 copy of a single app's recovery unit + its class-driven // capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on // an actual copy failure (no valid target is a recorded status, not an error). func (m *Manager) RunTier2(stackName string) error { // Customer turned Tier 2 off for this app (config panel) — skip without touching status. if m.settings != nil { if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled { m.logger.Printf("[INFO] [backup] Tier 2 for %s skipped — disabled by customer", stackName) return nil } } sourceDrive := m.GetAppDrivePath(stackName) if sourceDrive == "" { return fmt.Errorf("no source drive for %s", stackName) } sourceNsRoot := m.namespaceRoot(sourceDrive) unitDir := RecoveryUnitPath(sourceNsRoot, stackName) if _, err := os.Stat(unitDir); err != nil { return nil // no recovery unit yet — nothing to copy } // Class-driven legs (classified) / resolver legs (legacy) + loud gap warnings (§7-H). legs, warns := m.tier2CaptureSet(stackName, sourceNsRoot) // Two sizes: full = unit + all legs; state-only = unit + mandatory legs (the SSD ceiling). unitSize := dirSizeBytes(unitDir) var fullSize, stateOnlySize int64 = unitSize, unitSize for _, lg := range legs { sz := dirSizeBytes(lg.Src) fullSize += sz if lg.Class == ClassMandatory { stateOnlySize += sz } } target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize) if err != nil { reason := tier2NoTargetReason(err) m.recordTier2NoTarget(stackName, reason) m.logger.Printf("[INFO] [backup] Tier 2 for %s: no off-drive target — %s", stackName, reason) return nil } // Defense-in-depth off-drive guard (selection already enforced it). if system.SamePhysicalDevice(sourceDrive, target.NamespaceRoot) { m.recordTier2NoTarget(stackName, "a kiválasztott cél ugyanazon a fizikai lemezen van") return nil } // §2.2: the SSD is a STATE-ONLY tier — drop optional legs, tell the customer honestly. if target.StateOnly { kept := legs[:0] droppedOptional := false for _, lg := range legs { if lg.Class == ClassMandatory { kept = append(kept, lg) } else { droppedOptional = true } } legs = kept if droppedOptional { warns = append(warns, "A belső SSD-n csak a konfiguráció, adatbázis és a kötelező adatok férnek el — a választható tartalom nem került másolásra.") } } destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName) start := time.Now() mirror := m.tier2Mirror if mirror == nil { mirror = rsyncMirror } // Migration = delete-and-rebuild (marker LAST). First v2 run (marker absent): remove the old flat // appdata/ leg (recovery-unit/ is layout-identical — untouched). A half-migrated dest self-heals on // the next run because the marker is only written after every leg + reconcile succeed. markerPath := filepath.Join(destBase, tier2LayoutMarker) if _, mErr := os.Stat(markerPath); mErr != nil { if rmErr := tier2SafeRemove(destBase, filepath.Join(destBase, "appdata")); rmErr != nil && !os.IsNotExist(rmErr) { m.logger.Printf("[WARN] [backup] Tier 2 %s: migration cleanup of old flat appdata failed: %v", stackName, rmErr) } } // Unit leg (always). if err := mirror(unitDir, filepath.Join(destBase, "recovery-unit")); err != nil { m.recordTier2Failure(stackName, target, err) if m.tier2Notify != nil { m.tier2Notify(stackName, target.Label, time.Since(start), err) } return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err) } // Capture legs. legRels := make([]string, 0, len(legs)) mirroredSize := unitSize for _, lg := range legs { if err := mirror(lg.Src, filepath.Join(destBase, filepath.FromSlash(lg.DestRel))); err != nil { m.recordTier2Failure(stackName, target, err) if m.tier2Notify != nil { m.tier2Notify(stackName, target.Label, time.Since(start), err) } return fmt.Errorf("tier2 rsync leg %s for %s: %w", lg.DestRel, stackName, err) } legRels = append(legRels, lg.DestRel) mirroredSize += dirSizeBytes(lg.Src) } // Reconcile stale dest dirs (a bind removed / re-classed excluded), then write the marker LAST. m.tier2Reconcile(destBase, legRels) if err := os.WriteFile(markerPath, []byte(tier2LayoutVersion), 0644); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 %s: layout marker write failed (restore will refuse until next run): %v", stackName, err) } dur := time.Since(start) m.recordTier2Success(stackName, target, mirroredSize, strings.Join(warns, " "), dur) if m.tier2Notify != nil { m.tier2Notify(stackName, target.Label, dur, nil) } m.logger.Printf("[INFO] [backup] Tier 2 copied %s → %s (%s, %d leg(s), %s)%s", stackName, destBase, humanizeBytes(mirroredSize), len(legs), dur.Round(time.Second), map[bool]string{true: " [SSD: state-only]", false: ""}[target.StateOnly]) return nil } // RunAllTier2 runs Tier 2 for every deployed HDD app (apps whose data lives on an external drive — // non-HDD apps live on the rootfs and are already inside the PBS whole-guest snapshot). func (m *Manager) RunAllTier2() { if m.stackProvider == nil { return } if m.migrationActive() { m.logger.Printf("[INFO] [backup] Tier 2 kihagyva: migráció folyamatban") return } var n int for _, stack := range m.stackProvider.ListDeployedStacks() { // F6 (CAMPAIGN-3): volume-only apps (no HDD_PATH, backups on sys_drive) previously got NO // tier-2 copy — a single controller-level copy on one device. They now flow through too: their // recovery unit (which holds the db/volume dumps) gets a cross-drive second copy like any HDD // app. Apps with no recovery unit yet (infra/never-backed-up) are a cheap no-op inside RunTier2 // (the `os.Stat(unitDir)` guard), so this doesn't spuriously copy protected stacks. if m.settings != nil && (m.settings.IsDisconnected(m.GetAppDrivePath(stack.Name)) || m.settings.IsDecommissioned(m.GetAppDrivePath(stack.Name))) { continue } runOne := m.perAppTier2 if runOne == nil { runOne = m.RunTier2 } if err := runOne(stack.Name); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 failed for %s: %v", stack.Name, err) } n++ } m.logger.Printf("[INFO] [backup] Tier 2 run complete: %d app(s) processed (incl. volume-only — F6)", n) } // --- per-app config-panel view (drives the Tier-2 "Beállítás" page) --- // Tier2Option is one selectable off-drive destination in the config panel. type Tier2Option struct { Path string // registered storage path (the value persisted as PreferredTarget) Label string // human label for the dropdown } // Tier2Info is the per-app Tier-2 view the config panel renders. It exposes the effective target // (pinned or auto), whether that is the size-limited internal SSD, the honest no-target reason, and // the off-disk drives the customer may pin — so the control is meaningful even with a single target. type Tier2Info struct { IsHDDApp bool // false = the app lives on the rootfs (already inside the PBS whole-guest snapshot) SourceDrive string // where the app's data currently lives Disabled bool // customer turned Tier 2 off Preferred string // customer-pinned target path ("" = automatic) EffectiveLabel string // label of the target that WOULD be used right now EffectiveIsSSD bool // the effective target is the internal SSD (DB/config only) EffectiveDesc string // why this target (Hungarian) NoTarget bool // no off-drive target fits at all NoTargetReason string // honest reason when NoTarget Alternatives []Tier2Option } // Tier2Info builds the config-panel view for one app. Read-only (no status writes). func (m *Manager) Tier2Info(stackName string) Tier2Info { var info Tier2Info if m.stackProvider != nil { info.IsHDDApp = m.stackProvider.GetStackHDDPath(stackName) != "" } source := m.GetAppDrivePath(stackName) info.SourceDrive = source if m.settings != nil { if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil { info.Disabled = cd.UserDisabled info.Preferred = cd.PreferredTarget } // Eligible alternative drives: registered, schedulable, on a DIFFERENT physical disk. for _, sp := range m.settings.GetSchedulableStoragePaths() { if sp.Path == source || system.SamePhysicalDevice(source, sp.Path) { continue } label := sp.Label if label == "" { label = filepath.Base(sp.Path) } info.Alternatives = append(info.Alternatives, Tier2Option{Path: sp.Path, Label: label}) } } // Resolve what the runner WOULD pick right now: the v2 capture legs feed both sizes (the SSD // headroom guard tests the state-only set). sourceNsRoot := m.namespaceRoot(source) legs, _ := m.tier2CaptureSet(stackName, sourceNsRoot) unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName)) fullSize, stateOnlySize := unitSize, unitSize for _, lg := range legs { sz := dirSizeBytes(lg.Src) fullSize += sz if lg.Class == ClassMandatory { stateOnlySize += sz } } target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize) if err != nil { info.NoTarget = true info.NoTargetReason = tier2NoTargetReason(err) return info } info.EffectiveLabel = target.Label info.EffectiveIsSSD = target.IsSystemDrive info.EffectiveDesc = target.Reason return info } // --- status persistence (drives the "2. mentés" UI card) --- // withTier2Prefs carries the customer-preference fields (UserDisabled/PreferredTarget) from any // existing config into a freshly-built status struct, so a runner status write never clobbers them. func (m *Manager) withTier2Prefs(stackName string, cfg *settings.CrossDriveBackup) *settings.CrossDriveBackup { if m.settings != nil { if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil { cfg.UserDisabled = existing.UserDisabled cfg.PreferredTarget = existing.PreferredTarget } } return cfg } func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) { if m.settings == nil { return } if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{ Enabled: true, Method: "rsync", DestinationPath: target.NamespaceRoot, Schedule: "daily", LastRun: time.Now().Format(time.RFC3339), LastStatus: "ok", LastWarning: strings.TrimSpace(warning), LastDuration: dur.Round(time.Second).String(), LastSizeHuman: humanizeBytes(sizeBytes), })); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 status persist (ok) for %s failed: %v", stackName, err) } } func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) { if m.settings == nil { return } if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{ Enabled: true, Method: "rsync", DestinationPath: target.NamespaceRoot, Schedule: "daily", LastRun: time.Now().Format(time.RFC3339), LastStatus: "error", LastError: cause.Error(), })); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 status persist (error) for %s failed: %v", stackName, err) } } func (m *Manager) recordTier2NoTarget(stackName, reason string) { if m.settings == nil { return } if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{ Enabled: false, Method: "rsync", Schedule: "daily", LastStatus: "no_target", LastError: reason, })); err != nil { m.logger.Printf("[WARN] [backup] Tier 2 status persist (no_target) for %s failed: %v", stackName, err) } } func tier2NoTargetReason(err error) string { switch { case errors.Is(err, errSSDNoHeadroom): return "nincs elég hely a belső SSD-n — a nagy fájlok off-drive mentéséhez 2. meghajtó (vagy távoli tárhely) szükséges" case errors.Is(err, errNoOffDiskTarget): return "nincs másik fizikai meghajtó — a 2. mentéshez 2. meghajtó szükséges" default: return err.Error() } } // --- helpers --- // rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning). func rsyncMirror(src, dst string) error { if err := os.MkdirAll(dst, 0755); err != nil { return fmt.Errorf("mkdir %s: %w", dst, err) } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() // Trailing slashes: copy the CONTENTS of src into dst. cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/") out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) } return nil } // dirSizeBytes returns the total size of a directory via `du -sb` (0 if absent/error). func dirSizeBytes(dir string) int64 { if _, err := os.Stat(dir); err != nil { return 0 } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() out, err := exec.CommandContext(ctx, "du", "-sb", dir).Output() if err != nil { return 0 } fields := strings.Fields(string(out)) if len(fields) == 0 { return 0 } var size int64 if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil { return 0 } return size }