9f436c8a3b
F7/R-53: app_export.html built the app's public URL as '<sub>.{{$.CSRFToken}}',
so the "Megnyitás" link was wrong for every app with a subdomain and a session
CSRF token was written into a URL. Template now uses {{$.Domain}}, and
exportPageHandler supplies the key — it builds its own data map instead of
going through baseData, which is where every other page gets it. The page's
real CSRF path (csrfH() reading the meta tag) is correct and untouched.
The 7 red internal/backup tests are green again, with no behaviour change.
TestTier2V2_* / TestSharesTier2* all failed for one environmental reason:
Tier-2's off-drive guard asks system.SamePhysicalDevice (st_dev equality)
whether a target is really a second disk, and every t.TempDir() here shares one
filesystem — so the guard correctly refused the fixture's "two drives" and the
tests never reached their subject ("nincs másik fizikai meghajtó").
Seam in the package's existing style: a nil-defaulted Manager.samePhysicalDevice
field + sameDevice wrapper, seven call sites routed through it. Nil resolves to
system.SamePhysicalDevice, so production is byte-for-byte unchanged; only the two
fixtures inject a fake modelling one drive per directory subtree. No assertion
weakened, nothing skipped/renamed/deleted; all 7 mutation-proved.
Also: the ssh->pct-exec ASCII-grep and heredoc-credential traps are now in
CLAUDE.md's live-validation section.
644 lines
26 KiB
Go
644 lines
26 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
|
)
|
|
|
|
// Tier 2 = an off-drive (different physical disk) copy of an app's recovery unit + its CLASS-DRIVEN
|
|
// capture legs (Task 3b, architecture §2/§8). For a classified app the legs are the Task-3-core
|
|
// TierSecondary set — per-bind mandatory + optional HDD/userdata paths (so a classified app's copy
|
|
// tracks exactly what the catalog classed non-excluded; paperless's copy legitimately shrinks as
|
|
// `export` drops out). For a legacy app the legs are the resolver appdata dir(s), byte-identical to
|
|
// v0.134.x (the SQ5 footprint promise), mapped into the same layout. The dest is the v2
|
|
// relpath-mirroring layout (hdd/<rel>, userdata/<rel>, recovery-unit/, .felhom-tier2-layout marker) —
|
|
// N>1 dirs + nested binds represented natively (the old flat-appdata N>1 refusal is gone), migration
|
|
// is delete-and-rebuild of the derived copy, and a reconcile step prunes dest dirs a bind no longer
|
|
// covers. Target auto-pick: another registered NON-NETWORK user-data drive (holds the full set), else
|
|
// the internal SSD for the STATE-ONLY set (unit + mandatory), rootfs-headroom-guarded. NETWORK (NAS)
|
|
// storage is never a target (F-6C-1: rsync -og under root_squash → silently-wrong-owner restore). No
|
|
// off-drive target ⇒ an honest recorded status, not a silent no-op.
|
|
|
|
const gibibyte = 1024 * 1024 * 1024
|
|
|
|
var (
|
|
errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)")
|
|
errSSDNoHeadroom = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup")
|
|
// errTier2NetworkOnly (F-6C-1): the only off-disk candidate is a NAS network share, which can never
|
|
// be a tier-2 target — rsync -og chowns under a root_squash export and a wrong-owner restore is
|
|
// silently broken. The message IS the customer-facing reason.
|
|
errTier2NetworkOnly = errors.New("Hálózati tároló nem lehet a 2. mentés célja — a fájltulajdonos-adatok megőrzése ott nem garantálható.")
|
|
)
|
|
|
|
// appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD
|
|
// binds, via the stack provider (nil provider → legacy [stackName] fallback). See
|
|
// appbackup.AppDataDirNames.
|
|
func (m *Manager) appDataDirNames(stackName, hddPath string) []string {
|
|
var mounts []string
|
|
if m.stackProvider != nil {
|
|
mounts = m.stackProvider.GetStackHDDMounts(stackName)
|
|
}
|
|
return AppDataDirNames(hddPath, stackName, mounts)
|
|
}
|
|
|
|
// tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the
|
|
// WARN-on-missing-declared-dir rule; nil provider → false).
|
|
func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool {
|
|
if m.stackProvider == nil {
|
|
return false
|
|
}
|
|
return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName))
|
|
}
|
|
|
|
// Tier2Target is a resolved off-drive destination for an app's Tier 2 copy.
|
|
type Tier2Target struct {
|
|
NamespaceRoot string // felhom-data namespace root on the target drive
|
|
Label string // human label (UI)
|
|
IsSystemDrive bool // target is the internal SSD/system drive (DB/config only)
|
|
StateOnly bool // §2.2: this target carries unit + MANDATORY only (optional legs skipped) — the SSD tier
|
|
Reason string // why this target (Hungarian, for UI/logs)
|
|
}
|
|
|
|
// tier2FitsHeadroom reports whether a unit of unitGB fits on a system/rootfs drive while leaving a
|
|
// reserve free. Reserve = max(2 GB, 20% of total) — this is what protects the small (~8 GB) guest
|
|
// rootfs from being filled by a Tier 2 copy. Pure function (unit-tested).
|
|
func tier2FitsHeadroom(availGB, totalGB, unitGB float64) bool {
|
|
reserve := totalGB * 0.20
|
|
if reserve < 2.0 {
|
|
reserve = 2.0
|
|
}
|
|
return (availGB - unitGB) >= reserve
|
|
}
|
|
|
|
// selectTier2Target picks the off-drive destination for an app's Tier 2 copy. A customer-pinned
|
|
// target wins when still valid; otherwise it auto-picks: another user-data drive (holds the FULL set),
|
|
// else the internal SSD for the STATE-ONLY set (unit + mandatory), headroom-guarded. NETWORK storage
|
|
// (NAS) is never a valid target at any step — pinned or auto (F-6C-1: rsync -og under root_squash →
|
|
// silently-wrong-owner restore). fullSize sizes the real-drive path; stateOnlySize sizes the SSD path.
|
|
func (m *Manager) selectTier2Target(stackName string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
return m.selectTier2TargetFrom(stackName, sourceDrive, fullSize, stateOnlySize)
|
|
}
|
|
|
|
// selectTier2TargetFrom is selectTier2Target with the source drive supplied EXPLICITLY. It exists so
|
|
// the R-7b shares source — whose "source drive" is the drive a group of shares lives on, not an app's
|
|
// GetAppDrivePath — can reuse this selection and its headroom math verbatim instead of forking it.
|
|
// The app path above is a thin wrapper; nothing about its behaviour changed.
|
|
func (m *Manager) selectTier2TargetFrom(stackName, sourceDrive string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sawNetworkCandidate := false // an off-disk candidate existed but was network-only → better reason
|
|
|
|
// 0. Honor a customer-pinned target if it is still valid (registered, schedulable, off-disk,
|
|
// NON-network). An invalid pin (gone / same disk / network) silently falls through to auto.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.PreferredTarget != "" {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path != cd.PreferredTarget {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 %s: pinned target %s is network storage — invalid (F-6C-1); falling through to auto", stackName, sp.Path)
|
|
sawNetworkCandidate = true
|
|
break
|
|
}
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
break // pinned target is on the same physical disk — not off-drive; fall through
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true),
|
|
Label: label,
|
|
Reason: "kézi választás",
|
|
}, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1. Prefer another registered user-data drive on a DIFFERENT physical disk, NON-network.
|
|
if m.settings != nil {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
sawNetworkCandidate = true // F-6C-1: never a tier-2 target
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true), // Model A: in-guest mount IS the namespace root
|
|
Label: label,
|
|
Reason: "másik adatmeghajtó",
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
// 2. Fall back to the internal SSD (system data path) — STATE-ONLY set only.
|
|
sys := m.systemDataPath
|
|
if sys == "" || m.sameDevice(sourceDrive, sys) {
|
|
if sawNetworkCandidate {
|
|
return nil, errTier2NetworkOnly // the only off-disk candidate was a NAS
|
|
}
|
|
return nil, errNoOffDiskTarget // single drive / app already on the system disk
|
|
}
|
|
fits := m.tier2FitsSystemDrive
|
|
if m.tier2SSDFits != nil {
|
|
fits = m.tier2SSDFits
|
|
}
|
|
if !fits(sys, stateOnlySize) {
|
|
return nil, errSSDNoHeadroom // would fill the ~8 GB rootfs — refuse, don't fill
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sys, false), // system path is a real root → felhom-data appended
|
|
Label: "belső SSD (rendszer)",
|
|
IsSystemDrive: true,
|
|
StateOnly: true,
|
|
Reason: "nincs 2. adatmeghajtó — csak az adatbázis/konfiguráció fér a belső SSD-re; a nagy fájlokhoz 2. meghajtó kell",
|
|
}, nil
|
|
}
|
|
|
|
// tier2FitsSystemDrive checks the size-aware rootfs-headroom guard for the SSD target.
|
|
func (m *Manager) tier2FitsSystemDrive(sys string, unitSizeBytes int64) bool {
|
|
di := system.GetDiskUsage(sys)
|
|
if di == nil {
|
|
return false // can't determine free space → refuse (fail-closed for the rootfs)
|
|
}
|
|
return tier2FitsHeadroom(di.AvailGB, di.TotalGB, float64(unitSizeBytes)/gibibyte)
|
|
}
|
|
|
|
// Tier-2 v2 layout (Task 3b, architecture §8): backups/secondary/<stack>/ holds
|
|
// .felhom-tier2-layout (marker file, content "2" — written LAST, after all legs + reconcile)
|
|
// recovery-unit/ (the unit leg, layout-identical to v1)
|
|
// hdd/<relpath>/ (per-bind HDD legs, relpath-mirroring)
|
|
// userdata/<relpath>/ (per-bind USERDATA legs)
|
|
// Relpath-mirroring represents N>1 dirs + nested binds natively (the v1 flat-appdata N>1 refusal is
|
|
// lifted structurally) and makes restore position-derivable (dest relpath → live path under the app's
|
|
// current HDD_PATH). The whole tree is DERIVED from live data — migration is delete-and-rebuild.
|
|
const (
|
|
tier2LayoutMarker = ".felhom-tier2-layout"
|
|
tier2LayoutVersion = "2"
|
|
)
|
|
|
|
// tier2SafeRemove os.RemoveAll's target ONLY if it is strictly WITHIN a backups/secondary/<...>
|
|
// destBase (§9.2 destructive-write boundary — defense in depth against a mapping bug ever pointing a
|
|
// removal at live data; red-proofed). Never removes destBase itself.
|
|
func tier2SafeRemove(destBase, target string) error {
|
|
cb := filepath.Clean(destBase)
|
|
ct := filepath.Clean(target)
|
|
if !strings.Contains(filepath.ToSlash(cb), "/backups/secondary/") {
|
|
return fmt.Errorf("refusing tier-2 removal: destBase %q is not under backups/secondary/", cb)
|
|
}
|
|
if !strings.HasPrefix(ct, cb+string(filepath.Separator)) {
|
|
return fmt.Errorf("refusing tier-2 removal: %q is not strictly within %q", ct, cb)
|
|
}
|
|
return os.RemoveAll(ct)
|
|
}
|
|
|
|
// tier2RelClass classifies a dest dir (relpath dirRel, slash-form) against the current leg relpaths:
|
|
// keepInside = the dir is a leg or inside one (keep, don't descend); keepAncestor = on the path to a
|
|
// leg (keep, descend to find stale deeper); stale = neither (remove the topmost).
|
|
type tier2RelClass int
|
|
|
|
const (
|
|
tier2Stale tier2RelClass = iota
|
|
tier2KeepInside
|
|
tier2KeepAncestor
|
|
)
|
|
|
|
func classifyTier2Rel(dirRel string, legRels []string) tier2RelClass {
|
|
for _, lr := range legRels {
|
|
if dirRel == lr || strings.HasPrefix(dirRel, lr+"/") {
|
|
return tier2KeepInside // exact leg or descendant content
|
|
}
|
|
}
|
|
for _, lr := range legRels {
|
|
if strings.HasPrefix(lr, dirRel+"/") {
|
|
return tier2KeepAncestor // on the path to a deeper leg
|
|
}
|
|
}
|
|
return tier2Stale
|
|
}
|
|
|
|
// tier2Reconcile removes dest subdirs under hdd/ and userdata/ that are neither an ancestor nor a
|
|
// descendant of any current leg relpath (§7-D — the deferred-pruning answer: a bind removed/re-classed
|
|
// stops occupying the secondary drive within one run). Runs strictly inside destBase.
|
|
func (m *Manager) tier2Reconcile(destBase string, legRels []string) {
|
|
m.tier2ReconcileRoots(destBase, []string{"hdd", "userdata"}, legRels)
|
|
}
|
|
|
|
// tier2ReconcileRoots is tier2Reconcile with the top-level dest roots supplied explicitly — a pure
|
|
// extraction so the R-7b shares dest (whose roots are per-source-drive keys, not hdd/userdata) can
|
|
// reuse the SAME staleness classification and the SAME destBase-bounded removal guard.
|
|
func (m *Manager) tier2ReconcileRoots(destBase string, roots, legRels []string) {
|
|
var walk func(dirAbs, dirRel string)
|
|
walk = func(dirAbs, dirRel string) {
|
|
entries, err := os.ReadDir(dirAbs)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
if !e.IsDir() {
|
|
continue
|
|
}
|
|
childRel := dirRel + "/" + e.Name()
|
|
childAbs := filepath.Join(dirAbs, e.Name())
|
|
switch classifyTier2Rel(childRel, legRels) {
|
|
case tier2KeepInside:
|
|
// mirrored leg content — keep, no descent
|
|
case tier2KeepAncestor:
|
|
walk(childAbs, childRel)
|
|
default:
|
|
if err := tier2SafeRemove(destBase, childAbs); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 reconcile: %v", err)
|
|
} else {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 reconcile: removed stale dest dir %s", childRel)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, root := range roots {
|
|
walk(filepath.Join(destBase, root), root)
|
|
}
|
|
}
|
|
|
|
// RunTier2 makes/refreshes the off-drive v2 copy of a single app's recovery unit + its class-driven
|
|
// capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on
|
|
// an actual copy failure (no valid target is a recorded status, not an error).
|
|
func (m *Manager) RunTier2(stackName string) error {
|
|
// Customer turned Tier 2 off for this app (config panel) — skip without touching status.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s skipped — disabled by customer", stackName)
|
|
return nil
|
|
}
|
|
}
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sourceNsRoot := m.namespaceRoot(sourceDrive)
|
|
unitDir := RecoveryUnitPath(sourceNsRoot, stackName)
|
|
if _, err := os.Stat(unitDir); err != nil {
|
|
return nil // no recovery unit yet — nothing to copy
|
|
}
|
|
|
|
// Class-driven legs (classified) / resolver legs (legacy) + loud gap warnings (§7-H).
|
|
legs, warns := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
|
|
// Two sizes: full = unit + all legs; state-only = unit + mandatory legs (the SSD ceiling).
|
|
unitSize := dirSizeBytes(unitDir)
|
|
var fullSize, stateOnlySize int64 = unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
reason := tier2NoTargetReason(err)
|
|
m.recordTier2NoTarget(stackName, reason)
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s: no off-drive target — %s", stackName, reason)
|
|
return nil
|
|
}
|
|
// Defense-in-depth off-drive guard (selection already enforced it).
|
|
if m.sameDevice(sourceDrive, target.NamespaceRoot) {
|
|
m.recordTier2NoTarget(stackName, "a kiválasztott cél ugyanazon a fizikai lemezen van")
|
|
return nil
|
|
}
|
|
|
|
// §2.2: the SSD is a STATE-ONLY tier — drop optional legs, tell the customer honestly.
|
|
if target.StateOnly {
|
|
kept := legs[:0]
|
|
droppedOptional := false
|
|
for _, lg := range legs {
|
|
if lg.Class == ClassMandatory {
|
|
kept = append(kept, lg)
|
|
} else {
|
|
droppedOptional = true
|
|
}
|
|
}
|
|
legs = kept
|
|
if droppedOptional {
|
|
warns = append(warns, "A belső SSD-n csak a konfiguráció, adatbázis és a kötelező adatok férnek el — a választható tartalom nem került másolásra.")
|
|
}
|
|
}
|
|
|
|
destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName)
|
|
start := time.Now()
|
|
mirror := m.tier2Mirror
|
|
if mirror == nil {
|
|
mirror = rsyncMirror
|
|
}
|
|
|
|
// Migration = delete-and-rebuild (marker LAST). First v2 run (marker absent): remove the old flat
|
|
// appdata/ leg (recovery-unit/ is layout-identical — untouched). A half-migrated dest self-heals on
|
|
// the next run because the marker is only written after every leg + reconcile succeed.
|
|
markerPath := filepath.Join(destBase, tier2LayoutMarker)
|
|
if _, mErr := os.Stat(markerPath); mErr != nil {
|
|
if rmErr := tier2SafeRemove(destBase, filepath.Join(destBase, "appdata")); rmErr != nil && !os.IsNotExist(rmErr) {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: migration cleanup of old flat appdata failed: %v", stackName, rmErr)
|
|
}
|
|
}
|
|
|
|
// Unit leg (always).
|
|
if err := mirror(unitDir, filepath.Join(destBase, "recovery-unit")); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err)
|
|
}
|
|
// Capture legs.
|
|
legRels := make([]string, 0, len(legs))
|
|
mirroredSize := unitSize
|
|
for _, lg := range legs {
|
|
if err := mirror(lg.Src, filepath.Join(destBase, filepath.FromSlash(lg.DestRel))); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync leg %s for %s: %w", lg.DestRel, stackName, err)
|
|
}
|
|
legRels = append(legRels, lg.DestRel)
|
|
mirroredSize += dirSizeBytes(lg.Src)
|
|
}
|
|
|
|
// Reconcile stale dest dirs (a bind removed / re-classed excluded), then write the marker LAST.
|
|
m.tier2Reconcile(destBase, legRels)
|
|
if err := os.WriteFile(markerPath, []byte(tier2LayoutVersion), 0644); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: layout marker write failed (restore will refuse until next run): %v", stackName, err)
|
|
}
|
|
|
|
dur := time.Since(start)
|
|
m.recordTier2Success(stackName, target, mirroredSize, strings.Join(warns, " "), dur)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, dur, nil)
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 copied %s → %s (%s, %d leg(s), %s)%s",
|
|
stackName, destBase, humanizeBytes(mirroredSize), len(legs), dur.Round(time.Second),
|
|
map[bool]string{true: " [SSD: state-only]", false: ""}[target.StateOnly])
|
|
return nil
|
|
}
|
|
|
|
// RunAllTier2 runs Tier 2 for every deployed HDD app (apps whose data lives on an external drive —
|
|
// non-HDD apps live on the rootfs and are already inside the PBS whole-guest snapshot).
|
|
func (m *Manager) RunAllTier2() {
|
|
if m.stackProvider == nil {
|
|
return
|
|
}
|
|
if m.migrationActive() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 kihagyva: migráció folyamatban")
|
|
return
|
|
}
|
|
var n int
|
|
for _, stack := range m.stackProvider.ListDeployedStacks() {
|
|
// Reserved-name defense in depth (R-7b): the shares source owns backups/secondary/_shares and
|
|
// the _shares status record. Stack names come from the git-synced catalog, not customer input,
|
|
// so this cannot realistically fire — but if it ever did, the app would silently overwrite the
|
|
// shares tree, so it is refused loudly instead.
|
|
if stack.Name == SharesPseudoStack {
|
|
m.logger.Printf("[ERROR] [backup] Tier 2: stack %q uses the RESERVED shares key — skipped to protect the shares backup tree", stack.Name)
|
|
continue
|
|
}
|
|
// F6 (CAMPAIGN-3): volume-only apps (no HDD_PATH, backups on sys_drive) previously got NO
|
|
// tier-2 copy — a single controller-level copy on one device. They now flow through too: their
|
|
// recovery unit (which holds the db/volume dumps) gets a cross-drive second copy like any HDD
|
|
// app. Apps with no recovery unit yet (infra/never-backed-up) are a cheap no-op inside RunTier2
|
|
// (the `os.Stat(unitDir)` guard), so this doesn't spuriously copy protected stacks.
|
|
if m.settings != nil && (m.settings.IsDisconnected(m.GetAppDrivePath(stack.Name)) ||
|
|
m.settings.IsDecommissioned(m.GetAppDrivePath(stack.Name))) {
|
|
continue
|
|
}
|
|
runOne := m.perAppTier2
|
|
if runOne == nil {
|
|
runOne = m.RunTier2
|
|
}
|
|
if err := runOne(stack.Name); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 failed for %s: %v", stack.Name, err)
|
|
}
|
|
n++
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 run complete: %d app(s) processed (incl. volume-only — F6)", n)
|
|
|
|
// R-7b: the SHARES source runs after the per-app loop, in the SAME orchestrator run. It is a
|
|
// sibling job — nothing above it changed — and its failure never fails the app tier.
|
|
if err := m.RunSharesTier2(); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 shares job failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// --- per-app config-panel view (drives the Tier-2 "Beállítás" page) ---
|
|
|
|
// Tier2Option is one selectable off-drive destination in the config panel.
|
|
type Tier2Option struct {
|
|
Path string // registered storage path (the value persisted as PreferredTarget)
|
|
Label string // human label for the dropdown
|
|
}
|
|
|
|
// Tier2Info is the per-app Tier-2 view the config panel renders. It exposes the effective target
|
|
// (pinned or auto), whether that is the size-limited internal SSD, the honest no-target reason, and
|
|
// the off-disk drives the customer may pin — so the control is meaningful even with a single target.
|
|
type Tier2Info struct {
|
|
IsHDDApp bool // false = the app lives on the rootfs (already inside the PBS whole-guest snapshot)
|
|
SourceDrive string // where the app's data currently lives
|
|
Disabled bool // customer turned Tier 2 off
|
|
Preferred string // customer-pinned target path ("" = automatic)
|
|
EffectiveLabel string // label of the target that WOULD be used right now
|
|
EffectiveIsSSD bool // the effective target is the internal SSD (DB/config only)
|
|
EffectiveDesc string // why this target (Hungarian)
|
|
NoTarget bool // no off-drive target fits at all
|
|
NoTargetReason string // honest reason when NoTarget
|
|
Alternatives []Tier2Option
|
|
}
|
|
|
|
// Tier2Info builds the config-panel view for one app. Read-only (no status writes).
|
|
func (m *Manager) Tier2Info(stackName string) Tier2Info {
|
|
var info Tier2Info
|
|
if m.stackProvider != nil {
|
|
info.IsHDDApp = m.stackProvider.GetStackHDDPath(stackName) != ""
|
|
}
|
|
source := m.GetAppDrivePath(stackName)
|
|
info.SourceDrive = source
|
|
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil {
|
|
info.Disabled = cd.UserDisabled
|
|
info.Preferred = cd.PreferredTarget
|
|
}
|
|
// Eligible alternative drives: registered, schedulable, on a DIFFERENT physical disk.
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == source || m.sameDevice(source, sp.Path) {
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
info.Alternatives = append(info.Alternatives, Tier2Option{Path: sp.Path, Label: label})
|
|
}
|
|
}
|
|
|
|
// Resolve what the runner WOULD pick right now: the v2 capture legs feed both sizes (the SSD
|
|
// headroom guard tests the state-only set).
|
|
sourceNsRoot := m.namespaceRoot(source)
|
|
legs, _ := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName))
|
|
fullSize, stateOnlySize := unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
info.NoTarget = true
|
|
info.NoTargetReason = tier2NoTargetReason(err)
|
|
return info
|
|
}
|
|
info.EffectiveLabel = target.Label
|
|
info.EffectiveIsSSD = target.IsSystemDrive
|
|
info.EffectiveDesc = target.Reason
|
|
return info
|
|
}
|
|
|
|
// --- status persistence (drives the "2. mentés" UI card) ---
|
|
|
|
// withTier2Prefs carries the customer-preference fields (UserDisabled/PreferredTarget) from any
|
|
// existing config into a freshly-built status struct, so a runner status write never clobbers them.
|
|
func (m *Manager) withTier2Prefs(stackName string, cfg *settings.CrossDriveBackup) *settings.CrossDriveBackup {
|
|
if m.settings != nil {
|
|
if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
|
|
cfg.UserDisabled = existing.UserDisabled
|
|
cfg.PreferredTarget = existing.PreferredTarget
|
|
}
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) {
|
|
if m.settings == nil {
|
|
return
|
|
}
|
|
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
|
Enabled: true,
|
|
Method: "rsync",
|
|
DestinationPath: target.NamespaceRoot,
|
|
Schedule: "daily",
|
|
LastRun: time.Now().Format(time.RFC3339),
|
|
LastStatus: "ok",
|
|
LastWarning: strings.TrimSpace(warning),
|
|
LastDuration: dur.Round(time.Second).String(),
|
|
LastSizeHuman: humanizeBytes(sizeBytes),
|
|
})); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 status persist (ok) for %s failed: %v", stackName, err)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) {
|
|
if m.settings == nil {
|
|
return
|
|
}
|
|
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
|
Enabled: true,
|
|
Method: "rsync",
|
|
DestinationPath: target.NamespaceRoot,
|
|
Schedule: "daily",
|
|
LastRun: time.Now().Format(time.RFC3339),
|
|
LastStatus: "error",
|
|
LastError: cause.Error(),
|
|
})); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 status persist (error) for %s failed: %v", stackName, err)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) recordTier2NoTarget(stackName, reason string) {
|
|
if m.settings == nil {
|
|
return
|
|
}
|
|
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
|
Enabled: false,
|
|
Method: "rsync",
|
|
Schedule: "daily",
|
|
LastStatus: "no_target",
|
|
LastError: reason,
|
|
})); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 status persist (no_target) for %s failed: %v", stackName, err)
|
|
}
|
|
}
|
|
|
|
func tier2NoTargetReason(err error) string {
|
|
switch {
|
|
case errors.Is(err, errSSDNoHeadroom):
|
|
return "nincs elég hely a belső SSD-n — a nagy fájlok off-drive mentéséhez 2. meghajtó (vagy távoli tárhely) szükséges"
|
|
case errors.Is(err, errNoOffDiskTarget):
|
|
return "nincs másik fizikai meghajtó — a 2. mentéshez 2. meghajtó szükséges"
|
|
default:
|
|
return err.Error()
|
|
}
|
|
}
|
|
|
|
// --- helpers ---
|
|
|
|
// rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning).
|
|
func rsyncMirror(src, dst string) error {
|
|
if err := os.MkdirAll(dst, 0755); err != nil {
|
|
return fmt.Errorf("mkdir %s: %w", dst, err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
|
|
defer cancel()
|
|
// Trailing slashes: copy the CONTENTS of src into dst.
|
|
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/")
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// dirSizeBytes returns the total size of a directory via `du -sb` (0 if absent/error).
|
|
func dirSizeBytes(dir string) int64 {
|
|
if _, err := os.Stat(dir); err != nil {
|
|
return 0
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, "du", "-sb", dir).Output()
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
fields := strings.Fields(string(out))
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
var size int64
|
|
if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil {
|
|
return 0
|
|
}
|
|
return size
|
|
}
|