admin
3f048e042b
Tier2LastRun is the attempt clock and was rendered as 'Legutóbbi másolat' in the restore confirm dialog. New LastSuccess + SuccessTracked anchor; tier2Update makes the three rebuild sites safe by construction. F-DIAG: six distinct causes, target-aware redaction.
676 lines
28 KiB
Go
676 lines
28 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
|
)
|
|
|
|
// Tier 2 = an off-drive (different physical disk) copy of an app's recovery unit + its CLASS-DRIVEN
|
|
// capture legs (Task 3b, architecture §2/§8). For a classified app the legs are the Task-3-core
|
|
// TierSecondary set — per-bind mandatory + optional HDD/userdata paths (so a classified app's copy
|
|
// tracks exactly what the catalog classed non-excluded; paperless's copy legitimately shrinks as
|
|
// `export` drops out). For a legacy app the legs are the resolver appdata dir(s), byte-identical to
|
|
// v0.134.x (the SQ5 footprint promise), mapped into the same layout. The dest is the v2
|
|
// relpath-mirroring layout (hdd/<rel>, userdata/<rel>, recovery-unit/, .felhom-tier2-layout marker) —
|
|
// N>1 dirs + nested binds represented natively (the old flat-appdata N>1 refusal is gone), migration
|
|
// is delete-and-rebuild of the derived copy, and a reconcile step prunes dest dirs a bind no longer
|
|
// covers. Target auto-pick: another registered NON-NETWORK user-data drive (holds the full set), else
|
|
// the internal SSD for the STATE-ONLY set (unit + mandatory), rootfs-headroom-guarded. NETWORK (NAS)
|
|
// storage is never a target (F-6C-1: rsync -og under root_squash → silently-wrong-owner restore). No
|
|
// off-drive target ⇒ an honest recorded status, not a silent no-op.
|
|
|
|
const gibibyte = 1024 * 1024 * 1024
|
|
|
|
var (
|
|
errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)")
|
|
errSSDNoHeadroom = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup")
|
|
// errTier2NetworkOnly (F-6C-1): the only off-disk candidate is a NAS network share, which can never
|
|
// be a tier-2 target — rsync -og chowns under a root_squash export and a wrong-owner restore is
|
|
// silently broken. The message IS the customer-facing reason.
|
|
errTier2NetworkOnly = errors.New("Hálózati tároló nem lehet a 2. mentés célja — a fájltulajdonos-adatok megőrzése ott nem garantálható.")
|
|
)
|
|
|
|
// appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD
|
|
// binds, via the stack provider (nil provider → legacy [stackName] fallback). See
|
|
// appbackup.AppDataDirNames.
|
|
func (m *Manager) appDataDirNames(stackName, hddPath string) []string {
|
|
var mounts []string
|
|
if m.stackProvider != nil {
|
|
mounts = m.stackProvider.GetStackHDDMounts(stackName)
|
|
}
|
|
return AppDataDirNames(hddPath, stackName, mounts)
|
|
}
|
|
|
|
// tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the
|
|
// WARN-on-missing-declared-dir rule; nil provider → false).
|
|
func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool {
|
|
if m.stackProvider == nil {
|
|
return false
|
|
}
|
|
return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName))
|
|
}
|
|
|
|
// Tier2Target is a resolved off-drive destination for an app's Tier 2 copy.
|
|
type Tier2Target struct {
|
|
NamespaceRoot string // felhom-data namespace root on the target drive
|
|
Label string // human label (UI)
|
|
IsSystemDrive bool // target is the internal SSD/system drive (DB/config only)
|
|
StateOnly bool // §2.2: this target carries unit + MANDATORY only (optional legs skipped) — the SSD tier
|
|
Reason string // why this target (Hungarian, for UI/logs)
|
|
}
|
|
|
|
// tier2FitsHeadroom reports whether a unit of unitGB fits on a system/rootfs drive while leaving a
|
|
// reserve free. Reserve = max(2 GB, 20% of total) — this is what protects the small (~8 GB) guest
|
|
// rootfs from being filled by a Tier 2 copy. Pure function (unit-tested).
|
|
func tier2FitsHeadroom(availGB, totalGB, unitGB float64) bool {
|
|
reserve := totalGB * 0.20
|
|
if reserve < 2.0 {
|
|
reserve = 2.0
|
|
}
|
|
return (availGB - unitGB) >= reserve
|
|
}
|
|
|
|
// selectTier2Target picks the off-drive destination for an app's Tier 2 copy. A customer-pinned
|
|
// target wins when still valid; otherwise it auto-picks: another user-data drive (holds the FULL set),
|
|
// else the internal SSD for the STATE-ONLY set (unit + mandatory), headroom-guarded. NETWORK storage
|
|
// (NAS) is never a valid target at any step — pinned or auto (F-6C-1: rsync -og under root_squash →
|
|
// silently-wrong-owner restore). fullSize sizes the real-drive path; stateOnlySize sizes the SSD path.
|
|
func (m *Manager) selectTier2Target(stackName string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
return m.selectTier2TargetFrom(stackName, sourceDrive, fullSize, stateOnlySize)
|
|
}
|
|
|
|
// selectTier2TargetFrom is selectTier2Target with the source drive supplied EXPLICITLY. It exists so
|
|
// the R-7b shares source — whose "source drive" is the drive a group of shares lives on, not an app's
|
|
// GetAppDrivePath — can reuse this selection and its headroom math verbatim instead of forking it.
|
|
// The app path above is a thin wrapper; nothing about its behaviour changed.
|
|
func (m *Manager) selectTier2TargetFrom(stackName, sourceDrive string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sawNetworkCandidate := false // an off-disk candidate existed but was network-only → better reason
|
|
|
|
// 0. Honor a customer-pinned target if it is still valid (registered, schedulable, off-disk,
|
|
// NON-network). An invalid pin (gone / same disk / network) silently falls through to auto.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.PreferredTarget != "" {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path != cd.PreferredTarget {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 %s: pinned target %s is network storage — invalid (F-6C-1); falling through to auto", stackName, sp.Path)
|
|
sawNetworkCandidate = true
|
|
break
|
|
}
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
break // pinned target is on the same physical disk — not off-drive; fall through
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true),
|
|
Label: label,
|
|
Reason: "kézi választás",
|
|
}, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1. Prefer another registered user-data drive on a DIFFERENT physical disk, NON-network.
|
|
if m.settings != nil {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
sawNetworkCandidate = true // F-6C-1: never a tier-2 target
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true), // Model A: in-guest mount IS the namespace root
|
|
Label: label,
|
|
Reason: "másik adatmeghajtó",
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
// 2. Fall back to the internal SSD (system data path) — STATE-ONLY set only.
|
|
sys := m.systemDataPath
|
|
if sys == "" || m.sameDevice(sourceDrive, sys) {
|
|
if sawNetworkCandidate {
|
|
return nil, errTier2NetworkOnly // the only off-disk candidate was a NAS
|
|
}
|
|
return nil, errNoOffDiskTarget // single drive / app already on the system disk
|
|
}
|
|
fits := m.tier2FitsSystemDrive
|
|
if m.tier2SSDFits != nil {
|
|
fits = m.tier2SSDFits
|
|
}
|
|
if !fits(sys, stateOnlySize) {
|
|
return nil, errSSDNoHeadroom // would fill the ~8 GB rootfs — refuse, don't fill
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sys, false), // system path is a real root → felhom-data appended
|
|
Label: "belső SSD (rendszer)",
|
|
IsSystemDrive: true,
|
|
StateOnly: true,
|
|
Reason: "nincs 2. adatmeghajtó — csak az adatbázis/konfiguráció fér a belső SSD-re; a nagy fájlokhoz 2. meghajtó kell",
|
|
}, nil
|
|
}
|
|
|
|
// tier2FitsSystemDrive checks the size-aware rootfs-headroom guard for the SSD target.
|
|
func (m *Manager) tier2FitsSystemDrive(sys string, unitSizeBytes int64) bool {
|
|
di := system.GetDiskUsage(sys)
|
|
if di == nil {
|
|
return false // can't determine free space → refuse (fail-closed for the rootfs)
|
|
}
|
|
return tier2FitsHeadroom(di.AvailGB, di.TotalGB, float64(unitSizeBytes)/gibibyte)
|
|
}
|
|
|
|
// Tier-2 v2 layout (Task 3b, architecture §8): backups/secondary/<stack>/ holds
|
|
//
|
|
// .felhom-tier2-layout (marker file, content "2" — written LAST, after all legs + reconcile)
|
|
// recovery-unit/ (the unit leg, layout-identical to v1)
|
|
// hdd/<relpath>/ (per-bind HDD legs, relpath-mirroring)
|
|
// userdata/<relpath>/ (per-bind USERDATA legs)
|
|
//
|
|
// Relpath-mirroring represents N>1 dirs + nested binds natively (the v1 flat-appdata N>1 refusal is
|
|
// lifted structurally) and makes restore position-derivable (dest relpath → live path under the app's
|
|
// current HDD_PATH). The whole tree is DERIVED from live data — migration is delete-and-rebuild.
|
|
const (
|
|
tier2LayoutMarker = ".felhom-tier2-layout"
|
|
tier2LayoutVersion = "2"
|
|
)
|
|
|
|
// tier2SafeRemove os.RemoveAll's target ONLY if it is strictly WITHIN a backups/secondary/<...>
|
|
// destBase (§9.2 destructive-write boundary — defense in depth against a mapping bug ever pointing a
|
|
// removal at live data; red-proofed). Never removes destBase itself.
|
|
func tier2SafeRemove(destBase, target string) error {
|
|
cb := filepath.Clean(destBase)
|
|
ct := filepath.Clean(target)
|
|
if !strings.Contains(filepath.ToSlash(cb), "/backups/secondary/") {
|
|
return fmt.Errorf("refusing tier-2 removal: destBase %q is not under backups/secondary/", cb)
|
|
}
|
|
if !strings.HasPrefix(ct, cb+string(filepath.Separator)) {
|
|
return fmt.Errorf("refusing tier-2 removal: %q is not strictly within %q", ct, cb)
|
|
}
|
|
return os.RemoveAll(ct)
|
|
}
|
|
|
|
// tier2RelClass classifies a dest dir (relpath dirRel, slash-form) against the current leg relpaths:
|
|
// keepInside = the dir is a leg or inside one (keep, don't descend); keepAncestor = on the path to a
|
|
// leg (keep, descend to find stale deeper); stale = neither (remove the topmost).
|
|
type tier2RelClass int
|
|
|
|
const (
|
|
tier2Stale tier2RelClass = iota
|
|
tier2KeepInside
|
|
tier2KeepAncestor
|
|
)
|
|
|
|
func classifyTier2Rel(dirRel string, legRels []string) tier2RelClass {
|
|
for _, lr := range legRels {
|
|
if dirRel == lr || strings.HasPrefix(dirRel, lr+"/") {
|
|
return tier2KeepInside // exact leg or descendant content
|
|
}
|
|
}
|
|
for _, lr := range legRels {
|
|
if strings.HasPrefix(lr, dirRel+"/") {
|
|
return tier2KeepAncestor // on the path to a deeper leg
|
|
}
|
|
}
|
|
return tier2Stale
|
|
}
|
|
|
|
// tier2Reconcile removes dest subdirs under hdd/ and userdata/ that are neither an ancestor nor a
|
|
// descendant of any current leg relpath (§7-D — the deferred-pruning answer: a bind removed/re-classed
|
|
// stops occupying the secondary drive within one run). Runs strictly inside destBase.
|
|
func (m *Manager) tier2Reconcile(destBase string, legRels []string) {
|
|
m.tier2ReconcileRoots(destBase, []string{"hdd", "userdata"}, legRels)
|
|
}
|
|
|
|
// tier2ReconcileRoots is tier2Reconcile with the top-level dest roots supplied explicitly — a pure
|
|
// extraction so the R-7b shares dest (whose roots are per-source-drive keys, not hdd/userdata) can
|
|
// reuse the SAME staleness classification and the SAME destBase-bounded removal guard.
|
|
func (m *Manager) tier2ReconcileRoots(destBase string, roots, legRels []string) {
|
|
var walk func(dirAbs, dirRel string)
|
|
walk = func(dirAbs, dirRel string) {
|
|
entries, err := os.ReadDir(dirAbs)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
if !e.IsDir() {
|
|
continue
|
|
}
|
|
childRel := dirRel + "/" + e.Name()
|
|
childAbs := filepath.Join(dirAbs, e.Name())
|
|
switch classifyTier2Rel(childRel, legRels) {
|
|
case tier2KeepInside:
|
|
// mirrored leg content — keep, no descent
|
|
case tier2KeepAncestor:
|
|
walk(childAbs, childRel)
|
|
default:
|
|
if err := tier2SafeRemove(destBase, childAbs); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 reconcile: %v", err)
|
|
} else {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 reconcile: removed stale dest dir %s", childRel)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, root := range roots {
|
|
walk(filepath.Join(destBase, root), root)
|
|
}
|
|
}
|
|
|
|
// RunTier2 makes/refreshes the off-drive v2 copy of a single app's recovery unit + its class-driven
|
|
// capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on
|
|
// an actual copy failure (no valid target is a recorded status, not an error).
|
|
func (m *Manager) RunTier2(stackName string) error {
|
|
// Customer turned Tier 2 off for this app (config panel) — skip without touching status.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s skipped — disabled by customer", stackName)
|
|
return nil
|
|
}
|
|
}
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sourceNsRoot := m.namespaceRoot(sourceDrive)
|
|
unitDir := RecoveryUnitPath(sourceNsRoot, stackName)
|
|
if _, err := os.Stat(unitDir); err != nil {
|
|
return nil // no recovery unit yet — nothing to copy
|
|
}
|
|
|
|
// Class-driven legs (classified) / resolver legs (legacy) + loud gap warnings (§7-H).
|
|
legs, warns := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
|
|
// Two sizes: full = unit + all legs; state-only = unit + mandatory legs (the SSD ceiling).
|
|
unitSize := dirSizeBytes(unitDir)
|
|
var fullSize, stateOnlySize int64 = unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
reason := tier2NoTargetReason(err)
|
|
m.recordTier2NoTarget(stackName, reason)
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s: no off-drive target — %s", stackName, reason)
|
|
return nil
|
|
}
|
|
// Defense-in-depth off-drive guard (selection already enforced it).
|
|
if m.sameDevice(sourceDrive, target.NamespaceRoot) {
|
|
m.recordTier2NoTarget(stackName, "a kiválasztott cél ugyanazon a fizikai lemezen van")
|
|
return nil
|
|
}
|
|
|
|
// §2.2: the SSD is a STATE-ONLY tier — drop optional legs, tell the customer honestly.
|
|
if target.StateOnly {
|
|
kept := legs[:0]
|
|
droppedOptional := false
|
|
for _, lg := range legs {
|
|
if lg.Class == ClassMandatory {
|
|
kept = append(kept, lg)
|
|
} else {
|
|
droppedOptional = true
|
|
}
|
|
}
|
|
legs = kept
|
|
if droppedOptional {
|
|
warns = append(warns, "A belső SSD-n csak a konfiguráció, adatbázis és a kötelező adatok férnek el — a választható tartalom nem került másolásra.")
|
|
}
|
|
}
|
|
|
|
destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName)
|
|
start := time.Now()
|
|
mirror := m.tier2Mirror
|
|
if mirror == nil {
|
|
mirror = rsyncMirror
|
|
}
|
|
|
|
// Migration = delete-and-rebuild (marker LAST). First v2 run (marker absent): remove the old flat
|
|
// appdata/ leg (recovery-unit/ is layout-identical — untouched). A half-migrated dest self-heals on
|
|
// the next run because the marker is only written after every leg + reconcile succeed.
|
|
markerPath := filepath.Join(destBase, tier2LayoutMarker)
|
|
if _, mErr := os.Stat(markerPath); mErr != nil {
|
|
if rmErr := tier2SafeRemove(destBase, filepath.Join(destBase, "appdata")); rmErr != nil && !os.IsNotExist(rmErr) {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: migration cleanup of old flat appdata failed: %v", stackName, rmErr)
|
|
}
|
|
}
|
|
|
|
// Unit leg (always).
|
|
if err := mirror(unitDir, filepath.Join(destBase, "recovery-unit")); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err)
|
|
}
|
|
// Capture legs.
|
|
legRels := make([]string, 0, len(legs))
|
|
mirroredSize := unitSize
|
|
for _, lg := range legs {
|
|
if err := mirror(lg.Src, filepath.Join(destBase, filepath.FromSlash(lg.DestRel))); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync leg %s for %s: %w", lg.DestRel, stackName, err)
|
|
}
|
|
legRels = append(legRels, lg.DestRel)
|
|
mirroredSize += dirSizeBytes(lg.Src)
|
|
}
|
|
|
|
// Reconcile stale dest dirs (a bind removed / re-classed excluded), then write the marker LAST.
|
|
m.tier2Reconcile(destBase, legRels)
|
|
if err := os.WriteFile(markerPath, []byte(tier2LayoutVersion), 0644); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: layout marker write failed (restore will refuse until next run): %v", stackName, err)
|
|
}
|
|
|
|
dur := time.Since(start)
|
|
m.recordTier2Success(stackName, target, mirroredSize, strings.Join(warns, " "), dur)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, dur, nil)
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 copied %s → %s (%s, %d leg(s), %s)%s",
|
|
stackName, destBase, humanizeBytes(mirroredSize), len(legs), dur.Round(time.Second),
|
|
map[bool]string{true: " [SSD: state-only]", false: ""}[target.StateOnly])
|
|
return nil
|
|
}
|
|
|
|
// RunAllTier2 runs Tier 2 for every deployed HDD app (apps whose data lives on an external drive —
|
|
// non-HDD apps live on the rootfs and are already inside the PBS whole-guest snapshot).
|
|
func (m *Manager) RunAllTier2() {
|
|
if m.stackProvider == nil {
|
|
return
|
|
}
|
|
if m.migrationActive() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 kihagyva: migráció folyamatban")
|
|
return
|
|
}
|
|
var n int
|
|
for _, stack := range m.stackProvider.ListDeployedStacks() {
|
|
// Reserved-name defense in depth (R-7b): the shares source owns backups/secondary/_shares and
|
|
// the _shares status record. Stack names come from the git-synced catalog, not customer input,
|
|
// so this cannot realistically fire — but if it ever did, the app would silently overwrite the
|
|
// shares tree, so it is refused loudly instead.
|
|
if stack.Name == SharesPseudoStack {
|
|
m.logger.Printf("[ERROR] [backup] Tier 2: stack %q uses the RESERVED shares key — skipped to protect the shares backup tree", stack.Name)
|
|
continue
|
|
}
|
|
// F6 (CAMPAIGN-3): volume-only apps (no HDD_PATH, backups on sys_drive) previously got NO
|
|
// tier-2 copy — a single controller-level copy on one device. They now flow through too: their
|
|
// recovery unit (which holds the db/volume dumps) gets a cross-drive second copy like any HDD
|
|
// app. Apps with no recovery unit yet (infra/never-backed-up) are a cheap no-op inside RunTier2
|
|
// (the `os.Stat(unitDir)` guard), so this doesn't spuriously copy protected stacks.
|
|
if m.settings != nil && (m.settings.IsDisconnected(m.GetAppDrivePath(stack.Name)) ||
|
|
m.settings.IsDecommissioned(m.GetAppDrivePath(stack.Name))) {
|
|
continue
|
|
}
|
|
runOne := m.perAppTier2
|
|
if runOne == nil {
|
|
runOne = m.RunTier2
|
|
}
|
|
if err := runOne(stack.Name); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 failed for %s: %v", stack.Name, err)
|
|
}
|
|
n++
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 run complete: %d app(s) processed (incl. volume-only — F6)", n)
|
|
|
|
// R-7b: the SHARES source runs after the per-app loop, in the SAME orchestrator run. It is a
|
|
// sibling job — nothing above it changed — and its failure never fails the app tier.
|
|
if err := m.RunSharesTier2(); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 shares job failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// --- per-app config-panel view (drives the Tier-2 "Beállítás" page) ---
|
|
|
|
// Tier2Option is one selectable off-drive destination in the config panel.
|
|
type Tier2Option struct {
|
|
Path string // registered storage path (the value persisted as PreferredTarget)
|
|
Label string // human label for the dropdown
|
|
}
|
|
|
|
// Tier2Info is the per-app Tier-2 view the config panel renders. It exposes the effective target
|
|
// (pinned or auto), whether that is the size-limited internal SSD, the honest no-target reason, and
|
|
// the off-disk drives the customer may pin — so the control is meaningful even with a single target.
|
|
type Tier2Info struct {
|
|
IsHDDApp bool // false = the app lives on the rootfs (already inside the PBS whole-guest snapshot)
|
|
SourceDrive string // where the app's data currently lives
|
|
Disabled bool // customer turned Tier 2 off
|
|
Preferred string // customer-pinned target path ("" = automatic)
|
|
EffectiveLabel string // label of the target that WOULD be used right now
|
|
EffectiveIsSSD bool // the effective target is the internal SSD (DB/config only)
|
|
EffectiveDesc string // why this target (Hungarian)
|
|
NoTarget bool // no off-drive target fits at all
|
|
NoTargetReason string // honest reason when NoTarget
|
|
Alternatives []Tier2Option
|
|
}
|
|
|
|
// Tier2Info builds the config-panel view for one app. Read-only (no status writes).
|
|
func (m *Manager) Tier2Info(stackName string) Tier2Info {
|
|
var info Tier2Info
|
|
if m.stackProvider != nil {
|
|
info.IsHDDApp = m.stackProvider.GetStackHDDPath(stackName) != ""
|
|
}
|
|
source := m.GetAppDrivePath(stackName)
|
|
info.SourceDrive = source
|
|
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil {
|
|
info.Disabled = cd.UserDisabled
|
|
info.Preferred = cd.PreferredTarget
|
|
}
|
|
// Eligible alternative drives: registered, schedulable, on a DIFFERENT physical disk.
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == source || m.sameDevice(source, sp.Path) {
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
info.Alternatives = append(info.Alternatives, Tier2Option{Path: sp.Path, Label: label})
|
|
}
|
|
}
|
|
|
|
// Resolve what the runner WOULD pick right now: the v2 capture legs feed both sizes (the SSD
|
|
// headroom guard tests the state-only set).
|
|
sourceNsRoot := m.namespaceRoot(source)
|
|
legs, _ := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName))
|
|
fullSize, stateOnlySize := unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
info.NoTarget = true
|
|
info.NoTargetReason = tier2NoTargetReason(err)
|
|
return info
|
|
}
|
|
info.EffectiveLabel = target.Label
|
|
info.EffectiveIsSSD = target.IsSystemDrive
|
|
info.EffectiveDesc = target.Reason
|
|
return info
|
|
}
|
|
|
|
// --- status persistence (drives the "2. mentés" UI card) ---
|
|
|
|
// tier2Update applies a run outcome onto a COPY OF THE EXISTING ROW, then persists it.
|
|
//
|
|
// R-101 Part 2 — SAFE BY CONSTRUCTION, and this replaced a real hazard rather than tidying one. The
|
|
// three record* helpers each used to build a WHOLE `CrossDriveBackup` literal, with `withTier2Prefs`
|
|
// re-applying exactly two fields (UserDisabled, PreferredTarget). Every other field not named in the
|
|
// literal was silently zeroed on every status write. That is fine while the struct is stable and
|
|
// catastrophic the moment a field is added: R-101 adds `LastSuccess`, and under the old shape
|
|
// `recordTier2Failure` would have CLEARED it — the mirror-image of the defect being fixed, firing on
|
|
// the first failure instead of lying dormant.
|
|
//
|
|
// Starting from the existing row inverts the default: a new field carries over unless a caller
|
|
// deliberately overwrites it. The compile-safe form the R-100 review asked for; nothing is preserved
|
|
// by a list that can fall out of date.
|
|
//
|
|
// Callers must therefore CLEAR explicitly what a run invalidates (a stale LastError on success, a
|
|
// stale size on failure) — the old behaviour those clears reproduce is preserved exactly.
|
|
func (m *Manager) tier2Update(stackName string, mutate func(*settings.CrossDriveBackup)) {
|
|
if m.settings == nil {
|
|
return
|
|
}
|
|
var cfg settings.CrossDriveBackup
|
|
if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
|
|
cfg = *existing // value copy — EVERY field carries over by default
|
|
}
|
|
// One-time migration of a pre-anchor row. Under the old code `LastStatus=="ok"` with a LastRun
|
|
// means that run DID succeed, so adopting it as the initial anchor is truthful — and it is what
|
|
// keeps the deploy quiet: without it every existing row would flip to "never succeeded" at once
|
|
// (all 7 rows on the fleet were pre-anchor). A row whose last known state was an ERROR seeds
|
|
// nothing, because nothing in the old data evidences a success.
|
|
if !cfg.SuccessTracked {
|
|
if cfg.LastStatus == "ok" && cfg.LastRun != "" {
|
|
cfg.LastSuccess = cfg.LastRun
|
|
}
|
|
cfg.SuccessTracked = true
|
|
}
|
|
mutate(&cfg)
|
|
if err := m.settings.SetCrossDriveConfig(stackName, &cfg); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 status persist for %s failed: %v", stackName, err)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) {
|
|
now := time.Now().Format(time.RFC3339)
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.Enabled = true
|
|
c.Method = "rsync"
|
|
c.DestinationPath = target.NamespaceRoot
|
|
c.Schedule = "daily"
|
|
c.LastRun = now
|
|
// R-101: the anchor. Only this branch advances it; no failure branch clears it.
|
|
c.LastSuccess = now
|
|
c.LastStatus = "ok"
|
|
c.LastWarning = strings.TrimSpace(warning)
|
|
c.LastDuration = dur.Round(time.Second).String()
|
|
c.LastSizeHuman = humanizeBytes(sizeBytes)
|
|
c.LastError = "" // a success invalidates the previous error
|
|
})
|
|
}
|
|
|
|
func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) {
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.Enabled = true
|
|
c.Method = "rsync"
|
|
c.DestinationPath = target.NamespaceRoot
|
|
c.Schedule = "daily"
|
|
c.LastRun = time.Now().Format(time.RFC3339) // the ATTEMPT clock — advances on failure, by design
|
|
c.LastStatus = "error"
|
|
c.LastError = cause.Error()
|
|
// LastSuccess is deliberately UNTOUCHED: a failure neither advances nor clears the anchor.
|
|
// Clearing it would make one bad night read as "no copy has ever succeeded".
|
|
c.LastWarning = "" // a warning from the last successful run does not describe this one
|
|
c.LastDuration = "" // preserving the old literal's clears exactly
|
|
c.LastSizeHuman = "" // ditto — a stale size would describe a copy this run did not make
|
|
})
|
|
}
|
|
|
|
func (m *Manager) recordTier2NoTarget(stackName, reason string) {
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.Enabled = false
|
|
c.Method = "rsync"
|
|
c.Schedule = "daily"
|
|
c.DestinationPath = ""
|
|
c.LastStatus = "no_target"
|
|
c.LastError = reason
|
|
c.LastRun = ""
|
|
// LastSuccess survives: "there is no destination drive right now" is not evidence that the
|
|
// last successful copy never happened. The UI gates on LastRun here, so nothing is rendered.
|
|
c.LastWarning = ""
|
|
c.LastDuration = ""
|
|
c.LastSizeHuman = ""
|
|
})
|
|
}
|
|
|
|
func tier2NoTargetReason(err error) string {
|
|
switch {
|
|
case errors.Is(err, errSSDNoHeadroom):
|
|
return "nincs elég hely a belső SSD-n — a nagy fájlok off-drive mentéséhez 2. meghajtó (vagy távoli tárhely) szükséges"
|
|
case errors.Is(err, errNoOffDiskTarget):
|
|
return "nincs másik fizikai meghajtó — a 2. mentéshez 2. meghajtó szükséges"
|
|
default:
|
|
return err.Error()
|
|
}
|
|
}
|
|
|
|
// --- helpers ---
|
|
|
|
// rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning).
|
|
func rsyncMirror(src, dst string) error {
|
|
if err := os.MkdirAll(dst, 0755); err != nil {
|
|
return fmt.Errorf("mkdir %s: %w", dst, err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
|
|
defer cancel()
|
|
// Trailing slashes: copy the CONTENTS of src into dst.
|
|
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/")
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// dirSizeBytes returns the total size of a directory via `du -sb` (0 if absent/error).
|
|
func dirSizeBytes(dir string) int64 {
|
|
if _, err := os.Stat(dir); err != nil {
|
|
return 0
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, "du", "-sb", dir).Output()
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
fields := strings.Fields(string(out))
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
var size int64
|
|
if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil {
|
|
return 0
|
|
}
|
|
return size
|
|
}
|