48f3336956
gates / gates (push) Successful in 23s
The notes a background run SAVES — last night's backup line, the last error, the proof result, the restore outcome — are written in the BOX's language at the moment they are written. A household that switches sees the previous run's note in the old language until the next run rewrites it: the operator's §16 option 1, stated rather than hidden. EndRestoreOp no longer receives a Hungarian literal from anywhere. The language switch is a globe. Two text links wrapped in the sidebar footer and asked the reader to recognise "Magyar"/"English" as links; a globe is the one symbol every web user already reads as "language", so nobody has to read Hungarian to escape Hungarian. It is <details>/<summary> — a menu with no script, drawn inline because the icon sprite lives only in layout.html and the visitor pages have their own shell. Those visitor pages get the same globe, and a visitor's choice stays theirs: a display-only felhom_lang cookie that langFor reads ONLY when there is no session. A signed-in household can never inherit a language a previous visitor picked in the same browser. POST /lang is CSRF-exempt for a narrow reason written at the exemption — its only achievable effect is the language of the page the victim's own browser shows them — and safeBackPath refuses //evil.example as well as https://, because "starts with /" alone is not the test. §16 taken: a successful claim carries the cookie into the household's setting. TWO PARITY EXCEPTIONS, MEASURED: 106 fixtures compared with a real diff — exactly two change shapes (the dashboard footer, the globe in the shells) and 5 byte-identical, which are the three pages that must not change. I INTRODUCED A DEADLOCK AND THE SUITE CAUGHT IT BY HANGING. UpdateOffboxStatus holds the settings write lock while running its callback; boxLang() wants the read lock; sync.RWMutex is not reentrant. On a real box an off-site run would have hung forever HOLDING the settings lock. Fixed by resolving the language before the callback, and guarded by a test that names the file and line in a second instead of hanging for 25 minutes. MinAgent: 0.131.0 (unchanged). No hub release needed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
739 lines
32 KiB
Go
739 lines
32 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
|
)
|
|
|
|
// Tier 2 = an off-drive (different physical disk) copy of an app's recovery unit + its CLASS-DRIVEN
|
|
// capture legs (Task 3b, architecture §2/§8). For a classified app the legs are the Task-3-core
|
|
// TierSecondary set — per-bind mandatory + optional HDD/userdata paths (so a classified app's copy
|
|
// tracks exactly what the catalog classed non-excluded; paperless's copy legitimately shrinks as
|
|
// `export` drops out). For a legacy app the legs are the resolver appdata dir(s), byte-identical to
|
|
// v0.134.x (the SQ5 footprint promise), mapped into the same layout. The dest is the v2
|
|
// relpath-mirroring layout (hdd/<rel>, userdata/<rel>, recovery-unit/, .felhom-tier2-layout marker) —
|
|
// N>1 dirs + nested binds represented natively (the old flat-appdata N>1 refusal is gone), migration
|
|
// is delete-and-rebuild of the derived copy, and a reconcile step prunes dest dirs a bind no longer
|
|
// covers. Target auto-pick: another registered NON-NETWORK user-data drive (holds the full set), else
|
|
// the internal SSD for the STATE-ONLY set (unit + mandatory), rootfs-headroom-guarded. NETWORK (NAS)
|
|
// storage is never a target (F-6C-1: rsync -og under root_squash → silently-wrong-owner restore). No
|
|
// off-drive target ⇒ an honest recorded status, not a silent no-op.
|
|
|
|
const gibibyte = 1024 * 1024 * 1024
|
|
|
|
var (
|
|
errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)")
|
|
errSSDNoHeadroom = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup")
|
|
// errTier2NetworkOnly (F-6C-1): the only off-disk candidate is a NAS network share, which can never
|
|
// be a tier-2 target — rsync -og chowns under a root_squash export and a wrong-owner restore is
|
|
// silently broken. The message IS the customer-facing reason.
|
|
errTier2NetworkOnly = util.MsgError("err.backup.halozati_tarolo_nem_lehet_a_2")
|
|
)
|
|
|
|
// appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD
|
|
// binds, via the stack provider (nil provider → legacy [stackName] fallback). See
|
|
// appbackup.AppDataDirNames.
|
|
func (m *Manager) appDataDirNames(stackName, hddPath string) []string {
|
|
var mounts []string
|
|
if m.stackProvider != nil {
|
|
mounts = m.stackProvider.GetStackHDDMounts(stackName)
|
|
}
|
|
return AppDataDirNames(hddPath, stackName, mounts)
|
|
}
|
|
|
|
// tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the
|
|
// WARN-on-missing-declared-dir rule; nil provider → false).
|
|
func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool {
|
|
if m.stackProvider == nil {
|
|
return false
|
|
}
|
|
return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName))
|
|
}
|
|
|
|
// Tier2Target is a resolved off-drive destination for an app's Tier 2 copy.
|
|
type Tier2Target struct {
|
|
NamespaceRoot string // felhom-data namespace root on the target drive
|
|
Label string // human label (UI)
|
|
IsSystemDrive bool // target is the internal SSD/system drive (DB/config only)
|
|
StateOnly bool // §2.2: this target carries unit + MANDATORY only (optional legs skipped) — the SSD tier
|
|
Reason string // why this target (Hungarian, for UI/logs)
|
|
}
|
|
|
|
// tier2FitsHeadroom reports whether a unit of unitGB fits on a system/rootfs drive while leaving a
|
|
// reserve free. Reserve = max(2 GB, 20% of total) — this is what protects the small (~8 GB) guest
|
|
// rootfs from being filled by a Tier 2 copy. Pure function (unit-tested).
|
|
func tier2FitsHeadroom(availGB, totalGB, unitGB float64) bool {
|
|
reserve := totalGB * 0.20
|
|
if reserve < 2.0 {
|
|
reserve = 2.0
|
|
}
|
|
return (availGB - unitGB) >= reserve
|
|
}
|
|
|
|
// selectTier2Target picks the off-drive destination for an app's Tier 2 copy. A customer-pinned
|
|
// target wins when still valid; otherwise it auto-picks: another user-data drive (holds the FULL set),
|
|
// else the internal SSD for the STATE-ONLY set (unit + mandatory), headroom-guarded. NETWORK storage
|
|
// (NAS) is never a valid target at any step — pinned or auto (F-6C-1: rsync -og under root_squash →
|
|
// silently-wrong-owner restore). fullSize sizes the real-drive path; stateOnlySize sizes the SSD path.
|
|
func (m *Manager) selectTier2Target(stackName string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
return m.selectTier2TargetFrom(stackName, sourceDrive, fullSize, stateOnlySize)
|
|
}
|
|
|
|
// selectTier2TargetFrom is selectTier2Target with the source drive supplied EXPLICITLY. It exists so
|
|
// the R-7b shares source — whose "source drive" is the drive a group of shares lives on, not an app's
|
|
// GetAppDrivePath — can reuse this selection and its headroom math verbatim instead of forking it.
|
|
// The app path above is a thin wrapper; nothing about its behaviour changed.
|
|
func (m *Manager) selectTier2TargetFrom(stackName, sourceDrive string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
|
|
if sourceDrive == "" {
|
|
return nil, fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sawNetworkCandidate := false // an off-disk candidate existed but was network-only → better reason
|
|
|
|
// 0. Honor a customer-pinned target if it is still valid (registered, schedulable, off-disk,
|
|
// NON-network). An invalid pin (gone / same disk / network) silently falls through to auto.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.PreferredTarget != "" {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path != cd.PreferredTarget {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 %s: pinned target %s is network storage — invalid (F-6C-1); falling through to auto", stackName, sp.Path)
|
|
sawNetworkCandidate = true
|
|
break
|
|
}
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
break // pinned target is on the same physical disk — not off-drive; fall through
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true),
|
|
Label: label,
|
|
Reason: m.note("note.tier2.reason_manual"),
|
|
}, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1. Prefer another registered user-data drive on a DIFFERENT physical disk, NON-network.
|
|
if m.settings != nil {
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == sourceDrive || m.sameDevice(sourceDrive, sp.Path) {
|
|
continue
|
|
}
|
|
if sp.IsNetwork() {
|
|
sawNetworkCandidate = true // F-6C-1: never a tier-2 target
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sp.Path, true), // Model A: in-guest mount IS the namespace root
|
|
Label: label,
|
|
Reason: m.note("note.tier2.reason_other_drive"),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
// 2. Fall back to the internal SSD (system data path) — STATE-ONLY set only.
|
|
sys := m.systemDataPath
|
|
if sys == "" || m.sameDevice(sourceDrive, sys) {
|
|
if sawNetworkCandidate {
|
|
return nil, errTier2NetworkOnly // the only off-disk candidate was a NAS
|
|
}
|
|
return nil, errNoOffDiskTarget // single drive / app already on the system disk
|
|
}
|
|
fits := m.tier2FitsSystemDrive
|
|
if m.tier2SSDFits != nil {
|
|
fits = m.tier2SSDFits
|
|
}
|
|
if !fits(sys, stateOnlySize) {
|
|
return nil, errSSDNoHeadroom // would fill the ~8 GB rootfs — refuse, don't fill
|
|
}
|
|
return &Tier2Target{
|
|
NamespaceRoot: NamespaceRoot(sys, false), // system path is a real root → felhom-data appended
|
|
Label: m.note("note.tier2.label_internal_ssd"),
|
|
IsSystemDrive: true,
|
|
StateOnly: true,
|
|
Reason: m.note("note.tier2.reason_no_second"),
|
|
}, nil
|
|
}
|
|
|
|
// tier2FitsSystemDrive checks the size-aware rootfs-headroom guard for the SSD target.
|
|
func (m *Manager) tier2FitsSystemDrive(sys string, unitSizeBytes int64) bool {
|
|
di := system.GetDiskUsage(sys)
|
|
if di == nil {
|
|
return false // can't determine free space → refuse (fail-closed for the rootfs)
|
|
}
|
|
return tier2FitsHeadroom(di.AvailGB, di.TotalGB, float64(unitSizeBytes)/gibibyte)
|
|
}
|
|
|
|
// Tier-2 v2 layout (Task 3b, architecture §8): backups/secondary/<stack>/ holds
|
|
//
|
|
// .felhom-tier2-layout (marker file, content "2" — written LAST, after all legs + reconcile)
|
|
// recovery-unit/ (the unit leg, layout-identical to v1)
|
|
// hdd/<relpath>/ (per-bind HDD legs, relpath-mirroring)
|
|
// userdata/<relpath>/ (per-bind USERDATA legs)
|
|
//
|
|
// Relpath-mirroring represents N>1 dirs + nested binds natively (the v1 flat-appdata N>1 refusal is
|
|
// lifted structurally) and makes restore position-derivable (dest relpath → live path under the app's
|
|
// current HDD_PATH). The whole tree is DERIVED from live data — migration is delete-and-rebuild.
|
|
const (
|
|
tier2LayoutMarker = ".felhom-tier2-layout"
|
|
tier2LayoutVersion = "2"
|
|
)
|
|
|
|
// tier2SafeRemove os.RemoveAll's target ONLY if it is strictly WITHIN a backups/secondary/<...>
|
|
// destBase (§9.2 destructive-write boundary — defense in depth against a mapping bug ever pointing a
|
|
// removal at live data; red-proofed). Never removes destBase itself.
|
|
func tier2SafeRemove(destBase, target string) error {
|
|
cb := filepath.Clean(destBase)
|
|
ct := filepath.Clean(target)
|
|
if !strings.Contains(filepath.ToSlash(cb), "/backups/secondary/") {
|
|
return fmt.Errorf("refusing tier-2 removal: destBase %q is not under backups/secondary/", cb)
|
|
}
|
|
if !strings.HasPrefix(ct, cb+string(filepath.Separator)) {
|
|
return fmt.Errorf("refusing tier-2 removal: %q is not strictly within %q", ct, cb)
|
|
}
|
|
return os.RemoveAll(ct)
|
|
}
|
|
|
|
// tier2RelClass classifies a dest dir (relpath dirRel, slash-form) against the current leg relpaths:
|
|
// keepInside = the dir is a leg or inside one (keep, don't descend); keepAncestor = on the path to a
|
|
// leg (keep, descend to find stale deeper); stale = neither (remove the topmost).
|
|
type tier2RelClass int
|
|
|
|
const (
|
|
tier2Stale tier2RelClass = iota
|
|
tier2KeepInside
|
|
tier2KeepAncestor
|
|
)
|
|
|
|
func classifyTier2Rel(dirRel string, legRels []string) tier2RelClass {
|
|
for _, lr := range legRels {
|
|
if dirRel == lr || strings.HasPrefix(dirRel, lr+"/") {
|
|
return tier2KeepInside // exact leg or descendant content
|
|
}
|
|
}
|
|
for _, lr := range legRels {
|
|
if strings.HasPrefix(lr, dirRel+"/") {
|
|
return tier2KeepAncestor // on the path to a deeper leg
|
|
}
|
|
}
|
|
return tier2Stale
|
|
}
|
|
|
|
// tier2Reconcile removes dest subdirs under hdd/ and userdata/ that are neither an ancestor nor a
|
|
// descendant of any current leg relpath (§7-D — the deferred-pruning answer: a bind removed/re-classed
|
|
// stops occupying the secondary drive within one run). Runs strictly inside destBase.
|
|
func (m *Manager) tier2Reconcile(destBase string, legRels []string) {
|
|
m.tier2ReconcileRoots(destBase, []string{"hdd", "userdata"}, legRels)
|
|
}
|
|
|
|
// tier2ReconcileRoots is tier2Reconcile with the top-level dest roots supplied explicitly — a pure
|
|
// extraction so the R-7b shares dest (whose roots are per-source-drive keys, not hdd/userdata) can
|
|
// reuse the SAME staleness classification and the SAME destBase-bounded removal guard.
|
|
func (m *Manager) tier2ReconcileRoots(destBase string, roots, legRels []string) {
|
|
var walk func(dirAbs, dirRel string)
|
|
walk = func(dirAbs, dirRel string) {
|
|
entries, err := os.ReadDir(dirAbs)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
if !e.IsDir() {
|
|
continue
|
|
}
|
|
childRel := dirRel + "/" + e.Name()
|
|
childAbs := filepath.Join(dirAbs, e.Name())
|
|
switch classifyTier2Rel(childRel, legRels) {
|
|
case tier2KeepInside:
|
|
// mirrored leg content — keep, no descent
|
|
case tier2KeepAncestor:
|
|
walk(childAbs, childRel)
|
|
default:
|
|
if err := tier2SafeRemove(destBase, childAbs); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 reconcile: %v", err)
|
|
} else {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 reconcile: removed stale dest dir %s", childRel)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, root := range roots {
|
|
walk(filepath.Join(destBase, root), root)
|
|
}
|
|
}
|
|
|
|
// RunTier2 makes/refreshes the off-drive v2 copy of a single app's recovery unit + its class-driven
|
|
// capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on
|
|
// an actual copy failure (no valid target is a recorded status, not an error).
|
|
func (m *Manager) RunTier2(stackName string) error {
|
|
// Customer turned Tier 2 off for this app (config panel) — skip without touching status.
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s skipped — disabled by customer", stackName)
|
|
return nil
|
|
}
|
|
}
|
|
sourceDrive := m.GetAppDrivePath(stackName)
|
|
if sourceDrive == "" {
|
|
return fmt.Errorf("no source drive for %s", stackName)
|
|
}
|
|
sourceNsRoot := m.namespaceRoot(sourceDrive)
|
|
unitDir := RecoveryUnitPath(sourceNsRoot, stackName)
|
|
if _, err := os.Stat(unitDir); err != nil {
|
|
return nil // no recovery unit yet — nothing to copy
|
|
}
|
|
|
|
// Class-driven legs (classified) / resolver legs (legacy) + loud gap warnings (§7-H).
|
|
legs, warns := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
|
|
// Two sizes: full = unit + all legs; state-only = unit + mandatory legs (the SSD ceiling).
|
|
unitSize := dirSizeBytes(unitDir)
|
|
var fullSize, stateOnlySize int64 = unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
reason := m.tier2NoTargetReason(err)
|
|
m.recordTier2NoTarget(stackName, reason)
|
|
m.logger.Printf("[INFO] [backup] Tier 2 for %s: no off-drive target — %s", stackName, reason)
|
|
return nil
|
|
}
|
|
// Defense-in-depth off-drive guard (selection already enforced it).
|
|
if m.sameDevice(sourceDrive, target.NamespaceRoot) {
|
|
m.recordTier2NoTarget(stackName, m.note("note.tier2.same_disk"))
|
|
return nil
|
|
}
|
|
|
|
// §2.2: the SSD is a STATE-ONLY tier — drop optional legs, tell the customer honestly.
|
|
if target.StateOnly {
|
|
kept := legs[:0]
|
|
droppedOptional := false
|
|
for _, lg := range legs {
|
|
if lg.Class == ClassMandatory {
|
|
kept = append(kept, lg)
|
|
} else {
|
|
droppedOptional = true
|
|
}
|
|
}
|
|
legs = kept
|
|
if droppedOptional {
|
|
warns = append(warns, m.note("note.tier2.ssd_partial"))
|
|
}
|
|
}
|
|
|
|
destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName)
|
|
start := time.Now()
|
|
mirror := m.tier2Mirror
|
|
if mirror == nil {
|
|
mirror = rsyncMirror
|
|
}
|
|
|
|
// Migration = delete-and-rebuild (marker LAST). First v2 run (marker absent): remove the old flat
|
|
// appdata/ leg (recovery-unit/ is layout-identical — untouched). A half-migrated dest self-heals on
|
|
// the next run because the marker is only written after every leg + reconcile succeed.
|
|
markerPath := filepath.Join(destBase, tier2LayoutMarker)
|
|
if _, mErr := os.Stat(markerPath); mErr != nil {
|
|
if rmErr := tier2SafeRemove(destBase, filepath.Join(destBase, "appdata")); rmErr != nil && !os.IsNotExist(rmErr) {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: migration cleanup of old flat appdata failed: %v", stackName, rmErr)
|
|
}
|
|
}
|
|
|
|
// Unit leg — always, EXCEPT the one case R-403 measured (see r403_hollow.go for the mechanism).
|
|
//
|
|
// `mirror` is `rsync -a --delete`. That is correct for a derived copy and it stays. What was
|
|
// missing is the precondition: a source unit that carries NO data must not be mirrored over a
|
|
// destination unit that does, because `--delete` then removes the customer's last package. Proven
|
|
// on demo-hp 2026-08-31: 120 082 104 B → 7 036 B in one run, reported as a success.
|
|
//
|
|
// The fence is EXACTLY this shape and no wider. Complete→complete, complete→hollow and
|
|
// hollow→hollow all mirror as before; a data leg that legitimately shrinks is untouched (this
|
|
// guard is on the unit leg only). §8 row 5's derived-copy rule is unchanged.
|
|
destUnit := filepath.Join(destBase, "recovery-unit")
|
|
unitLegSkipped := unitIsHollow(unitDir) && unitCarriesData(destUnit)
|
|
if unitLegSkipped {
|
|
// Loud, and it names the app, the reason and the consequence. Paths and counts only — a unit's
|
|
// compose/app.yaml carries portable secrets and nothing from inside it is logged here.
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: unit leg SKIPPED — the recovery unit on the source drive lists no database dumps and no volume tars, while the existing copy at %s does. The copy was PRESERVED rather than replaced with an empty one (R-403). The other legs continue.",
|
|
stackName, destUnit)
|
|
warns = append(warns, m.note(tier2UnitPreservedKey))
|
|
} else if err := mirror(unitDir, destUnit); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err)
|
|
}
|
|
// Capture legs.
|
|
legRels := make([]string, 0, len(legs))
|
|
mirroredSize := unitSize
|
|
for _, lg := range legs {
|
|
if err := mirror(lg.Src, filepath.Join(destBase, filepath.FromSlash(lg.DestRel))); err != nil {
|
|
m.recordTier2Failure(stackName, target, err)
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, time.Since(start), err)
|
|
}
|
|
return fmt.Errorf("tier2 rsync leg %s for %s: %w", lg.DestRel, stackName, err)
|
|
}
|
|
legRels = append(legRels, lg.DestRel)
|
|
mirroredSize += dirSizeBytes(lg.Src)
|
|
}
|
|
|
|
// Reconcile stale dest dirs (a bind removed / re-classed excluded), then write the marker LAST.
|
|
m.tier2Reconcile(destBase, legRels)
|
|
if err := os.WriteFile(markerPath, []byte(tier2LayoutVersion), 0644); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 %s: layout marker write failed (restore will refuse until next run): %v", stackName, err)
|
|
}
|
|
|
|
dur := time.Since(start)
|
|
// R-403: the package date is read from the DESTINATION unit's own manifest, so it describes what
|
|
// is actually in the copy whether the leg mirrored or was preserved. Reading the artifact rather
|
|
// than assuming the run's own timestamp is what stops the surface calling a preserved package
|
|
// fresh — a data loss traded for a comforting lie is not a fix.
|
|
m.recordTier2SuccessWithUnit(stackName, target, mirroredSize, strings.Join(warns, " "), dur,
|
|
unitLegSkipped, unitPackageDate(destUnit))
|
|
if m.tier2Notify != nil {
|
|
m.tier2Notify(stackName, target.Label, dur, nil)
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 copied %s → %s (%s, %d leg(s), %s)%s%s",
|
|
stackName, destBase, humanizeBytes(mirroredSize), len(legs), dur.Round(time.Second),
|
|
map[bool]string{true: " [SSD: state-only]", false: ""}[target.StateOnly],
|
|
map[bool]string{true: " [unit leg SKIPPED — existing package preserved, R-403]", false: ""}[unitLegSkipped])
|
|
return nil
|
|
}
|
|
|
|
// RunAllTier2 runs Tier 2 for every deployed HDD app (apps whose data lives on an external drive —
|
|
// non-HDD apps live on the rootfs and are already inside the PBS whole-guest snapshot).
|
|
func (m *Manager) RunAllTier2() {
|
|
if m.stackProvider == nil {
|
|
return
|
|
}
|
|
if m.migrationActive() {
|
|
m.logger.Printf("[INFO] [backup] Tier 2 kihagyva: migráció folyamatban")
|
|
return
|
|
}
|
|
var n int
|
|
for _, stack := range m.stackProvider.ListDeployedStacks() {
|
|
// Reserved-name defense in depth (R-7b): the shares source owns backups/secondary/_shares and
|
|
// the _shares status record. Stack names come from the git-synced catalog, not customer input,
|
|
// so this cannot realistically fire — but if it ever did, the app would silently overwrite the
|
|
// shares tree, so it is refused loudly instead.
|
|
if stack.Name == SharesPseudoStack {
|
|
m.logger.Printf("[ERROR] [backup] Tier 2: stack %q uses the RESERVED shares key — skipped to protect the shares backup tree", stack.Name)
|
|
continue
|
|
}
|
|
// F6 (CAMPAIGN-3): volume-only apps (no HDD_PATH, backups on sys_drive) previously got NO
|
|
// tier-2 copy — a single controller-level copy on one device. They now flow through too: their
|
|
// recovery unit (which holds the db/volume dumps) gets a cross-drive second copy like any HDD
|
|
// app. Apps with no recovery unit yet (infra/never-backed-up) are a cheap no-op inside RunTier2
|
|
// (the `os.Stat(unitDir)` guard), so this doesn't spuriously copy protected stacks.
|
|
if m.settings != nil && (m.settings.IsDisconnected(m.GetAppDrivePath(stack.Name)) ||
|
|
m.settings.IsDecommissioned(m.GetAppDrivePath(stack.Name))) {
|
|
continue
|
|
}
|
|
// Slice 4: never mirror over a HELD app's copy — it is the restore point the hold text names.
|
|
// See the matching skip in captureAllRecoveryUnits.
|
|
if m.isHeld(stack.Name) {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — the app is HELD; its copy is the restore point and is preserved", stack.Name)
|
|
continue
|
|
}
|
|
runOne := m.perAppTier2
|
|
if runOne == nil {
|
|
runOne = m.RunTier2
|
|
}
|
|
if err := runOne(stack.Name); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 failed for %s: %v", stack.Name, err)
|
|
}
|
|
n++
|
|
}
|
|
m.logger.Printf("[INFO] [backup] Tier 2 run complete: %d app(s) processed (incl. volume-only — F6)", n)
|
|
|
|
// R-7b: the SHARES source runs after the per-app loop, in the SAME orchestrator run. It is a
|
|
// sibling job — nothing above it changed — and its failure never fails the app tier.
|
|
if err := m.RunSharesTier2(); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 shares job failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// --- per-app config-panel view (drives the Tier-2 "Beállítás" page) ---
|
|
|
|
// Tier2Option is one selectable off-drive destination in the config panel.
|
|
type Tier2Option struct {
|
|
Path string // registered storage path (the value persisted as PreferredTarget)
|
|
Label string // human label for the dropdown
|
|
}
|
|
|
|
// Tier2Info is the per-app Tier-2 view the config panel renders. It exposes the effective target
|
|
// (pinned or auto), whether that is the size-limited internal SSD, the honest no-target reason, and
|
|
// the off-disk drives the customer may pin — so the control is meaningful even with a single target.
|
|
type Tier2Info struct {
|
|
IsHDDApp bool // false = the app lives on the rootfs (already inside the PBS whole-guest snapshot)
|
|
SourceDrive string // where the app's data currently lives
|
|
Disabled bool // customer turned Tier 2 off
|
|
Preferred string // customer-pinned target path ("" = automatic)
|
|
EffectiveLabel string // label of the target that WOULD be used right now
|
|
EffectiveIsSSD bool // the effective target is the internal SSD (DB/config only)
|
|
EffectiveDesc string // why this target (Hungarian)
|
|
NoTarget bool // no off-drive target fits at all
|
|
NoTargetReason string // honest reason when NoTarget
|
|
Alternatives []Tier2Option
|
|
}
|
|
|
|
// Tier2Info builds the config-panel view for one app. Read-only (no status writes).
|
|
func (m *Manager) Tier2Info(stackName string) Tier2Info {
|
|
var info Tier2Info
|
|
if m.stackProvider != nil {
|
|
info.IsHDDApp = m.stackProvider.GetStackHDDPath(stackName) != ""
|
|
}
|
|
source := m.GetAppDrivePath(stackName)
|
|
info.SourceDrive = source
|
|
|
|
if m.settings != nil {
|
|
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil {
|
|
info.Disabled = cd.UserDisabled
|
|
info.Preferred = cd.PreferredTarget
|
|
}
|
|
// Eligible alternative drives: registered, schedulable, on a DIFFERENT physical disk.
|
|
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
|
if sp.Path == source || m.sameDevice(source, sp.Path) {
|
|
continue
|
|
}
|
|
label := sp.Label
|
|
if label == "" {
|
|
label = filepath.Base(sp.Path)
|
|
}
|
|
info.Alternatives = append(info.Alternatives, Tier2Option{Path: sp.Path, Label: label})
|
|
}
|
|
}
|
|
|
|
// Resolve what the runner WOULD pick right now: the v2 capture legs feed both sizes (the SSD
|
|
// headroom guard tests the state-only set).
|
|
sourceNsRoot := m.namespaceRoot(source)
|
|
legs, _ := m.tier2CaptureSet(stackName, sourceNsRoot)
|
|
unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName))
|
|
fullSize, stateOnlySize := unitSize, unitSize
|
|
for _, lg := range legs {
|
|
sz := dirSizeBytes(lg.Src)
|
|
fullSize += sz
|
|
if lg.Class == ClassMandatory {
|
|
stateOnlySize += sz
|
|
}
|
|
}
|
|
target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
|
|
if err != nil {
|
|
info.NoTarget = true
|
|
info.NoTargetReason = m.tier2NoTargetReason(err)
|
|
return info
|
|
}
|
|
info.EffectiveLabel = target.Label
|
|
info.EffectiveIsSSD = target.IsSystemDrive
|
|
info.EffectiveDesc = target.Reason
|
|
return info
|
|
}
|
|
|
|
// --- status persistence (drives the "2. mentés" UI card) ---
|
|
|
|
// tier2Update applies a run outcome onto a COPY OF THE EXISTING ROW, then persists it.
|
|
//
|
|
// R-101 Part 2 — SAFE BY CONSTRUCTION, and this replaced a real hazard rather than tidying one. The
|
|
// three record* helpers each used to build a WHOLE `CrossDriveBackup` literal, with `withTier2Prefs`
|
|
// re-applying exactly two fields (UserDisabled, PreferredTarget). Every other field not named in the
|
|
// literal was silently zeroed on every status write. That is fine while the struct is stable and
|
|
// catastrophic the moment a field is added: R-101 adds `LastSuccess`, and under the old shape
|
|
// `recordTier2Failure` would have CLEARED it — the mirror-image of the defect being fixed, firing on
|
|
// the first failure instead of lying dormant.
|
|
//
|
|
// Starting from the existing row inverts the default: a new field carries over unless a caller
|
|
// deliberately overwrites it. The compile-safe form the R-100 review asked for; nothing is preserved
|
|
// by a list that can fall out of date.
|
|
//
|
|
// Callers must therefore CLEAR explicitly what a run invalidates (a stale LastError on success, a
|
|
// stale size on failure) — the old behaviour those clears reproduce is preserved exactly.
|
|
func (m *Manager) tier2Update(stackName string, mutate func(*settings.CrossDriveBackup)) {
|
|
if m.settings == nil {
|
|
return
|
|
}
|
|
var cfg settings.CrossDriveBackup
|
|
if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
|
|
cfg = *existing // value copy — EVERY field carries over by default
|
|
}
|
|
// One-time migration of a pre-anchor row. Under the old code `LastStatus=="ok"` with a LastRun
|
|
// means that run DID succeed, so adopting it as the initial anchor is truthful — and it is what
|
|
// keeps the deploy quiet: without it every existing row would flip to "never succeeded" at once
|
|
// (all 7 rows on the fleet were pre-anchor). A row whose last known state was an ERROR seeds
|
|
// nothing, because nothing in the old data evidences a success.
|
|
if !cfg.SuccessTracked {
|
|
if cfg.LastStatus == "ok" && cfg.LastRun != "" {
|
|
cfg.LastSuccess = cfg.LastRun
|
|
}
|
|
cfg.SuccessTracked = true
|
|
}
|
|
mutate(&cfg)
|
|
if err := m.settings.SetCrossDriveConfig(stackName, &cfg); err != nil {
|
|
m.logger.Printf("[WARN] [backup] Tier 2 status persist for %s failed: %v", stackName, err)
|
|
}
|
|
}
|
|
|
|
// recordTier2Success records a completed run for a path that HAS NO UNIT LEG — the shares
|
|
// pseudo-stack mirrors share legs and carries no recovery unit, so there is no leg to skip and no
|
|
// package whose date could be older than the run. It is the thin caller; ONE implementation lives
|
|
// below.
|
|
func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) {
|
|
m.recordTier2SuccessWithUnit(stackName, target, sizeBytes, warning, dur, false, "")
|
|
}
|
|
|
|
// recordTier2SuccessWithUnit records a completed run INCLUDING what happened to its unit leg.
|
|
// `unitLegSkipped` and `unitPkgDate` (R-403) travel with the rest rather than through a second write,
|
|
// because two writers to one record is how a status and the thing it describes drift apart.
|
|
// `unitPkgDate` is read from the destination unit's OWN manifest, so it is a fact about the copy and
|
|
// not about the run.
|
|
func (m *Manager) recordTier2SuccessWithUnit(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration, unitLegSkipped bool, unitPkgDate string) {
|
|
now := time.Now().Format(time.RFC3339)
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.UnitLegSkipped = unitLegSkipped
|
|
c.UnitPackageDate = unitPkgDate
|
|
c.Enabled = true
|
|
c.Method = "rsync"
|
|
c.DestinationPath = target.NamespaceRoot
|
|
c.Schedule = "daily"
|
|
c.LastRun = now
|
|
// R-101: the anchor. Only this branch advances it; no failure branch clears it.
|
|
c.LastSuccess = now
|
|
c.LastStatus = "ok"
|
|
c.LastWarning = strings.TrimSpace(warning)
|
|
c.LastDuration = dur.Round(time.Second).String()
|
|
c.LastSizeHuman = humanizeBytes(sizeBytes)
|
|
c.LastError = "" // a success invalidates the previous error
|
|
})
|
|
}
|
|
|
|
func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) {
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.Enabled = true
|
|
c.Method = "rsync"
|
|
c.DestinationPath = target.NamespaceRoot
|
|
c.Schedule = "daily"
|
|
c.LastRun = time.Now().Format(time.RFC3339) // the ATTEMPT clock — advances on failure, by design
|
|
c.LastStatus = "error"
|
|
c.LastError = cause.Error()
|
|
// LastSuccess is deliberately UNTOUCHED: a failure neither advances nor clears the anchor.
|
|
// Clearing it would make one bad night read as "no copy has ever succeeded".
|
|
c.LastWarning = "" // a warning from the last successful run does not describe this one
|
|
c.LastDuration = "" // preserving the old literal's clears exactly
|
|
c.LastSizeHuman = "" // ditto — a stale size would describe a copy this run did not make
|
|
})
|
|
}
|
|
|
|
func (m *Manager) recordTier2NoTarget(stackName, reason string) {
|
|
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
|
c.Enabled = false
|
|
c.Method = "rsync"
|
|
c.Schedule = "daily"
|
|
c.DestinationPath = ""
|
|
c.LastStatus = "no_target"
|
|
c.LastError = reason
|
|
c.LastRun = ""
|
|
// LastSuccess survives: "there is no destination drive right now" is not evidence that the
|
|
// last successful copy never happened. The UI gates on LastRun here, so nothing is rendered.
|
|
c.LastWarning = ""
|
|
c.LastDuration = ""
|
|
c.LastSizeHuman = ""
|
|
})
|
|
}
|
|
|
|
// tier2NoTargetReason is SAVED (CrossDriveConfig.LastError / Tier2Info.NoTargetReason) and read on the
|
|
// per-app card later, so it is written in the box's language at write time (release C, R-557). The
|
|
// branch is on a SENTINEL, never on these words (R-553) — which is why translating them is safe.
|
|
func (m *Manager) tier2NoTargetReason(err error) string {
|
|
switch {
|
|
case errors.Is(err, errSSDNoHeadroom):
|
|
return m.note("note.tier2.no_space_ssd")
|
|
case errors.Is(err, errNoOffDiskTarget):
|
|
return m.note("note.tier2.no_other_drive")
|
|
default:
|
|
return m.noteErr(err)
|
|
}
|
|
}
|
|
|
|
// --- helpers ---
|
|
|
|
// rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning).
|
|
func rsyncMirror(src, dst string) error {
|
|
if err := os.MkdirAll(dst, 0755); err != nil {
|
|
return fmt.Errorf("mkdir %s: %w", dst, err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
|
|
defer cancel()
|
|
// Trailing slashes: copy the CONTENTS of src into dst.
|
|
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/")
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// tier2UnitPreservedWarning is the customer-facing half of the R-403 skip. It rides in
|
|
// CrossDriveBackup.LastWarning, which the per-app card already renders, so the refusal reaches the
|
|
// SURFACE and not only the log — the shape recordTier2NoTarget established.
|
|
const tier2UnitPreservedKey = "note.tier2.unit_preserved"
|
|
|
|
// unitPackageDate returns the `created_at` of a recovery unit's manifest — WHEN the package in that
|
|
// directory was captured. "" when the manifest is absent or unparseable, which the surface must read
|
|
// as UNKNOWN and never as "now".
|
|
func unitPackageDate(unitDir string) string {
|
|
if man := readManifest(UnitManifestFile(unitDir)); man != nil {
|
|
return man.CreatedAt
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// dirSizeBytes returns the total size of a directory via `du -sb` (0 if absent/error).
|
|
func dirSizeBytes(dir string) int64 {
|
|
if _, err := os.Stat(dir); err != nil {
|
|
return 0
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, "du", "-sb", dir).Output()
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
fields := strings.Fields(string(out))
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
var size int64
|
|
if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil {
|
|
return 0
|
|
}
|
|
return size
|
|
}
|