felhom-controller/controller/internal/backup/tier2.go

package backup

import (
	"context"
	"errors"
	"fmt"
	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
)

// Tier 2 = an off-drive (different physical disk) copy of an app's recovery unit + its CLASS-DRIVEN
// capture legs (Task 3b, architecture §2/§8). For a classified app the legs are the Task-3-core
// TierSecondary set — per-bind mandatory + optional HDD/userdata paths (so a classified app's copy
// tracks exactly what the catalog classed non-excluded; paperless's copy legitimately shrinks as
// `export` drops out). For a legacy app the legs are the resolver appdata dir(s), byte-identical to
// v0.134.x (the SQ5 footprint promise), mapped into the same layout. The dest is the v2
// relpath-mirroring layout (hdd/<rel>, userdata/<rel>, recovery-unit/, .felhom-tier2-layout marker) —
// N>1 dirs + nested binds represented natively (the old flat-appdata N>1 refusal is gone), migration
// is delete-and-rebuild of the derived copy, and a reconcile step prunes dest dirs a bind no longer
// covers. Target auto-pick: another registered NON-NETWORK user-data drive (holds the full set), else
// the internal SSD for the STATE-ONLY set (unit + mandatory), rootfs-headroom-guarded. NETWORK (NAS)
// storage is never a target (F-6C-1: rsync -og under root_squash → silently-wrong-owner restore). No
// off-drive target ⇒ an honest recorded status, not a silent no-op.

const gibibyte = 1024 * 1024 * 1024

var (
	errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)")
	errSSDNoHeadroom   = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup")
	// errTier2NetworkOnly (F-6C-1): the only off-disk candidate is a NAS network share, which can never
	// be a tier-2 target — rsync -og chowns under a root_squash export and a wrong-owner restore is
	// silently broken. The message IS the customer-facing reason.
	errTier2NetworkOnly = util.MsgError("err.backup.halozati_tarolo_nem_lehet_a_2")
)

// appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD
// binds, via the stack provider (nil provider → legacy [stackName] fallback). See
// appbackup.AppDataDirNames.
func (m *Manager) appDataDirNames(stackName, hddPath string) []string {
	var mounts []string
	if m.stackProvider != nil {
		mounts = m.stackProvider.GetStackHDDMounts(stackName)
	}
	return AppDataDirNames(hddPath, stackName, mounts)
}

// tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the
// WARN-on-missing-declared-dir rule; nil provider → false).
func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool {
	if m.stackProvider == nil {
		return false
	}
	return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName))
}

// Tier2Target is a resolved off-drive destination for an app's Tier 2 copy.
type Tier2Target struct {
	NamespaceRoot string // felhom-data namespace root on the target drive
	Label         string // human label (UI)
	IsSystemDrive bool   // target is the internal SSD/system drive (DB/config only)
	StateOnly     bool   // §2.2: this target carries unit + MANDATORY only (optional legs skipped) — the SSD tier
	Reason        string // why this target (Hungarian, for UI/logs)
	// Unproven (`16` D2): the agent cannot prove this target is another physical disk (a VM, a USB bridge
	// that hides the serial). The copy is still made; it does not count as a second copy, and the app's
	// row says why. UnprovenReason is the agent's reason code.
	Unproven       bool
	UnprovenReason string
}

// tier2FitsHeadroom reports whether a unit of unitGB fits on a system/rootfs drive while leaving a
// reserve free. Reserve = max(2 GB, 20% of total) — this is what protects the small (~8 GB) guest
// rootfs from being filled by a Tier 2 copy. Pure function (unit-tested).
func tier2FitsHeadroom(availGB, totalGB, unitGB float64) bool {
	reserve := totalGB * 0.20
	if reserve < 2.0 {
		reserve = 2.0
	}
	return (availGB - unitGB) >= reserve
}

// selectTier2Target picks the off-drive destination for an app's Tier 2 copy. A customer-pinned
// target wins when still valid; otherwise it auto-picks: another user-data drive (holds the FULL set),
// else the internal SSD for the STATE-ONLY set (unit + mandatory), headroom-guarded. NETWORK storage
// (NAS) is never a valid target at any step — pinned or auto (F-6C-1: rsync -og under root_squash →
// silently-wrong-owner restore). fullSize sizes the real-drive path; stateOnlySize sizes the SSD path.
func (m *Manager) selectTier2Target(stackName string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
	sourceDrive := m.GetAppDrivePath(stackName)
	if sourceDrive == "" {
		return nil, fmt.Errorf("no source drive for %s", stackName)
	}
	return m.selectTier2TargetFrom(stackName, sourceDrive, fullSize, stateOnlySize)
}

// selectTier2TargetFrom is selectTier2Target with the source drive supplied EXPLICITLY. It exists so
// the R-7b shares source — whose "source drive" is the drive a group of shares lives on, not an app's
// GetAppDrivePath — can reuse this selection and its headroom math verbatim instead of forking it.
// The app path above is a thin wrapper; nothing about its behaviour changed.
func (m *Manager) selectTier2TargetFrom(stackName, sourceDrive string, fullSize, stateOnlySize int64) (*Tier2Target, error) {
	if sourceDrive == "" {
		return nil, fmt.Errorf("no source drive for %s", stackName)
	}
	sawNetworkCandidate := false // an off-disk candidate existed but was network-only → better reason

	// 0. Honor a customer-pinned target if it is still valid (registered, schedulable, off-disk,
	//    NON-network). An invalid pin (gone / same disk / network) silently falls through to auto.
	if m.settings != nil {
		if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.PreferredTarget != "" {
			for _, sp := range m.settings.GetSchedulableStoragePaths() {
				if sp.Path != cd.PreferredTarget {
					continue
				}
				if sp.IsNetwork() {
					m.logger.Printf("[INFO] [backup] Tier 2 %s: pinned target %s is network storage — invalid (F-6C-1); falling through to auto", stackName, sp.Path)
					sawNetworkCandidate = true
					break
				}
				if sp.Path == sourceDrive {
					break
				}
				rel, why := m.diskRelation(sourceDrive, sp.Path)
				if rel == DiskSame {
					break // pinned target is on the same physical disk — not off-drive; fall through (`16` D1)
				}
				label := sp.Label
				if label == "" {
					label = filepath.Base(sp.Path)
				}
				return &Tier2Target{
					NamespaceRoot:  NamespaceRoot(sp.Path, true),
					Label:          label,
					Reason:         m.note("note.tier2.reason_manual"),
					Unproven:       rel == DiskUnknown,
					UnprovenReason: unprovenWhy(rel, why),
				}, nil
			}
		}
	}

	// 1. Prefer another registered user-data drive on a DIFFERENT physical disk, NON-network. A drive the
	//    agent PROVES is another disk wins; failing that, the first drive it cannot judge (`16` D2: the copy
	//    is made, not counted). A drive on the SAME disk is never a target (`16` D1).
	if m.settings != nil {
		var firstUnknown *Tier2Target
		for _, sp := range m.settings.GetSchedulableStoragePaths() {
			if sp.Path == sourceDrive {
				continue
			}
			// R-1179: a drive marked away is never a target. Its mount point is an empty folder on the guest's root
			// disk (or the host's), so the night's copy would fill the wrong disk and not be where the household
			// thinks. Decommissioned drives are already left out by GetSchedulableStoragePaths.
			if sp.Disconnected {
				continue
			}
			if sp.IsNetwork() {
				sawNetworkCandidate = true // F-6C-1: never a tier-2 target
				continue
			}
			rel, why := m.diskRelation(sourceDrive, sp.Path)
			if rel == DiskSame {
				continue
			}
			label := sp.Label
			if label == "" {
				label = filepath.Base(sp.Path)
			}
			t := &Tier2Target{
				NamespaceRoot:  NamespaceRoot(sp.Path, true), // Model A: in-guest mount IS the namespace root
				Label:          label,
				Reason:         m.note("note.tier2.reason_other_drive"),
				Unproven:       rel == DiskUnknown,
				UnprovenReason: unprovenWhy(rel, why),
			}
			if rel == DiskDifferent {
				return t, nil
			}
			if firstUnknown == nil {
				firstUnknown = t
			}
		}
		if firstUnknown != nil {
			return firstUnknown, nil
		}
	}

	// 2. Fall back to the internal SSD (system data path) — STATE-ONLY set only.
	sys := m.systemDataPath
	sysRel, sysWhy := DiskSame, ""
	if sys != "" {
		sysRel, sysWhy = m.diskRelation(sourceDrive, sys)
	}
	if sys == "" || sysRel == DiskSame {
		if sawNetworkCandidate {
			return nil, errTier2NetworkOnly // the only off-disk candidate was a NAS
		}
		return nil, errNoOffDiskTarget // single drive / app already on the system disk
	}
	fits := m.tier2FitsSystemDrive
	if m.tier2SSDFits != nil {
		fits = m.tier2SSDFits
	}
	if !fits(sys, stateOnlySize) {
		return nil, errSSDNoHeadroom // would fill the ~8 GB rootfs — refuse, don't fill
	}
	return &Tier2Target{
		NamespaceRoot: NamespaceRoot(sys, false), // system path is a real root → felhom-data appended
		Label:         m.note("note.tier2.label_internal_ssd"),
		IsSystemDrive: true,
		StateOnly:     true,
		Reason:        m.note("note.tier2.reason_no_second"),
		// `16` D2: an app on a drive the agent cannot judge against the system disk — made, not counted.
		Unproven:       sysRel == DiskUnknown,
		UnprovenReason: unprovenWhy(sysRel, sysWhy),
	}, nil
}

// unprovenWhy keeps the agent's reason only for an unknown verdict.
func unprovenWhy(rel DiskRelation, why string) string {
	if rel == DiskUnknown {
		return why
	}
	return ""
}

// tier2FitsSystemDrive checks the size-aware rootfs-headroom guard for the SSD target.
func (m *Manager) tier2FitsSystemDrive(sys string, unitSizeBytes int64) bool {
	di := system.GetDiskUsage(sys)
	if di == nil {
		return false // can't determine free space → refuse (fail-closed for the rootfs)
	}
	return tier2FitsHeadroom(di.AvailGB, di.TotalGB, float64(unitSizeBytes)/gibibyte)
}

// Tier-2 v2 layout (Task 3b, architecture §8): backups/secondary/<stack>/ holds
//
//	.felhom-tier2-layout   (marker file, content "2" — written LAST, after all legs + reconcile)
//	recovery-unit/         (the unit leg, layout-identical to v1)
//	hdd/<relpath>/         (per-bind HDD legs, relpath-mirroring)
//	userdata/<relpath>/    (per-bind USERDATA legs)
//
// Relpath-mirroring represents N>1 dirs + nested binds natively (the v1 flat-appdata N>1 refusal is
// lifted structurally) and makes restore position-derivable (dest relpath → live path under the app's
// current HDD_PATH). The whole tree is DERIVED from live data — migration is delete-and-rebuild.
const (
	tier2LayoutMarker  = ".felhom-tier2-layout"
	tier2LayoutVersion = "2"
)

// tier2SafeRemove os.RemoveAll's target ONLY if it is strictly WITHIN a backups/secondary/<...>
// destBase (§9.2 destructive-write boundary — defense in depth against a mapping bug ever pointing a
// removal at live data; red-proofed). Never removes destBase itself.
func tier2SafeRemove(destBase, target string) error {
	cb := filepath.Clean(destBase)
	ct := filepath.Clean(target)
	if !strings.Contains(filepath.ToSlash(cb), "/backups/secondary/") {
		return fmt.Errorf("refusing tier-2 removal: destBase %q is not under backups/secondary/", cb)
	}
	if !strings.HasPrefix(ct, cb+string(filepath.Separator)) {
		return fmt.Errorf("refusing tier-2 removal: %q is not strictly within %q", ct, cb)
	}
	return os.RemoveAll(ct)
}

// tier2RelClass classifies a dest dir (relpath dirRel, slash-form) against the current leg relpaths:
// keepInside = the dir is a leg or inside one (keep, don't descend); keepAncestor = on the path to a
// leg (keep, descend to find stale deeper); stale = neither (remove the topmost).
type tier2RelClass int

const (
	tier2Stale tier2RelClass = iota
	tier2KeepInside
	tier2KeepAncestor
)

func classifyTier2Rel(dirRel string, legRels []string) tier2RelClass {
	for _, lr := range legRels {
		if dirRel == lr || strings.HasPrefix(dirRel, lr+"/") {
			return tier2KeepInside // exact leg or descendant content
		}
	}
	for _, lr := range legRels {
		if strings.HasPrefix(lr, dirRel+"/") {
			return tier2KeepAncestor // on the path to a deeper leg
		}
	}
	return tier2Stale
}

// tier2Reconcile removes dest subdirs under hdd/ and userdata/ that are neither an ancestor nor a
// descendant of any current leg relpath (§7-D — the deferred-pruning answer: a bind removed/re-classed
// stops occupying the secondary drive within one run). Runs strictly inside destBase.
func (m *Manager) tier2Reconcile(destBase string, legRels []string) {
	m.tier2ReconcileRoots(destBase, []string{"hdd", "userdata"}, legRels)
}

// tier2ReconcileRoots is tier2Reconcile with the top-level dest roots supplied explicitly — a pure
// extraction so the R-7b shares dest (whose roots are per-source-drive keys, not hdd/userdata) can
// reuse the SAME staleness classification and the SAME destBase-bounded removal guard.
func (m *Manager) tier2ReconcileRoots(destBase string, roots, legRels []string) {
	var walk func(dirAbs, dirRel string)
	walk = func(dirAbs, dirRel string) {
		entries, err := os.ReadDir(dirAbs)
		if err != nil {
			return
		}
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}
			childRel := dirRel + "/" + e.Name()
			childAbs := filepath.Join(dirAbs, e.Name())
			switch classifyTier2Rel(childRel, legRels) {
			case tier2KeepInside:
				// mirrored leg content — keep, no descent
			case tier2KeepAncestor:
				walk(childAbs, childRel)
			default:
				if err := tier2SafeRemove(destBase, childAbs); err != nil {
					m.logger.Printf("[WARN] [backup] Tier 2 reconcile: %v", err)
				} else {
					m.logger.Printf("[INFO] [backup] Tier 2 reconcile: removed stale dest dir %s", childRel)
				}
			}
		}
	}
	for _, root := range roots {
		walk(filepath.Join(destBase, root), root)
	}
}

// RunTier2 makes/refreshes the off-drive v2 copy of a single app's recovery unit + its class-driven
// capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on
// an actual copy failure (no valid target is a recorded status, not an error).
func (m *Manager) RunTier2(stackName string) error {
	// Customer turned Tier 2 off for this app (config panel) — skip without touching status.
	if m.settings != nil {
		if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled {
			m.logger.Printf("[INFO] [backup] Tier 2 for %s skipped — disabled by customer", stackName)
			return nil
		}
	}
	sourceDrive := m.GetAppDrivePath(stackName)
	if sourceDrive == "" {
		return fmt.Errorf("no source drive for %s", stackName)
	}
	sourceNsRoot := m.namespaceRoot(sourceDrive)
	unitDir := RecoveryUnitPath(sourceNsRoot, stackName)
	if _, err := os.Stat(unitDir); err != nil {
		return nil // no recovery unit yet — nothing to copy
	}

	// Class-driven legs (classified) / resolver legs (legacy) + loud gap warnings (§7-H).
	legs, missingLegs, warns := m.tier2CaptureSetFull(stackName, sourceNsRoot)

	// Two sizes: full = unit + all legs; state-only = unit + mandatory legs (the SSD ceiling).
	unitSize := dirSizeBytes(unitDir)
	var fullSize, stateOnlySize int64 = unitSize, unitSize
	for _, lg := range legs {
		sz := dirSizeBytes(lg.Src)
		fullSize += sz
		if lg.Class == ClassMandatory {
			stateOnlySize += sz
		}
	}

	target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
	if err != nil {
		reason := m.tier2NoTargetReason(err)
		m.recordTier2NoTarget(stackName, reason)
		m.logger.Printf("[INFO] [backup] Tier 2 for %s: no off-drive target — %s", stackName, reason)
		return nil
	}
	// Defense-in-depth off-drive guard (selection already enforced it) — asked again right before the run,
	// so a disk swapped in the same bay is re-judged (`16` §3 "before each Tier-2 run").
	if rel, _ := m.diskRelation(sourceDrive, target.NamespaceRoot); rel == DiskSame {
		m.recordTier2NoTarget(stackName, m.note("note.tier2.same_disk"))
		return nil
	}

	// §2.2: the SSD is a STATE-ONLY tier — drop optional legs, tell the customer honestly.
	if target.StateOnly {
		kept := legs[:0]
		droppedOptional := false
		for _, lg := range legs {
			if lg.Class == ClassMandatory {
				kept = append(kept, lg)
			} else {
				droppedOptional = true
			}
		}
		legs = kept
		if droppedOptional {
			warns = append(warns, m.note("note.tier2.ssd_partial"))
		}
	}

	destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName)
	start := time.Now()
	mirror := m.tier2Mirror
	if mirror == nil {
		mirror = rsyncMirror
	}

	// Migration = delete-and-rebuild (marker LAST). First v2 run (marker absent): remove the old flat
	// appdata/ leg (recovery-unit/ is layout-identical — untouched). A half-migrated dest self-heals on
	// the next run because the marker is only written after every leg + reconcile succeed.
	markerPath := filepath.Join(destBase, tier2LayoutMarker)
	if _, mErr := os.Stat(markerPath); mErr != nil {
		if rmErr := tier2SafeRemove(destBase, filepath.Join(destBase, "appdata")); rmErr != nil && !os.IsNotExist(rmErr) {
			m.logger.Printf("[WARN] [backup] Tier 2 %s: migration cleanup of old flat appdata failed: %v", stackName, rmErr)
		}
	}

	// Unit leg — always, EXCEPT the one case R-403 measured (see r403_hollow.go for the mechanism).
	//
	// `mirror` is `rsync -a --delete`. That is correct for a derived copy and it stays. What was
	// missing is the precondition: a source unit that carries NO data must not be mirrored over a
	// destination unit that does, because `--delete` then removes the customer's last package. Proven
	// on demo-hp 2026-08-31: 120 082 104 B → 7 036 B in one run, reported as a success.
	//
	// The fence is EXACTLY this shape and no wider. Complete→complete, complete→hollow and
	// hollow→hollow all mirror as before; a data leg that legitimately shrinks is untouched (this
	// guard is on the unit leg only). §8 row 5's derived-copy rule is unchanged.
	destUnit := filepath.Join(destBase, "recovery-unit")
	unitLegSkipped := unitIsHollow(unitDir) && unitCarriesData(destUnit)
	if unitLegSkipped {
		// Loud, and it names the app, the reason and the consequence. Paths and counts only — a unit's
		// compose/app.yaml carries portable secrets and nothing from inside it is logged here.
		m.logger.Printf("[WARN] [backup] Tier 2 %s: unit leg SKIPPED — the recovery unit on the source drive lists no database dumps and no volume tars, while the existing copy at %s does. The copy was PRESERVED rather than replaced with an empty one (R-403). The other legs continue.",
			stackName, destUnit)
		warns = append(warns, m.note(tier2UnitPreservedKey))
	} else if err := mirror(unitDir, destUnit); err != nil {
		m.recordTier2Failure(stackName, target, err)
		if m.tier2Notify != nil {
			m.tier2Notify(stackName, target.Label, time.Since(start), err)
		}
		return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err)
	}
	// Capture legs.
	legRels := make([]string, 0, len(legs))
	mirroredSize := unitSize
	for _, lg := range legs {
		dst := filepath.Join(destBase, filepath.FromSlash(lg.DestRel))
		if m.emptiedSourceRefused(stackName+" "+lg.DestRel, lg.Src, dst) {
			legRels = append(legRels, lg.DestRel) // kept as it is (R-1181)
			warns = append(warns, m.note("note.tier2.emptied_kept", lg.DestRel))
			continue
		}
		if err := mirror(lg.Src, dst); err != nil {
			m.recordTier2Failure(stackName, target, err)
			if m.tier2Notify != nil {
				m.tier2Notify(stackName, target.Label, time.Since(start), err)
			}
			return fmt.Errorf("tier2 rsync leg %s for %s: %w", lg.DestRel, stackName, err)
		}
		legRels = append(legRels, lg.DestRel)
		mirroredSize += dirSizeBytes(lg.Src)
		m.releaseKeptCopy(dst)
	}
	// D25 / R-1181: a declared folder that is MISSING keeps its copy (kept + listed), never reconciled away.
	for _, lg := range missingLegs {
		legRels = append(legRels, lg.DestRel)
		m.keepCopy("app", stackName+": "+lg.DestRel, m.storageLabel(sourceDrive), "folder_missing", destBase,
			filepath.Join(destBase, filepath.FromSlash(lg.DestRel)))
	}

	// Reconcile stale dest dirs (a bind removed / re-classed excluded), then write the marker LAST.
	m.tier2Reconcile(destBase, legRels)
	if err := os.WriteFile(markerPath, []byte(tier2LayoutVersion), 0644); err != nil {
		m.logger.Printf("[WARN] [backup] Tier 2 %s: layout marker write failed (restore will refuse until next run): %v", stackName, err)
	}

	dur := time.Since(start)
	// R-403: the package date is read from the DESTINATION unit's own manifest, so it describes what
	// is actually in the copy whether the leg mirrored or was preserved. Reading the artifact rather
	// than assuming the run's own timestamp is what stops the surface calling a preserved package
	// fresh — a data loss traded for a comforting lie is not a fix.
	m.recordTier2SuccessWithUnit(stackName, target, mirroredSize, strings.Join(warns, " "), dur,
		unitLegSkipped, unitPackageDate(destUnit))
	if m.tier2Notify != nil {
		m.tier2Notify(stackName, target.Label, dur, nil)
	}
	m.logger.Printf("[INFO] [backup] Tier 2 copied %s → %s (%s, %d leg(s), %s)%s%s",
		stackName, destBase, humanizeBytes(mirroredSize), len(legs), dur.Round(time.Second),
		map[bool]string{true: " [SSD: state-only]", false: ""}[target.StateOnly],
		map[bool]string{true: " [unit leg SKIPPED — existing package preserved, R-403]", false: ""}[unitLegSkipped])
	return nil
}

// RunAllTier2 runs Tier 2 for every deployed HDD app (apps whose data lives on an external drive —
// non-HDD apps live on the rootfs and are already inside the PBS whole-guest snapshot).
func (m *Manager) RunAllTier2() {
	if m.stackProvider == nil {
		return
	}
	if m.migrationActive() {
		m.logger.Printf("[INFO] [backup] Tier 2 kihagyva: migráció folyamatban")
		return
	}
	var n int
	for _, stack := range m.stackProvider.ListDeployedStacks() {
		// Reserved-name defense in depth (R-7b): the shares source owns backups/secondary/_shares and
		// the _shares status record. Stack names come from the git-synced catalog, not customer input,
		// so this cannot realistically fire — but if it ever did, the app would silently overwrite the
		// shares tree, so it is refused loudly instead.
		if stack.Name == SharesPseudoStack {
			m.logger.Printf("[ERROR] [backup] Tier 2: stack %q uses the RESERVED shares key — skipped to protect the shares backup tree", stack.Name)
			continue
		}
		// F6 (CAMPAIGN-3): volume-only apps (no HDD_PATH, backups on sys_drive) previously got NO
		// tier-2 copy — a single controller-level copy on one device. They now flow through too: their
		// recovery unit (which holds the db/volume dumps) gets a cross-drive second copy like any HDD
		// app. Apps with no recovery unit yet (infra/never-backed-up) are a cheap no-op inside RunTier2
		// (the `os.Stat(unitDir)` guard), so this doesn't spuriously copy protected stacks.
		if m.settings != nil && (m.settings.IsDisconnected(m.GetAppDrivePath(stack.Name)) ||
			m.settings.IsDecommissioned(m.GetAppDrivePath(stack.Name))) {
			continue
		}
		// Slice 4: never mirror over a HELD app's copy — it is the restore point the hold text names.
		// See the matching skip in captureAllRecoveryUnits.
		if m.isHeld(stack.Name) {
			m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — the app is HELD; its copy is the restore point and is preserved", stack.Name)
			continue
		}
		// R-1180: never mirror an app while a restore of it runs — the mirror (rsync --delete) would remove files the
		// restore is still bringing back from that same copy. Skipped and said; the next night mirrors it.
		if m.restoreRunningFor(stack.Name) {
			m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — a restore of it is running; it is mirrored at the next run (R-1180)", stack.Name)
			continue
		}
		// R-645: the mirror of an app not running its pinned version stays as it is.
		if why, skip := m.versionSkip(stack.Name); skip {
			m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — it is not running its pinned version (%s); the last good copy is kept (R-645)", stack.Name, why)
			continue
		}
		runOne := m.perAppTier2
		if runOne == nil {
			runOne = m.RunTier2
		}
		if err := runOne(stack.Name); err != nil {
			m.logger.Printf("[WARN] [backup] Tier 2 failed for %s: %v", stack.Name, err)
		}
		n++
	}
	m.logger.Printf("[INFO] [backup] Tier 2 run complete: %d app(s) processed (incl. volume-only — F6)", n)

	// R-7b: the SHARES source runs after the per-app loop, in the SAME orchestrator run. It is a
	// sibling job — nothing above it changed — and its failure never fails the app tier.
	if m.restoreRunningFor(SharesPseudoStack) {
		m.logger.Printf("[WARN] [backup] Tier 2 shares job skipped — a restore of the shares is running; mirrored at the next run (R-1180)")
		return
	}
	if err := m.RunSharesTier2(); err != nil {
		m.logger.Printf("[WARN] [backup] Tier 2 shares job failed: %v", err)
	}
}

// --- per-app config-panel view (drives the Tier-2 "Beállítás" page) ---

// Tier2Option is one selectable off-drive destination in the config panel.
type Tier2Option struct {
	Path  string // registered storage path (the value persisted as PreferredTarget)
	Label string // human label for the dropdown
}

// Tier2Info is the per-app Tier-2 view the config panel renders. It exposes the effective target
// (pinned or auto), whether that is the size-limited internal SSD, the honest no-target reason, and
// the off-disk drives the customer may pin — so the control is meaningful even with a single target.
type Tier2Info struct {
	IsHDDApp       bool   // false = the app lives on the rootfs (already inside the PBS whole-guest snapshot)
	SourceDrive    string // where the app's data currently lives
	Disabled       bool   // customer turned Tier 2 off
	Preferred      string // customer-pinned target path ("" = automatic)
	EffectiveLabel string // label of the target that WOULD be used right now
	EffectiveIsSSD bool   // the effective target is the internal SSD (DB/config only)
	EffectiveDesc  string // why this target (Hungarian)
	NoTarget       bool   // no off-drive target fits at all
	NoTargetReason string // honest reason when NoTarget
	Alternatives   []Tier2Option
}

// Tier2Info builds the config-panel view for one app. Read-only (no status writes).
func (m *Manager) Tier2Info(stackName string) Tier2Info {
	var info Tier2Info
	if m.stackProvider != nil {
		info.IsHDDApp = m.stackProvider.GetStackHDDPath(stackName) != ""
	}
	source := m.GetAppDrivePath(stackName)
	info.SourceDrive = source

	if m.settings != nil {
		if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil {
			info.Disabled = cd.UserDisabled
			info.Preferred = cd.PreferredTarget
		}
		// Eligible alternative drives: registered, schedulable, on a DIFFERENT physical disk.
		for _, sp := range m.settings.GetSchedulableStoragePaths() {
			if sp.Path == source {
				continue
			}
			if rel, _ := m.diskRelation(source, sp.Path); rel == DiskSame {
				continue
			}
			label := sp.Label
			if label == "" {
				label = filepath.Base(sp.Path)
			}
			info.Alternatives = append(info.Alternatives, Tier2Option{Path: sp.Path, Label: label})
		}
	}

	// Resolve what the runner WOULD pick right now: the v2 capture legs feed both sizes (the SSD
	// headroom guard tests the state-only set).
	sourceNsRoot := m.namespaceRoot(source)
	legs, _ := m.tier2CaptureSet(stackName, sourceNsRoot)
	unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName))
	fullSize, stateOnlySize := unitSize, unitSize
	for _, lg := range legs {
		sz := dirSizeBytes(lg.Src)
		fullSize += sz
		if lg.Class == ClassMandatory {
			stateOnlySize += sz
		}
	}
	target, err := m.selectTier2Target(stackName, fullSize, stateOnlySize)
	if err != nil {
		info.NoTarget = true
		info.NoTargetReason = m.tier2NoTargetReason(err)
		return info
	}
	info.EffectiveLabel = target.Label
	info.EffectiveIsSSD = target.IsSystemDrive
	info.EffectiveDesc = target.Reason
	return info
}

// --- status persistence (drives the "2. mentés" UI card) ---

// tier2Update applies a run outcome onto a COPY OF THE EXISTING ROW, then persists it.
//
// R-101 Part 2 — SAFE BY CONSTRUCTION, and this replaced a real hazard rather than tidying one. The
// three record* helpers each used to build a WHOLE `CrossDriveBackup` literal, with `withTier2Prefs`
// re-applying exactly two fields (UserDisabled, PreferredTarget). Every other field not named in the
// literal was silently zeroed on every status write. That is fine while the struct is stable and
// catastrophic the moment a field is added: R-101 adds `LastSuccess`, and under the old shape
// `recordTier2Failure` would have CLEARED it — the mirror-image of the defect being fixed, firing on
// the first failure instead of lying dormant.
//
// Starting from the existing row inverts the default: a new field carries over unless a caller
// deliberately overwrites it. The compile-safe form the R-100 review asked for; nothing is preserved
// by a list that can fall out of date.
//
// Callers must therefore CLEAR explicitly what a run invalidates (a stale LastError on success, a
// stale size on failure) — the old behaviour those clears reproduce is preserved exactly.
func (m *Manager) tier2Update(stackName string, mutate func(*settings.CrossDriveBackup)) {
	if m.settings == nil {
		return
	}
	var cfg settings.CrossDriveBackup
	if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
		cfg = *existing // value copy — EVERY field carries over by default
	}
	// One-time migration of a pre-anchor row. Under the old code `LastStatus=="ok"` with a LastRun
	// means that run DID succeed, so adopting it as the initial anchor is truthful — and it is what
	// keeps the deploy quiet: without it every existing row would flip to "never succeeded" at once
	// (all 7 rows on the fleet were pre-anchor). A row whose last known state was an ERROR seeds
	// nothing, because nothing in the old data evidences a success.
	if !cfg.SuccessTracked {
		if cfg.LastStatus == "ok" && cfg.LastRun != "" {
			cfg.LastSuccess = cfg.LastRun
		}
		cfg.SuccessTracked = true
	}
	mutate(&cfg)
	if err := m.settings.SetCrossDriveConfig(stackName, &cfg); err != nil {
		m.logger.Printf("[WARN] [backup] Tier 2 status persist for %s failed: %v", stackName, err)
	}
}

// recordTier2Success records a completed run for a path that HAS NO UNIT LEG — the shares
// pseudo-stack mirrors share legs and carries no recovery unit, so there is no leg to skip and no
// package whose date could be older than the run. It is the thin caller; ONE implementation lives
// below.
func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) {
	m.recordTier2SuccessWithUnit(stackName, target, sizeBytes, warning, dur, false, "")
}

// recordTier2SuccessWithUnit records a completed run INCLUDING what happened to its unit leg.
// `unitLegSkipped` and `unitPkgDate` (R-403) travel with the rest rather than through a second write,
// because two writers to one record is how a status and the thing it describes drift apart.
// `unitPkgDate` is read from the destination unit's OWN manifest, so it is a fact about the copy and
// not about the run.
func (m *Manager) recordTier2SuccessWithUnit(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration, unitLegSkipped bool, unitPkgDate string) {
	now := time.Now().Format(time.RFC3339)
	m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
		c.UnitLegSkipped = unitLegSkipped
		c.UnitPackageDate = unitPkgDate
		c.Enabled = true
		c.Method = "rsync"
		c.DestinationPath = target.NamespaceRoot
		c.Schedule = "daily"
		c.LastRun = now
		// R-101: the anchor. Only this branch advances it; no failure branch clears it.
		c.LastSuccess = now
		c.LastStatus = "ok"
		c.LastWarning = strings.TrimSpace(warning)
		c.LastDuration = dur.Round(time.Second).String()
		c.LastSizeHuman = humanizeBytes(sizeBytes)
		c.LastError = "" // a success invalidates the previous error
		// `16` D2: a copy on a disk the agent cannot prove is another one — kept, not counted.
		c.Unproven = target.Unproven
		c.UnprovenReason = target.UnprovenReason
	})
}

func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) {
	m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
		c.Enabled = true
		c.Method = "rsync"
		c.DestinationPath = target.NamespaceRoot
		c.Schedule = "daily"
		c.LastRun = time.Now().Format(time.RFC3339) // the ATTEMPT clock — advances on failure, by design
		c.LastStatus = "error"
		c.LastError = cause.Error()
		// LastSuccess is deliberately UNTOUCHED: a failure neither advances nor clears the anchor.
		// Clearing it would make one bad night read as "no copy has ever succeeded".
		c.LastWarning = ""   // a warning from the last successful run does not describe this one
		c.LastDuration = ""  // preserving the old literal's clears exactly
		c.LastSizeHuman = "" // ditto — a stale size would describe a copy this run did not make
	})
}

func (m *Manager) recordTier2NoTarget(stackName, reason string) {
	m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
		c.Enabled = false
		c.Method = "rsync"
		c.Schedule = "daily"
		c.DestinationPath = ""
		c.LastStatus = "no_target"
		c.LastError = reason
		c.LastRun = ""
		// LastSuccess survives: "there is no destination drive right now" is not evidence that the
		// last successful copy never happened. The UI gates on LastRun here, so nothing is rendered.
		c.LastWarning = ""
		c.LastDuration = ""
		c.LastSizeHuman = ""
	})
}

// tier2NoTargetReason is SAVED (CrossDriveConfig.LastError / Tier2Info.NoTargetReason) and read on the
// per-app card later, so it is written in the box's language at write time (release C, R-557). The
// branch is on a SENTINEL, never on these words (R-553) — which is why translating them is safe.
func (m *Manager) tier2NoTargetReason(err error) string {
	switch {
	case errors.Is(err, errSSDNoHeadroom):
		return m.note("note.tier2.no_space_ssd")
	case errors.Is(err, errNoOffDiskTarget):
		return m.note("note.tier2.no_other_drive")
	default:
		return m.noteErr(err)
	}
}

// --- helpers ---

// rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning).
//
// R-1214 (night soak 2026-10-21; measured on 9202 2026-10-22, felhom.eu `audits/night-2026-10-22/P4-R1214/`): run
// in the controller's own process, the copy's page cache is charged to the GUEST's memory and evicts the apps' working
// memory — on a 4 GB guest with 3.5 GB in use a 6 GB copy drove memory „full" pressure to 6.9–16.8 % (10-s average,
// 2.5–5.7 s of full stall); the same copy inside a container capped at 512 MB: 0.9–1.0 %, 0.3–0.4 s. So the copy runs
// in a capped container of the controller's own image (it carries rsync) — ONLY when both paths are under /mnt/,
// where the controller and the Docker daemon see the same paths. Any other path (the controller's own data volume)
// differs for the daemon: a capped `--delete` there would mirror an EMPTY folder over the copy. Those paths, a box
// whose image cannot be read, and a container that cannot START (docker's own exit 125) take the plain rsync, as
// before. Pinned by TestR1214_*.
//
// The container must never outlive the copy (the stray-copy fix, tier2_copy_lifecycle.go): it has a fixed name and a
// label; a leftover of the same copy is removed first; on any error — the timeout included, which kills only the
// docker CLI — it is removed by name. Every copy, capped or in-process, is in the controller's record of copies while
// it runs, and a restore is refused meanwhile. Pinned by TestStrayCopy_*.
func rsyncMirror(src, dst string) error {
	if err := os.MkdirAll(dst, 0755); err != nil {
		return fmt.Errorf("mkdir %s: %w", dst, err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), tier2CopyDeadline())
	defer cancel()
	// Trailing slashes: copy the CONTENTS of src into dst.
	s, d := strings.TrimRight(src, "/"), strings.TrimRight(dst, "/")
	rs := []string{"-a", "--delete", s + "/", d + "/"}
	name, app := tier2CopyName(d)
	tier2CopyBegin(app)
	defer tier2CopyEnd(app)
	if tier2CappedPath(s) && tier2CappedPath(d) {
		if img := tier2SelfImage(); img != "" {
			// A leftover of this same copy (one a removal could not stop) never runs beside a new one.
			if !tier2RemoveCopy(name, "a leftover of the same copy, before a new one") {
				return fmt.Errorf("a leftover copy container %s could not be removed — not copying beside it", name)
			}
			args := append([]string{"run", "--rm", "--name", name, "--label", tier2CopyLabel + "=1",
				"--label", tier2CopyAppLabel + "=" + tier2DockerSafe(app),
				"--memory", tier2CopyMemCap, "--memory-swap", tier2CopyMemCap,
				"--network", "none", "--pull", "never", "--no-healthcheck", // the image's controller health check means nothing here
				// --mount (not -v): a path the daemon does not see is REFUSED (exit 125 → the in-process copy) instead of
				// created empty — an empty source would let `--delete` empty the copy (review 2026-10-22 night, C4).
				"--mount", "type=bind,src=" + s + ",dst=" + s + ",readonly", "--mount", "type=bind,src=" + d + ",dst=" + d,
				"--entrypoint", "rsync", img}, rs...)
			out, code, err := tier2RunCapped(ctx, args)
			if err == nil {
				return nil
			}
			// The docker CLI ending does not stop the container (a timeout kills only the CLI): remove it by name.
			removed := tier2RemoveCopy(name, "the copy ended with an error")
			if ctx.Err() != nil {
				tier2Logf("[WARN] [tier2] the copy into %s ran past its time limit (%s) — its container %s was removed (removed=%v)", d, tier2CopyDeadline(), name, removed)
				return fmt.Errorf("copy timed out after %s: %v: %s", tier2CopyDeadline(), err, strings.TrimSpace(string(out)))
			}
			if code != 125 {
				return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
			}
			if !removed {
				return fmt.Errorf("the copy container %s did not start and could not be removed: %s", name, strings.TrimSpace(string(out)))
			}
			tier2Logf("[WARN] [tier2] the capped copy container did not start (%s) — copying in-process, as before (R-1214)", strings.TrimSpace(string(out)))
		}
	}
	cmd := exec.CommandContext(ctx, "rsync", rs...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}

// tier2CopyMemCap bounds the capped copy's memory: rsync itself needs a few MB per 100 000 files; the rest is the page
// cache the copy creates, reclaimed inside the cap instead of from the apps.
const tier2CopyMemCap = "512m"

// tier2CappedPath: a path the controller and the Docker daemon see identically (the guest's /mnt is bind-mounted into
// the controller at /mnt). Never a relative path, never `..`, and NO SYMLINK anywhere in it: a link under /mnt that
// points outside /mnt resolves to a different place for the daemon (found by the commit's security review,
// 2026-10-22) — the path must resolve to itself. The path must exist (rsyncMirror creates dst first).
func tier2CappedPath(p string) bool {
	if !strings.HasPrefix(p, tier2CappedRoot) || filepath.Clean(p) != p {
		return false
	}
	real, err := filepath.EvalSymlinks(p)
	return err == nil && real == p
}

// tier2CappedRoot is "/mnt/" in production (a test seam: tests point it at a temp dir).
var tier2CappedRoot = "/mnt/"

// Seams (tests never reach Docker — tests that do act on the host they run on).
var (
	tier2SelfImage = func() string {
		// The image ID, not its tag: a tag can be removed or re-pointed; with --pull never the copy runs exactly the
		// image the controller runs (review 2026-10-22 night, C7).
		out, err := dockerexec.Command("docker", "inspect", "felhom-controller", "--format", "{{.Image}}").Output()
		if err != nil {
			return ""
		}
		return strings.TrimSpace(string(out))
	}
	tier2RunCapped = func(ctx context.Context, args []string) ([]byte, int, error) {
		out, err := dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput()
		var ee *exec.ExitError
		if errors.As(err, &ee) {
			return out, ee.ExitCode(), err
		}
		return out, 0, err
	}
	tier2Logf = log.Printf
)

// tier2UnitPreservedWarning is the customer-facing half of the R-403 skip. It rides in
// CrossDriveBackup.LastWarning, which the per-app card already renders, so the refusal reaches the
// SURFACE and not only the log — the shape recordTier2NoTarget established.
const tier2UnitPreservedKey = "note.tier2.unit_preserved"

// unitPackageDate returns the `created_at` of a recovery unit's manifest — WHEN the package in that
// directory was captured. "" when the manifest is absent or unparseable, which the surface must read
// as UNKNOWN and never as "now".
func unitPackageDate(unitDir string) string {
	if man := readManifest(UnitManifestFile(unitDir)); man != nil {
		return man.CreatedAt
	}
	return ""
}

// dirSizeBytes returns the total size of a directory via `du -sb` (0 if absent/error).
func dirSizeBytes(dir string) int64 {
	if _, err := os.Stat(dir); err != nil {
		return 0
	}
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, "du", "-sb", dir).Output()
	if err != nil {
		return 0
	}
	fields := strings.Fields(string(out))
	if len(fields) == 0 {
		return 0
	}
	var size int64
	if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil {
		return 0
	}
	return size
}

// restoreRunningFor reports whether a restore of stack is in flight (the restore record the pages follow, R-550).
// The shares restores are recorded under their display name.
func (m *Manager) restoreRunningFor(stack string) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	if !m.opRunning {
		return false
	}
	if stack == SharesPseudoStack {
		return m.opStack == SharesDisplayName || m.opStack == SharesPseudoStack
	}
	return m.opStack == stack
}