Offsite tier policy engine: mandatory userdata, raw-data quota, restore rework (Task 3a, v0.134.0)

Each toggled app's offsite push = one multi-path restic snapshot (recovery unit + TierOffsite
mandatory userdata via ComputeCaptureSet); legacy/undeployed stay unit-only. Loud capture gaps
(SP-3.4: restic 0.14.0 silently skips missing paths). Quota = stats --mode raw-data (SP-1;
displayed size drops once). Pre-push enlargement gate blocks the userdata enlargement over-quota
(unit-only push continues; EnlargedBlocked; edge-triggered notify). forget --group-by host,tags
on both sites (SP-2). Restore reworked: scratch off the rootfs + headroom gate (F-A1), unit-only
default via --include, size-first full, place-to-live missing-only merge (never --delete).
UI: unit/full-two-step/place actions + per-app blocked note; route POST /backup/offbox/place.
HUB FLAG: offbox_enlarge_blocked event needs hub allowlist for push delivery.
+13 tests; all 10 §10 red-proofs verified. No tier-2/.fab/hub/agent changes.
This commit is contained in:
2026-07-14 22:51:54 +02:00
parent 0c6e151c1c
commit 2d20859858
17 changed files with 1413 additions and 109 deletions
+15
View File
@@ -36,6 +36,21 @@ type Manager struct {
offboxRunner offboxRunner
offboxNotify func(dur time.Duration, snapshots int, err error)
// offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable
// in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb).
offboxSizer func(path string) int64
// offboxEnlargeBlockedNotify (3a), if set, is called ONCE per app that NEWLY enters the
// quota-blocked (enlargement-refused) state — edge-triggered against the persisted EnlargedBlocked
// set so a nightly schedule can't re-notify a persistently-blocked app (the hub owns cooldown; the
// controller must not add a timer). Wired in cmd/controller/main.go.
offboxEnlargeBlockedNotify func(stack string, estBytes int64, usedGB, quotaGB int)
// offboxPlaceCopier (3a) — the place-to-live missing-only merge seam (nil → rsyncRestoreMissing,
// the `-a --ignore-existing` additive copy). Never rsyncMirror (--delete trap).
offboxPlaceCopier func(src, dst string) (int, error)
// offboxFreeFn (3a) — the free-space probe for the restore headroom gate, overridable in tests (the
// Windows `go test` host has no `df`). Nil → the real diskFreeBytes (df --output=avail).
offboxFreeFn func(path string) int64
// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
discoverDBs func(ctx context.Context) ([]DiscoveredDB, error)
+115 -21
View File
@@ -11,6 +11,7 @@ import (
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
@@ -52,6 +53,23 @@ func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err
m.offboxNotify = fn
}
// SetOffboxSizer overrides the mandatory-set byte estimator (tests). SetOffboxEnlargeBlockedNotifier
// wires the edge-triggered enlargement-blocked notification (main.go). SetOffboxPlaceCopier overrides
// the place-to-live missing-only merge (tests).
func (m *Manager) SetOffboxSizer(fn func(path string) int64) { m.offboxSizer = fn }
func (m *Manager) SetOffboxEnlargeBlockedNotifier(fn func(stack string, estBytes int64, usedGB, quotaGB int)) {
m.offboxEnlargeBlockedNotify = fn
}
func (m *Manager) SetOffboxPlaceCopier(fn func(src, dst string) (int, error)) { m.offboxPlaceCopier = fn }
// offboxSize returns the mandatory-set byte estimator (nil seam → the real du -sb dirSizeBytes).
func (m *Manager) offboxSize() func(string) int64 {
if m.offboxSizer != nil {
return m.offboxSizer
}
return dirSizeBytes
}
func (m *Manager) runner() offboxRunner {
if m.offboxRunner != nil {
return m.offboxRunner
@@ -395,6 +413,14 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
apps := m.settings.GetOffboxApps()
t := m.settings.GetOffboxTarget()
base, env := m.offboxBaseArgs(t)
// Edge-trigger for the enlarge-blocked notification: capture the PRIOR blocked set so we notify only
// apps that NEWLY cross into the blocked state (a persistently-blocked app doesn't re-notify nightly).
priorBlocked := map[string]bool{}
if t != nil {
for _, s := range t.EnlargedBlocked {
priorBlocked[s] = true
}
}
start := time.Now()
m.logger.Printf("[INFO] [offbox] backup run started (%d app(s) toggled)", len(apps))
if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" }); err != nil {
@@ -403,6 +429,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
var backedUp int
var missing []string
var runResult offboxRunResult
var runErr error
if usedGB, quota, over := offboxQuotaState(t); over {
// SLICE 4 soft-quota gate (pre-run): NEW backups are refused at ≥100% of the shared-model quota —
@@ -414,8 +441,16 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
m.offboxRecordStats(ctx, base, env) // the prune may have brought the size back down — refresh
runErr = fmt.Errorf("A távoli mentés túllépte a tárhelykeretet (%d/%d GB) — törölj régi mentéseket vagy kérj nagyobb keretet.", usedGB, quota)
} else {
backedUp, missing, runErr = m.runOffboxInternal(ctx, apps, base, env)
runResult, runErr = m.runOffboxInternal(ctx, apps, base, env, t)
backedUp = runResult.backedUp
missing = runResult.missing
}
// Sorted names of apps whose enlargement was blocked this run (replaces the persisted set; empty clears).
var blockedNames []string
for _, b := range runResult.blocked {
blockedNames = append(blockedNames, b.stack)
}
sort.Strings(blockedNames)
// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
@@ -440,6 +475,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
o.LastStatus = "ok"
o.LastError = ""
o.SnapshotCount = snapshots
o.EnlargedBlocked = blockedNames // replace each run (sorted); empty slice clears it
var warns []string
// Zero-toggle honesty (take-two obs.): a configured target with NOTHING selected reports
// its emptiness instead of a bare success — the customer thinks offsite runs, but nothing
@@ -451,6 +487,13 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
warns = append(warns, fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
len(missing), strings.Join(missing, ", ")))
}
// 3a: capture-gap warnings (structurally-refused / on-disk-missing mandatory paths, undeployed).
warns = append(warns, runResult.warns...)
// 3a: the pre-push enlargement gate blocked some apps' userdata — config+DB still saved.
if len(blockedNames) > 0 {
warns = append(warns, fmt.Sprintf("Figyelmeztetés: a tárhelykeret miatt %d alkalmazásnál csak konfiguráció- és adatbázis-mentés készült: %s.",
len(blockedNames), strings.Join(blockedNames, ", ")))
}
// SLICE 4: approaching the soft quota (≥80%, <100%) — warn on an otherwise-OK run.
if qw := offboxQuotaWarning(o); qw != "" {
warns = append(warns, qw)
@@ -463,6 +506,17 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
if m.offboxNotify != nil {
m.offboxNotify(dur, snapshots, runErr)
}
// Edge-triggered enlarge-blocked notification: only apps that NEWLY crossed into the blocked state
// (vs the prior persisted set) notify — a persistently-blocked app never re-notifies nightly. Uses
// the pre-run last-known repo size (the same figure the gate used).
if runErr == nil && m.offboxEnlargeBlockedNotify != nil && t != nil && t.QuotaGB > 0 {
usedGB := int(t.RepoSizeBytes / offboxGiB)
for _, b := range runResult.blocked {
if !priorBlocked[b.stack] {
m.offboxEnlargeBlockedNotify(b.stack, b.estBytes, usedGB, t.QuotaGB)
}
}
}
switch {
case runErr != nil:
m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
@@ -553,12 +607,24 @@ func offboxUnitTime(src, manifestPath string) time.Time {
return time.Time{}
}
// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
// offboxRunResult carries the outcome of a per-app offbox run: how many apps were backed up, which
// had no discoverable unit (skipped), which had their enlargement quota-blocked (unit-only), and the
// aggregated Hungarian customer warnings (capture gaps + undeployed).
type offboxRunResult struct {
backedUp int
missing []string
blocked []offboxBlocked
warns []string
}
// runOffboxInternal does the repo-ensure + per-app DISCOVER → capture-set → gate → multi-path backup +
// prune. Caller holds the running flag. Each app's snapshot is ONE multi-path restic snapshot
// (recovery unit + the app's MANDATORY offsite capture set, §6). The pre-push enlargement gate (§9,
// decision #1) blocks only the ENLARGEMENT — the unit-only push always continues. Returns the result +
// the first hard error (repo-ensure or a restic backup exec failure).
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string, t *settings.OffboxTarget) (res offboxRunResult, err error) {
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
return 0, nil, rerr // fail fast (dead NAS surfaces here)
return res, rerr // fail fast (dead NAS surfaces here)
}
// Pre-run hygiene: clear any lock restic can prove stale before we start (cheap; the --remove-all
// crash-lock escalation lives in resticStep for the locks restic can't self-detect).
@@ -568,11 +634,30 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
src, ok := m.discoverOffboxUnit(stack)
if !ok {
m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
missing = append(missing, stack)
res.missing = append(res.missing, stack)
continue
}
// Task 3-core TierOffsite capture set: mandatory userdata paths added to the unit snapshot,
// plus loud warnings for structurally-refused / on-disk-missing mandatory paths (SP-3.4).
extra, capWarns := m.offboxCaptureSet(stack)
res.warns = append(res.warns, capWarns...)
// Pre-push enlargement gate (§9): if last-known repo raw-data bytes + the mandatory-set estimate
// would cross the soft quota, push UNIT-ONLY (protection never regresses) and record the block.
if len(extra) > 0 && t != nil && t.QuotaGB > 0 {
var est int64
for _, p := range extra {
est += m.offboxSize()(p)
}
if t.RepoSizeBytes+est >= int64(t.QuotaGB)*offboxGiB {
m.logger.Printf("[INFO] [offbox] %s: enlargement blocked by quota (est %s + repo %s ≥ %d GB) — unit-only push continues",
stack, humanizeBytes(est), humanizeBytes(t.RepoSizeBytes), t.QuotaGB)
res.blocked = append(res.blocked, offboxBlocked{stack: stack, estBytes: est})
extra = nil
}
}
args := append([]string{"backup", "--tag", "felhom-offbox", "--tag", stack, src}, extra...)
bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, "backup", "--tag", "felhom-offbox", "--tag", stack, src)
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, args...)
cancel()
if berr != nil {
m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
@@ -581,25 +666,29 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
}
continue
}
backedUp++
m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
res.backedUp++
m.logger.Printf("[INFO] [offbox] backed up %s (%s, %d mandatory path(s))", stack, src, len(extra))
}
if firstErr != nil {
return backedUp, missing, firstErr
return res, firstErr
}
// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
// prune takes an EXCLUSIVE lock — the exact step whose crash left the C2 stale lock — so it goes
// through resticStep for the --remove-all self-heal too.
// Retention: keep a sane window, prune the rest. SP-2: `--group-by host,tags` so an app's OLD
// unit-only-shape snapshots share a group with its NEW enlarged shape (same <stack> tag) and age
// out naturally — the default host,paths grouping would strand old-shape snapshots in their own
// permanently-retained group. prune takes an EXCLUSIVE lock (the C2 stale-lock step) → resticStep.
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
defer cancel()
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
}
return backedUp, missing, nil
return res, nil
}
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic restore-size.
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic REPO SIZE. Since
// v0.134.0 the repo size is measured with `stats --mode raw-data` (actual deduplicated+compressed
// bytes — what the customer's Storage Box really fills), NOT the old modeless restore-size which
// multiplied by the retained-snapshot count (SP-1). The displayed size drops one-time after deploy.
const offboxGiB = int64(1) << 30
// OffboxReportStatus is the NON-SECRET offsite summary carried on the hub report (SLICE 4) — the input
@@ -673,7 +762,9 @@ func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
}
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
defer cancel()
fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
// SP-2: `--group-by host,tags` (mirrors runOffboxInternal's forget) so old unit-only-shape snapshots
// age out with the enlarged shape instead of stranding in a permanently-retained host,paths group.
fargs := append(append([]string{}, base...), "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
} else {
@@ -695,9 +786,12 @@ func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int
if json.Unmarshal(out, &snaps) != nil {
return 0
}
// Repo size (best-effort, restore-size). Bytes feed the soft-quota gate (SLICE 4); a failed stats
// call keeps the last-known value (stale-but-safe).
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--json")...); serr == nil {
// Repo size (best-effort, RAW-DATA mode). SP-1: `--mode raw-data` reports the actual
// deduplicated+compressed repo bytes (what the Storage Box really fills), not the modeless
// restore-size that multiplies by the retained-snapshot count. Bytes feed the soft-quota gate
// (SLICE 4); a failed stats call keeps the last-known value (stale-but-safe). RAW-DATA TRAP:
// total_file_count is 0 in this mode — read total_size only.
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--mode", "raw-data", "--json")...); serr == nil {
var st struct {
TotalSize int64 `json:"total_size"`
}
@@ -0,0 +1,542 @@
package backup
import (
"context"
"os"
pathpkg "path"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// mandAbs builds the capture-set Abs the code produces: ComputeCaptureSet uses path.Join (slash) —
// the 3-core separator rule — so on the Windows test host the mandatory path is drive + "/rel".
func mandAbs(drive, rel string) string { return pathpkg.Join(drive, rel) }
// offbox3aProvider is a configurable StackDataProvider for the 3a capture-set tests: per-stack HDD
// path + classified binds.
type offbox3aProvider struct {
hdd map[string]string
binds map[string][]ClassifiedBind
has map[string]bool
}
func (p *offbox3aProvider) GetStackComposePath(string) (string, bool) { return "", false }
func (p *offbox3aProvider) ListDeployedStacks() []StackSummary { return nil }
func (p *offbox3aProvider) GetStackHDDMounts(string) []string { return nil }
func (p *offbox3aProvider) GetStackHDDPath(n string) string { return p.hdd[n] }
func (p *offbox3aProvider) GetDockerVolumes(string) []string { return nil }
func (p *offbox3aProvider) StopStack(string) error { return nil }
func (p *offbox3aProvider) StartStack(string) error { return nil }
func (p *offbox3aProvider) RefreshAndIsRunning(string) bool { return false }
func (p *offbox3aProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) { return RecoveryInfo{}, false }
func (p *offbox3aProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
func (p *offbox3aProvider) RecreateStackFromUnit(_, _ string, _ map[string]string) error { return nil }
func (p *offbox3aProvider) GetStackClassifiedBinds(n string) ([]ClassifiedBind, bool) {
return p.binds[n], p.has[n]
}
func mandatoryHDD(rel string) ClassifiedBind {
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootHDD, RelPath: rel}, Class: appbackup.ClassMandatory}
}
func optionalUserdata(rel string) ClassifiedBind {
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootUserdata, RelPath: rel, ReadOnly: true}, Class: appbackup.ClassOptional}
}
func excludedHDD(rel string) ClassifiedBind {
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootHDD, RelPath: rel}, Class: appbackup.ClassExcluded}
}
// classifiedOffboxManager: a configured offbox manager + a classified provider + the drive registered
// as a schedulable storage path (so discoverOffboxUnit finds units on it).
func classifiedOffboxManager(t *testing.T, drive string) (*Manager, *settings.Settings, *offbox3aProvider) {
t.Helper()
m, sett := newOffboxManager(t)
prov := &offbox3aProvider{hdd: map[string]string{}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{}}
m.SetStackProvider(prov)
if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "USB", Schedulable: true}); err != nil {
t.Fatal(err)
}
return m, sett, prov
}
// mkUnit lays down a discoverable recovery unit for stack on drive.
func mkUnit(t *testing.T, drive, stack string) string {
t.Helper()
u := RecoveryUnitPath(drive, stack)
if err := os.MkdirAll(u, 0o755); err != nil {
t.Fatal(err)
}
return u
}
// captureBackupRunner records the FULL argv of each backup call (keyed by stack tag) + forget argv, and
// answers the probes so RunOffboxBackup completes.
type backupCapture struct {
mu sync.Mutex
byStack map[string][]string
forgets [][]string
backups int
}
func (c *backupCapture) runner() offboxRunner {
c.byStack = map[string][]string{}
return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
c.mu.Lock()
defer c.mu.Unlock()
switch {
case contains(args, "cat") && contains(args, "config"):
return []byte(`{"version":2}`), nil
case contains(args, "backup"):
c.backups++
c.byStack[tagOf(args)] = append([]string{}, args...)
return nil, nil
case contains(args, "forget"):
c.forgets = append(c.forgets, append([]string{}, args...))
return nil, nil
case contains(args, "snapshots"):
return []byte(`[]`), nil
case contains(args, "stats"):
return []byte(`{"total_size":123}`), nil
}
return nil, nil
}
}
// --- Scenario A: classified enlarged push (immich shape) ---
func TestOffbox3a_EnlargedPush_MandatoryOnly(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
unit := mkUnit(t, drive, "immich")
if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
t.Fatal(err)
}
// The optional :ro library exists on disk — so if the tier filter ever leaked it, the stat-filter
// would NOT hide it (this makes the RP-A tier-filter red-proof observable).
if err := os.MkdirAll(filepath.Join(drive, "userdata", "media", "photos"), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd["immich"] = drive
prov.has["immich"] = true
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich"), optionalUserdata("media/photos")}
_ = sett.SetAppOffbox("immich", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
if cap.backups != 1 {
t.Fatalf("exactly ONE snapshot per app, got %d backup calls", cap.backups)
}
args := cap.byStack["immich"]
wantMandatory := mandAbs(drive, "appdata/immich")
if !contains(args, unit) {
t.Errorf("backup argv missing the unit path %q: %v", unit, args)
}
if !contains(args, wantMandatory) {
t.Errorf("backup argv missing the mandatory userdata path %q: %v", wantMandatory, args)
}
if contains(args, mandAbs(drive, "userdata/media/photos")) {
t.Errorf("OPTIONAL :ro path must NOT ship offsite: %v", args)
}
}
// --- Scenario B: legacy / undeployed stay unit-only ---
func TestOffbox3a_LegacyUnitOnly(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
unit := mkUnit(t, drive, "sonarr")
if err := os.MkdirAll(filepath.Join(drive, "appdata", "sonarr"), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd["sonarr"] = drive
prov.has["sonarr"] = false // block REJECTED / absent → legacy (binds present but no class semantics)
prov.binds["sonarr"] = []ClassifiedBind{mandatoryHDD("appdata/sonarr")}
_ = sett.SetAppOffbox("sonarr", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
args := cap.byStack["sonarr"]
// unit-only: exactly the base shape, last arg is the unit, no extra resolved paths.
if args[len(args)-1] != unit {
t.Errorf("legacy app argv must END at the unit (no resolved paths), got %v", args)
}
for _, a := range args {
if strings.Contains(a, "appdata") || strings.Contains(a, "userdata") {
t.Errorf("legacy app resolved a bind into offsite argv (SQ5 regression): %v", args)
}
}
}
func TestOffbox3a_UndeployedUnitOnlyWithWarning(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
_ = mkUnit(t, drive, "immich")
prov.hdd["immich"] = "" // undeployed → no live HDD_PATH
prov.has["immich"] = true
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
_ = sett.SetAppOffbox("immich", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
args := cap.byStack["immich"]
if strings.Contains(strings.Join(args, " "), "appdata") {
t.Errorf("undeployed app must push unit-only: %v", args)
}
if w := sett.GetOffboxTarget().LastWarning; !strings.Contains(w, "nincs telepítve") {
t.Errorf("undeployed warning missing from LastWarning: %q", w)
}
}
// --- Scenario C: pre-push enlargement gate ---
func TestOffbox3a_EnlargementGateBlocks(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
for _, app := range []string{"immich", "small"} {
_ = mkUnit(t, drive, app)
if err := os.MkdirAll(filepath.Join(drive, "appdata", app), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd[app] = drive
prov.has[app] = true
prov.binds[app] = []ClassifiedBind{mandatoryHDD("appdata/" + app)}
_ = sett.SetAppOffbox(app, true)
}
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.QuotaGB = 50; o.RepoSizeBytes = 20 << 30 })
// immich's mandatory set is 40 GiB (20+40 ≥ 50 → blocked); small's is 1 GiB (20+1 < 50 → fits).
m.SetOffboxSizer(func(p string) int64 {
if strings.Contains(p, "immich") {
return 40 << 30
}
return 1 << 30
})
var noteMu sync.Mutex
var notes []string
m.SetOffboxEnlargeBlockedNotifier(func(stack string, _ int64, usedGB, quotaGB int) {
noteMu.Lock()
defer noteMu.Unlock()
notes = append(notes, stack)
if usedGB != 20 || quotaGB != 50 {
t.Errorf("notifier numbers wrong: used=%d quota=%d", usedGB, quotaGB)
}
})
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run must be OK (a blocked enlargement is not a run failure): %v", err)
}
// immich → unit-only; small → enlarged.
if strings.Contains(strings.Join(cap.byStack["immich"], " "), "appdata") {
t.Errorf("blocked immich must be unit-only: %v", cap.byStack["immich"])
}
if !contains(cap.byStack["small"], mandAbs(drive, "appdata/small")) {
t.Errorf("fitting 'small' must still push enlarged: %v", cap.byStack["small"])
}
tgt := sett.GetOffboxTarget()
if len(tgt.EnlargedBlocked) != 1 || tgt.EnlargedBlocked[0] != "immich" {
t.Errorf("EnlargedBlocked = %v, want [immich]", tgt.EnlargedBlocked)
}
if tgt.LastStatus != "ok" {
t.Errorf("run status = %q, want ok", tgt.LastStatus)
}
if !strings.Contains(tgt.LastWarning, "tárhelykeret miatt") || !strings.Contains(tgt.LastWarning, "immich") {
t.Errorf("blocked LastWarning missing: %q", tgt.LastWarning)
}
if len(notes) != 1 || notes[0] != "immich" {
t.Errorf("notifier must fire ONCE for immich, got %v", notes)
}
// EnlargedBlocked clears on a subsequent run where nothing is blocked.
m.SetOffboxSizer(func(string) int64 { return 1 << 30 }) // now immich fits too
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatal(err)
}
if b := sett.GetOffboxTarget().EnlargedBlocked; len(b) != 0 {
t.Errorf("EnlargedBlocked must clear when nothing is blocked, got %v", b)
}
}
// --- Scenario D: capture gaps are loud (SP-3.4) ---
func TestOffbox3a_CaptureGapsAreLoud(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
_ = mkUnit(t, drive, "app")
if err := os.MkdirAll(filepath.Join(drive, "appdata", "good"), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd["app"] = drive
prov.has["app"] = true
prov.binds["app"] = []ClassifiedBind{
mandatoryHDD("appdata/good"), // exists → captured
mandatoryHDD("../evil"), // D1: traversal → Skipped
mandatoryHDD("appdata/ghost"), // D2: passes guards but absent on disk → stat-filtered
}
_ = sett.SetAppOffbox("app", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
args := cap.byStack["app"]
joined := strings.Join(args, " ")
if !contains(args, mandAbs(drive, "appdata/good")) {
t.Errorf("the valid mandatory path must still push: %v", args)
}
if strings.Contains(joined, "evil") {
t.Errorf("traversal path escaped into argv: %v", args)
}
if strings.Contains(joined, "ghost") {
t.Errorf("stat-missing mandatory path must NOT be in argv (SP-3.4 silent-skip): %v", args)
}
if w := sett.GetOffboxTarget().LastWarning; !strings.Contains(w, "nem kerültek a távoli mentésbe") {
t.Errorf("capture-gap warning missing from LastWarning: %q", w)
}
}
// --- §8 all-excluded row (radarr shape): unit-only, NO warning ---
func TestOffbox3a_AllExcludedUnitOnlyNoWarning(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
unit := mkUnit(t, drive, "radarr")
prov.hdd["radarr"] = drive
prov.has["radarr"] = true
prov.binds["radarr"] = []ClassifiedBind{excludedHDD("appdata/radarr"), excludedHDD("downloads")}
_ = sett.SetAppOffbox("radarr", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
if args := cap.byStack["radarr"]; args[len(args)-1] != unit {
t.Errorf("all-excluded app must be unit-only: %v", args)
}
if w := sett.GetOffboxTarget().LastWarning; strings.Contains(w, "nem kerültek") {
t.Errorf("all-excluded is correct, NOT a gap — no warning expected, got %q", w)
}
}
// --- Scenario E: raw-data stats mode ---
func TestOffbox3a_StatsRawDataMode(t *testing.T) {
m, sett := newOffboxManager(t)
var statsArgs []string
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
switch {
case contains(args, "snapshots"):
return []byte(`[{"id":"a"}]`), nil
case contains(args, "stats"):
statsArgs = append([]string{}, args...)
return []byte(`{"total_size":987654321}`), nil
}
return nil, nil
})
base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
m.offboxRecordStats(context.Background(), base, env)
if !contains(statsArgs, "--mode") || valAfter(statsArgs, "--mode") != "raw-data" {
t.Fatalf("stats must run in raw-data mode, got %v", statsArgs)
}
if got := sett.GetOffboxTarget().RepoSizeBytes; got != 987654321 {
t.Errorf("RepoSizeBytes = %d, want 987654321 (parsed from raw-data total_size)", got)
}
}
// --- Scenario F: both forget call sites carry --group-by host,tags ---
func TestOffbox3a_ForgetGrouping_MainRun(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
_ = mkUnit(t, drive, "app")
prov.has["app"] = false
_ = sett.SetAppOffbox("app", true)
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatal(err)
}
if len(cap.forgets) != 1 {
t.Fatalf("expected one forget call, got %d", len(cap.forgets))
}
if valAfter(cap.forgets[0], "--group-by") != "host,tags" {
t.Errorf("main-run forget missing --group-by host,tags: %v", cap.forgets[0])
}
}
func TestOffbox3a_ForgetGrouping_OverQuotaPrune(t *testing.T) {
m, sett := newOffboxManager(t)
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.QuotaGB = 50; o.RepoSizeBytes = 51 << 30 })
var forgetArgs []string
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
switch {
case contains(args, "cat") && contains(args, "config"):
return []byte(`{}`), nil
case contains(args, "forget"):
forgetArgs = append([]string{}, args...)
case contains(args, "snapshots"):
return []byte(`[]`), nil
case contains(args, "stats"):
return []byte(`{"total_size":1}`), nil
}
return nil, nil
})
_ = m.RunOffboxBackup(context.Background()) // over-quota → prune-only path
if valAfter(forgetArgs, "--group-by") != "host,tags" {
t.Errorf("over-quota prune forget missing --group-by host,tags: %v", forgetArgs)
}
}
// --- Scenario E-restore: unit-only restore argv (ID-first + --include) + scratch OFF the rootfs ---
func TestOffbox3a_UnitOnlyRestoreArgv(t *testing.T) {
drive := t.TempDir()
m, _, prov := classifiedOffboxManager(t, drive)
prov.hdd["immich"] = drive
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 }) // plenty
unitPath := filepath.ToSlash(filepath.Join(drive, "backups", "primary", "immich"))
var restoreArgs []string
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
switch {
case contains(args, "snapshots"):
return []byte(`[{"short_id":"deadbeef","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(filepath.Join(drive, "appdata", "immich")) + `","` + unitPath + `"]}]`), nil
case contains(args, "restore"):
restoreArgs = append([]string{}, args...)
}
return nil, nil
})
if err := m.RestoreOffboxScratch(context.Background(), "immich", false); err != nil {
t.Fatalf("unit-only restore: %v", err)
}
if valAfter(restoreArgs, "restore") != "deadbeef" {
t.Errorf("restore must be ID-first (deadbeef): %v", restoreArgs)
}
if valAfter(restoreArgs, "--include") != unitPath {
t.Errorf("unit-only restore must --include the absolute unit path %q: %v", unitPath, restoreArgs)
}
target := valAfter(restoreArgs, "--target")
if !strings.HasPrefix(target, drive) || strings.Contains(target, m.cfg.Paths.DataDir) {
t.Errorf("scratch target must be on the data drive, never DataDir: %q", target)
}
}
// full restore refuses fail-closed when the snapshot size is unknown (no restore call made).
func TestOffbox3a_FullRestoreRefusesOnSizeUnknown(t *testing.T) {
drive := t.TempDir()
m, _, prov := classifiedOffboxManager(t, drive)
prov.hdd["immich"] = drive
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
unitPath := filepath.Join(drive, "backups", "primary", "immich")
restoreCalled := false
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
switch {
case contains(args, "snapshots"):
return []byte(`[{"short_id":"a","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(unitPath) + `"]}]`), nil
case contains(args, "stats"):
return nil, context.DeadlineExceeded // size lookup fails → unknown
case contains(args, "restore"):
restoreCalled = true
}
return nil, nil
})
err := m.RestoreOffboxScratch(context.Background(), "immich", true)
if err == nil || !strings.Contains(err.Error(), "nem állapítható meg") {
t.Fatalf("full restore must refuse fail-closed on unknown size, got err=%v", err)
}
if restoreCalled {
t.Error("no restic restore call may run when the size is unknown")
}
}
// old rootfs scratch is cleaned up on a new restore.
func TestOffbox3a_LegacyRootfsScratchCleanup(t *testing.T) {
drive := t.TempDir()
m, _, prov := classifiedOffboxManager(t, drive)
prov.hdd["immich"] = drive
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", "immich")
if err := os.MkdirAll(legacy, 0o755); err != nil {
t.Fatal(err)
}
unitPath := filepath.Join(drive, "backups", "primary", "immich")
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
if contains(args, "snapshots") {
return []byte(`[{"short_id":"a","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(unitPath) + `"]}]`), nil
}
return nil, nil
})
if err := m.RestoreOffboxScratch(context.Background(), "immich", false); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(legacy); !os.IsNotExist(err) {
t.Errorf("legacy rootfs scratch %s must be removed, stat err=%v", legacy, err)
}
}
// --- Scenario G: place-to-live mapping (pure) + the wrong cases ---
func TestMapOffsiteRestorePaths(t *testing.T) {
old := "/old/ns"
newNs := "/new/ns"
scratch := "/scratch"
snap := []string{
old + "/backups/primary/app",
old + "/appdata/app",
old + "/userdata/media/x",
}
got, err := mapOffsiteRestorePaths(snap, "app", scratch, newNs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if len(got) != 3 {
t.Fatalf("got %d placements, want 3: %+v", len(got), got)
}
byDst := map[string]placement{}
for _, pl := range got {
byDst[pl.dst] = pl
}
// anchor derived by trimming backups/primary/app off the unit path → oldNs; dst = newNs/<rel>,
// src = scratch/<abs-source> (SP-3.1). Built with filepath.Join to match the code (OS separators).
check := func(snapPath, rel string, isUnit bool) {
dst := filepath.Join(newNs, rel)
pl, ok := byDst[dst]
if !ok {
t.Errorf("missing placement for dst %q", dst)
return
}
if pl.src != filepath.Join(scratch, snapPath) {
t.Errorf("src for %q = %q, want %q", snapPath, pl.src, filepath.Join(scratch, snapPath))
}
if pl.isUnit != isUnit {
t.Errorf("isUnit for %q = %v, want %v", snapPath, pl.isUnit, isUnit)
}
}
check(old+"/backups/primary/app", "backups/primary/app", true)
check(old+"/appdata/app", "appdata/app", false)
check(old+"/userdata/media/x", "userdata/media/x", false)
// Wrong cases — each REFUSES the whole placement.
if _, err := mapOffsiteRestorePaths([]string{old + "/appdata/app"}, "app", scratch, newNs); err == nil {
t.Error("no unit path → must refuse")
}
if _, err := mapOffsiteRestorePaths([]string{old + "/backups/primary/app", "/elsewhere/x"}, "app", scratch, newNs); err == nil {
t.Error("a path outside the namespace → must refuse")
}
if _, err := mapOffsiteRestorePaths([]string{old + "/backups/primary/app", old + "/backups/secondary/y"}, "app", scratch, newNs); err == nil {
t.Error("a non-unit path in the reserved backups/ zone → must refuse")
}
}
@@ -0,0 +1,76 @@
package backup
import (
"fmt"
"os"
"strings"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
)
// Offsite capture-set resolution (Task 3a, architecture doc §2/§6). Turns an app's Task-3-core
// TierOffsite capture set (recovery unit + MANDATORY userdata only) into the extra absolute paths
// appended to the app's restic snapshot, plus the Hungarian customer warnings for LOUD capture gaps.
//
// SP-3.4 is law here: restic 0.14.0 does NOT error on a missing source path — it skips with a warning,
// exits 0, and silently writes a partial snapshot. So a skipped/missing MANDATORY path is detected in
// THIS function (the structural-guard Skipped list + an os.Stat filter) and surfaced in BOTH the
// English log and the Hungarian LastWarning. A restic exit code proves nothing about a missing path.
// offboxBlocked records an app whose enlarged (userdata-carrying) push was refused by the pre-push
// quota gate. The unit-only push still proceeds (never a protection regression). estBytes is the
// mandatory-set size estimate that would have been added.
type offboxBlocked struct {
stack string
estBytes int64
}
// offboxCaptureSet computes an app's OFFSITE mandatory capture paths to add to its recovery-unit
// snapshot, plus any Hungarian warnings for capture gaps. It never returns optional/excluded paths
// (the TierOffsite filter drops them — §2). Returns (nil, nil) for the legacy / no-provider / no-block
// world: offsite stays UNIT-ONLY, byte-identical to pre-v0.134.0 (the SQ5 cost-regression guard).
func (m *Manager) offboxCaptureSet(stack string) (extra []string, warns []string) {
if m.stackProvider == nil {
return nil, nil // no provider wired → legacy world → unit only
}
binds, has := m.stackProvider.GetStackClassifiedBinds(stack)
if !has {
return nil, nil // no backup block → legacy → unit only
}
// Resolve against the app's LIVE HDD_PATH (raw — NOT GetAppDrivePath, whose systemDataPath fallback
// would resolve userdata onto the wrong drive). Empty ⇒ undeployed / no HDD (decision §2.4):
// mandatory-path resolution needs the live HDD_PATH, so push unit-only + a loud WARN.
hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack))
if hdd == "" {
m.logger.Printf("[WARN] [offbox] %s: not deployed — offsite push is unit-only (mandatory userdata not resolvable)", stack)
return nil, []string{fmt.Sprintf("Figyelmeztetés: a(z) %s nincs telepítve — csak a mentési egység került a távoli mentésbe.", stack)}
}
nsRoot := m.namespaceRoot(hdd)
cs := appbackup.ComputeCaptureSet(binds, has, appbackup.TierOffsite, nsRoot)
var gaps []string
// Structurally-refused MANDATORY paths (traversal / bare drive-root / reserved backups/ zone) are
// loud ERROR gaps — the path the customer thinks is protected is not in the snapshot.
for _, sk := range cs.Skipped {
if sk.Class == appbackup.ClassMandatory {
m.logger.Printf("[ERROR] [offbox] %s: mandatory path refused by a structural guard (%s): %s/%s — NOT in the offsite snapshot",
stack, sk.Reason, sk.Root, sk.RelPath)
gaps = append(gaps, sk.RelPath)
}
}
// Stat-filter (§2.5): a declared mandatory path absent on disk. restic would skip it SILENTLY
// (SP-3.4), so drop it from argv AND warn — never a silent "looks backed up but isn't".
for _, p := range cs.Paths {
if _, err := os.Stat(p.Abs); err != nil {
m.logger.Printf("[WARN] [offbox] %s: mandatory data path missing on disk, skipped from offsite: %s", stack, p.Abs)
gaps = append(gaps, p.RelPath)
continue
}
extra = append(extra, p.Abs)
}
if len(gaps) > 0 {
warns = append(warns, fmt.Sprintf("Figyelmeztetés: a(z) %s alkalmazás egyes adatmappái nem kerültek a távoli mentésbe: %s.",
stack, strings.Join(gaps, ", ")))
}
return extra, warns
}
@@ -0,0 +1,377 @@
package backup
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// Offsite restore rework (Task 3a §7). With mandatory userdata now in snapshots, restore needs three
// changes over the old dump-to-rootfs-scratch:
// 1. scratch relocated off the ~8 GB guest rootfs onto a data drive, behind a headroom gate (F-A1);
// 2. a unit-only DEFAULT restore (`--include <absolute-unit-path>`, SP-3.2) — full is a deliberate,
// size-gated second action;
// 3. place-to-live = a missing-only merge (never --delete) so the SQ3 immich case is restorable
// from offsite alone.
// ID-first everywhere (§3): `restic stats --tag` is UNPROVEN on 0.14.0, so the size lookup resolves the
// snapshot ID via `snapshots latest --tag` and calls `stats <ID>`.
const (
// offboxUnitOnlyFreeFloor — a unit-only restore needs at least this much free on the scratch drive.
// Catalog recovery units are MB1 GB (SQ4); 2 GiB is a safe floor without a per-snapshot size probe.
offboxUnitOnlyFreeFloor = int64(2) << 30
)
// SetOffboxFreeFn overrides the restore free-space probe (tests; the Windows go-test host has no df).
func (m *Manager) SetOffboxFreeFn(fn func(path string) int64) { m.offboxFreeFn = fn }
// offboxFree returns the free-space probe (nil seam → the real diskFreeBytes).
func (m *Manager) offboxFree() func(string) int64 {
if m.offboxFreeFn != nil {
return m.offboxFreeFn
}
return diskFreeBytes
}
// diskFreeBytes returns available bytes on the filesystem holding path (0 on any error). Mirrors
// appexport.DiskFree; kept local so the backup package needs no cross-package dependency.
func diskFreeBytes(path string) int64 {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, "df", "--output=avail", "-B1", path).Output()
if err != nil {
return 0
}
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
if len(lines) < 2 {
return 0
}
var size int64
fmt.Sscanf(strings.TrimSpace(lines[1]), "%d", &size)
return size
}
// offboxUnitPathOf returns the snapshot path that is the recovery unit for stack (suffix
// backups/primary/<stack>), or "" if none is present.
func offboxUnitPathOf(paths []string, stack string) string {
suffix := "/backups/primary/" + stack
for _, p := range paths {
if strings.HasSuffix(p, suffix) {
return p
}
}
return ""
}
// offboxLatestSnapshot resolves the newest snapshot for stack: its short ID + captured paths, via
// `snapshots latest --tag <stack> --json`. When the tag spans more than one group (old unit-only shape
// + new enlarged shape), it returns the newest by time.
func (m *Manager) offboxLatestSnapshot(ctx context.Context, stack string) (id string, paths []string, err error) {
t := m.settings.GetOffboxTarget()
base, env := m.offboxBaseArgs(t)
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer cancel()
out, serr := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "latest", "--tag", stack, "--json")...)
if serr != nil {
return "", nil, fmt.Errorf("offbox snapshots %s: %w: %s", stack, serr, truncate(out))
}
var snaps []struct {
ShortID string `json:"short_id"`
ID string `json:"id"`
Time time.Time `json:"time"`
Paths []string `json:"paths"`
}
if json.Unmarshal(out, &snaps) != nil || len(snaps) == 0 {
return "", nil, fmt.Errorf("offbox: nincs pillanatkép a(z) %s alkalmazáshoz", stack)
}
best := 0
for i := 1; i < len(snaps); i++ {
if snaps[i].Time.After(snaps[best].Time) {
best = i
}
}
id = snaps[best].ShortID
if id == "" {
id = snaps[best].ID
}
return id, snaps[best].Paths, nil
}
// offboxSnapshotSize returns the restore-size (logical bytes) of ONE snapshot via `stats <ID> --json`
// (default mode — for a single snapshot ID this is exactly that snapshot's on-disk-when-restored size,
// the correct headroom meaning; SP-1). ID-first: never `stats --tag` (unproven on 0.14.0).
func (m *Manager) offboxSnapshotSize(ctx context.Context, id string) (int64, error) {
t := m.settings.GetOffboxTarget()
base, env := m.offboxBaseArgs(t)
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer cancel()
out, err := m.runner()(sctx, env, append(append([]string{}, base...), "stats", id, "--json")...)
if err != nil {
return 0, fmt.Errorf("offbox stats %s: %w: %s", id, err, truncate(out))
}
var st struct {
TotalSize int64 `json:"total_size"`
}
if json.Unmarshal(out, &st) != nil || st.TotalSize <= 0 {
return 0, fmt.Errorf("offbox: a(z) %s pillanatkép mérete ismeretlen", id)
}
return st.TotalSize, nil
}
// offboxRestoreScratchDir returns the on-DATA-DRIVE scratch dir for an app's offsite restore
// (<nsRoot>/backups/offsite-restore/<app>) plus the namespace root (an existing dir, for the free-space
// probe). NEVER cfg.Paths.DataDir (the rootfs — the F-A1 filler). App's HDD drive first; else the first
// schedulable storage path; else a Hungarian refusal.
func (m *Manager) offboxRestoreScratchDir(stack string) (scratch, nsRoot string, err error) {
if m.stackProvider != nil {
if hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)); hdd != "" {
nr := m.namespaceRoot(hdd)
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
}
}
for _, sp := range m.settings.GetSchedulableStoragePaths() {
if strings.TrimSpace(sp.Path) != "" {
nr := m.namespaceRoot(sp.Path)
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
}
}
return "", "", fmt.Errorf("nincs elérhető adatmeghajtó a visszaállításhoz")
}
// RestoreOffboxScratch restores an app's latest offsite snapshot to an on-data-drive scratch dir
// (non-destructive — never overwrites live data). full=false (the default) restores the recovery UNIT
// only (`--include <absolute-unit-path>`, SP-3.2); full=true restores the whole snapshot (unit +
// mandatory userdata) behind a size×1.1 headroom gate. Fail-closed: an unknown snapshot size refuses a
// full restore.
func (m *Manager) RestoreOffboxScratch(ctx context.Context, stack string, full bool) error {
if !m.OffboxConfigured() {
return fmt.Errorf("off-box backup not configured")
}
if !isSafeStackName(stack) {
return fmt.Errorf("invalid stack name")
}
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
if err != nil {
return err
}
unitPath := offboxUnitPathOf(paths, stack)
if unitPath == "" {
return fmt.Errorf("a(z) %s pillanatképében nincs mentési egység — a visszaállítás nem indítható", stack)
}
scratch, nsRoot, err := m.offboxRestoreScratchDir(stack)
if err != nil {
return err
}
// Headroom gate (F-A1) — probed on the namespace root (an existing dir).
free := m.offboxFree()(nsRoot)
if full {
size, serr := m.offboxSnapshotSize(ctx, id)
if serr != nil {
// SizeUnknown never renders as fits — fail closed.
return fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
}
need := size + size/10 // ×1.1
if free < need {
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
}
} else if free < offboxUnitOnlyFreeFloor {
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(offboxUnitOnlyFreeFloor), humanizeBytes(free))
}
// F-A1 hygiene: drop the legacy rootfs scratch (DataDir/offbox-restore/<app>) best-effort.
legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", stack)
if _, sErr := os.Stat(legacy); sErr == nil {
if rmErr := os.RemoveAll(legacy); rmErr != nil {
m.logger.Printf("[WARN] [offbox] could not remove legacy rootfs restore scratch %s: %v", legacy, rmErr)
} else {
m.logger.Printf("[INFO] [offbox] removed legacy rootfs restore scratch %s", legacy)
}
}
if err := os.MkdirAll(scratch, 0o755); err != nil {
return fmt.Errorf("restore dir: %w", err)
}
t := m.settings.GetOffboxTarget()
base, env := m.offboxBaseArgs(t)
rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
defer cancel()
m.unlockStale(rctx, base, env) // pre-restore hygiene
args := []string{"restore", id, "--target", scratch}
if !full {
args = append(args, "--include", unitPath) // SP-3.2: absolute snapshot unit path = unit-only
}
out, rerr := m.resticStep(rctx, env, base, "restore:"+stack, args...)
if rerr != nil {
return fmt.Errorf("offbox restore %s: %w: %s", stack, rerr, truncate(out))
}
m.logger.Printf("[INFO] [offbox] restored %s (%s, full=%v) → %s", stack, id, full, scratch)
return nil
}
// OffboxRestorePrepareFull resolves the latest snapshot's restore-size and verifies scratch headroom
// for a FULL restore WITHOUT starting it (the two-step size-first gate). Returns the human size on
// success, or a Hungarian error to flash on refusal (size unknown / no headroom — fail-closed).
func (m *Manager) OffboxRestorePrepareFull(ctx context.Context, stack string) (string, error) {
if !m.OffboxConfigured() {
return "", fmt.Errorf("off-box backup not configured")
}
if !isSafeStackName(stack) {
return "", fmt.Errorf("invalid stack name")
}
id, _, err := m.offboxLatestSnapshot(ctx, stack)
if err != nil {
return "", err
}
size, serr := m.offboxSnapshotSize(ctx, id)
if serr != nil {
return "", fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
}
_, nsRoot, derr := m.offboxRestoreScratchDir(stack)
if derr != nil {
return "", derr
}
need := size + size/10
if free := m.offboxFree()(nsRoot); free < need {
return "", fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
}
return humanizeBytes(size), nil
}
// OffboxFullScratchReady reports whether a (non-empty) full-restore scratch exists for stack — the gate
// for showing the place-to-live action. PlaceOffsiteRestore re-validates per-path completeness.
func (m *Manager) OffboxFullScratchReady(stack string) bool {
if !isSafeStackName(stack) {
return false
}
scratch, _, err := m.offboxRestoreScratchDir(stack)
if err != nil {
return false
}
if fi, sErr := os.Stat(scratch); sErr != nil || !fi.IsDir() {
return false
}
entries, _ := os.ReadDir(scratch)
return len(entries) > 0
}
// placement is one source→dest pair for place-to-live: src is the reconstructed absolute path under the
// scratch (SP-3.1), dst is the live location under the app's current namespace root.
type placement struct {
src string
dst string
isUnit bool
}
// mapOffsiteRestorePaths maps a completed full-scratch restore to live placements (pure). The anchor
// oldNs is derived by trimming backups/primary/<stack> off the unit path (the snapshot may come from a
// DIFFERENT drive after churn — liveNsRoot is where it goes). Refuses the WHOLE placement (no partial
// writes) on: no unit path; a path outside oldNs (escape); a `..` segment; a non-unit path in the
// reserved backups/ zone.
func mapOffsiteRestorePaths(snapPaths []string, stack, scratch, liveNsRoot string) ([]placement, error) {
unitSuffix := "/backups/primary/" + stack
oldNs := ""
for _, p := range snapPaths {
if strings.HasSuffix(p, unitSuffix) {
oldNs = strings.TrimSuffix(p, unitSuffix)
break
}
}
if oldNs == "" {
return nil, fmt.Errorf("a pillanatképben nincs mentési egység (backups/primary/%s)", stack)
}
out := make([]placement, 0, len(snapPaths))
for _, p := range snapPaths {
if p != oldNs && !strings.HasPrefix(p, oldNs+"/") {
return nil, fmt.Errorf("a pillanatkép egy útvonala a névtéren kívülre mutat: %s", p)
}
rel := strings.TrimPrefix(p, oldNs+"/")
for _, seg := range strings.Split(rel, "/") {
if seg == ".." {
return nil, fmt.Errorf("a pillanatkép egy útvonala érvénytelen (..): %s", p)
}
}
isUnit := rel == "backups/primary/"+stack
if !isUnit && (rel == "backups" || strings.HasPrefix(rel, "backups/")) {
return nil, fmt.Errorf("nem-egység útvonal a fenntartott backups zónában: %s", p)
}
out = append(out, placement{
src: filepath.Join(scratch, p), // SP-3.1: abs source reconstructed under the target
dst: filepath.Join(liveNsRoot, rel),
isUnit: isUnit,
})
}
return out, nil
}
// placeCopier returns the place-to-live missing-only merge (nil seam → rsyncRestoreMissing, the
// `-a --ignore-existing` additive copy). NEVER rsyncMirror (--delete).
func (m *Manager) placeCopier() func(src, dst string) (int, error) {
if m.offboxPlaceCopier != nil {
return m.offboxPlaceCopier
}
return rsyncRestoreMissing
}
// PlaceOffsiteRestore places a COMPLETED full-scratch restore into the app's live locations via a
// missing-only merge (§7.3), so the SQ3 immich case is restorable from offsite alone. The recovery
// unit is placed ONLY if the live unit is ABSENT (never overwrites a local unit); every other path is
// merged missing-only. Does NOT deploy/start anything — RecreateStackFromUnit / the restore flow owns
// that. Single-flight. Requires a completed full scratch (deterministic path + existence check).
func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error {
if !m.OffboxConfigured() {
return fmt.Errorf("off-box backup not configured")
}
if !isSafeStackName(stack) {
return fmt.Errorf("invalid stack name")
}
if err := m.acquireRunning(); err != nil {
return fmt.Errorf("egy másik mentési/visszaállítási művelet már fut")
}
defer m.releaseRunning()
scratch, _, err := m.offboxRestoreScratchDir(stack)
if err != nil {
return err
}
if _, sErr := os.Stat(scratch); sErr != nil {
return fmt.Errorf("nincs előkészített teljes visszaállítás — futtass előbb egy teljes visszaállítást")
}
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
if err != nil {
return err
}
_ = id
liveNs := m.AppNamespaceRoot(stack)
if liveNs == "" {
return fmt.Errorf("a(z) %s élő adatmeghajtója nem határozható meg", stack)
}
placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs)
if err != nil {
return err // whole-placement refusal (no partial writes)
}
copier := m.placeCopier()
var placed int
for _, pl := range placements {
if _, sErr := os.Stat(pl.src); sErr != nil {
// The full scratch is incomplete for this path (e.g. only a unit-only restore ran) — refuse
// rather than place a partial set.
return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src))
}
if pl.isUnit {
if _, liveErr := os.Stat(pl.dst); liveErr == nil {
m.logger.Printf("[INFO] [offbox] place %s: live recovery unit present — not overwriting", stack)
continue // never overwrite a local unit
}
}
n, cErr := copier(pl.src, pl.dst)
if cErr != nil {
return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr)
}
placed += n
}
m.logger.Printf("[INFO] [offbox] placed %s from offsite scratch: %d file(s) merged (missing-only)", stack, placed)
return nil
}
+9
View File
@@ -286,6 +286,15 @@ func (n *Notifier) NotifyBackupFailed(message, errMsg string) {
n.PushEvent("backup_failed", "error", message, BackupDetails{Error: errMsg})
}
// NotifyOffboxEnlargeBlocked sends a WARNING (not a failure) when an app's enlarged offsite push was
// refused by the pre-push quota gate — its config+DB were still saved. Customer-facing (Hungarian
// body). NOTE: the event type "offbox_enlarge_blocked" must be added to the hub's allowedEventTypes +
// customerMessages for delivery (a hub-side task, flagged — until then the hub 400s/drops it and the
// in-dashboard LastWarning + /backups/remote note carry the message).
func (n *Notifier) NotifyOffboxEnlargeBlocked(message string) {
n.PushEvent("offbox_enlarge_blocked", "warning", message, nil)
}
// (NotifyBackupCompleted removed 2026-06-16 — the backup_completed event had no callers
// since slice 8C moved whole-guest backup to the agent. The hub's backup-deadline check
// now reads the agent host-report's PBS snapshots instead of this event. DB-dump events
+5
View File
@@ -147,6 +147,11 @@ type OffboxTarget struct {
// LastWarning is a customer-visible notice set on an otherwise-OK run when SOME toggled apps had
// no discoverable recovery unit (partial run). Empty on a fully-successful or failed run.
LastWarning string `json:"last_warning,omitempty"`
// EnlargedBlocked (3a) lists the apps whose ENLARGED (mandatory-userdata) offsite push was refused
// by the pre-push quota gate on the last run — their unit-only push still succeeded. Replaced each
// OK run (sorted; empty clears). Drives the per-app "config+DB only" note on /backups/remote and
// the edge-triggered enlarge-blocked notification. Not a secret (app-name list).
EnlargedBlocked []string `json:"enlarged_blocked,omitempty"`
// EscrowState (fork-4) gates offsite RUNS on the repo password being escrowed under R: ""|"pending"
// |"escrowed". Enabling offsite stages the password to the agent and sets "pending"; no offsite run
// proceeds until an operator confirms the escrow ceremony ("escrowed") — so no un-recoverable
+26
View File
@@ -661,6 +661,14 @@ func (s *Server) backupsOffboxData(data map[string]interface{}) {
}
// SLICE 4 soft-quota usage bar (rendered only when a quota is set — shared model).
data["OffboxQuotaPct"] = backup.OffboxQuotaPercent(offboxTgt)
// 3a: per-app "config+DB only" note set — apps whose enlarged push the quota gate blocked last run.
blocked := map[string]bool{}
if offboxTgt != nil {
for _, a := range offboxTgt.EnlargedBlocked {
blocked[a] = true
}
}
data["OffboxBlockedSet"] = blocked
}
// offboxStaleWarningMarker is the substring the zero-toggled offbox run writes into
@@ -757,6 +765,24 @@ func (s *Server) backupsAppsHandler(w http.ResponseWriter, r *http.Request) {
func (s *Server) backupsRestoreHandler(w http.ResponseWriter, r *http.Request) {
data := s.backupsCommonData("backups-restore", "Biztonsági mentés — Visszaállítás", r)
s.backupsOffboxData(data) // restore-to-verify lists the offbox-toggled apps
// Full-restore two-step reveal (§7.2): after the size+headroom prepare step, offboxRestoreHandler
// redirects here with the app + human size so the confirm section can show the size BEFORE starting.
if fp := strings.TrimSpace(r.URL.Query().Get("full_prep")); fp != "" {
data["FullPrepApp"] = fp
data["FullPrepSize"] = r.URL.Query().Get("full_size")
}
// Per-app place-to-live availability (a completed full scratch exists → offer the merge action).
ready := map[string]bool{}
if s.backupMgr != nil {
if apps, ok := data["OffboxApps"].([]OffboxAppRow); ok {
for _, a := range apps {
if a.Enabled && s.backupMgr.OffboxFullScratchReady(a.Name) {
ready[a.Name] = true
}
}
}
}
data["OffboxScratchReady"] = ready
s.executeTemplate(w, r, "backups_restore", data)
}
+62 -11
View File
@@ -4,7 +4,6 @@ import (
"context"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"time"
@@ -88,6 +87,8 @@ func (s *Server) offboxConfigHandler(w http.ResponseWriter, r *http.Request) {
tgt.LastDuration, tgt.RepoSizeHuman, tgt.SnapshotCount = prev.LastDuration, prev.RepoSizeHuman, prev.SnapshotCount
tgt.LastWarning = prev.LastWarning
tgt.EscrowState = prev.EscrowState
tgt.RepoSizeBytes = prev.RepoSizeBytes
tgt.EnlargedBlocked = prev.EnlargedBlocked
}
// fork-4: enabling offsite stages the repo password to the agent for the R-escrow ceremony and marks
// it PENDING — no offsite RUN proceeds until escrow is confirmed (atomicity). Re-editing an already
@@ -213,8 +214,10 @@ func (s *Server) offboxRunHandler(w http.ResponseWriter, r *http.Request) {
offboxRedirect(w, r, "A távoli mentés elindult (a futás után az állapot frissül).", false)
}
// offboxRestoreHandler restores an app's off-box data to a scratch dir (non-destructive — does NOT
// overwrite live data; the operator inspects the restored files).
// offboxRestoreHandler restores an app's off-box data to an on-data-drive scratch dir (§7, F-A1;
// non-destructive — does NOT overwrite live data). mode=unit (default) restores the recovery unit
// only; mode=full is size-gated and two-step (first POST computes the size + headroom and redirects
// with a reveal cue; the revealed confirm POSTs mode=full&confirm=1, re-checked at execution).
func (s *Server) offboxRestoreHandler(w http.ResponseWriter, r *http.Request) {
if s.backupMgr == nil || !s.backupMgr.OffboxConfigured() {
offboxRedirectTo(w, r, "/backups/restore", "A távoli mentési cél nincs beállítva.", true)
@@ -226,25 +229,73 @@ func (s *Server) offboxRestoreHandler(w http.ResponseWriter, r *http.Request) {
offboxRedirectTo(w, r, "/backups/restore", "Hiányzó alkalmazás.", true)
return
}
// Part B: fast-path refuse a concurrent op, then run async on a BACKGROUND context. The old code
// bounded on r.Context()+30m — a proxy read-timeout then CANCELED the SFTP restore mid-flight
// (worse than F4: not just an error page, an aborted restore). Background ctx fixes that.
mode := strings.TrimSpace(r.FormValue("mode"))
if mode == "" {
mode = "unit"
}
// Step 1 of the full two-step: compute size + headroom BEFORE any restic restore; on a refusal
// flash the Hungarian reason, else redirect with the reveal params (size shown before starting).
if mode == "full" && r.FormValue("confirm") != "1" {
pctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
sizeHuman, err := s.backupMgr.OffboxRestorePrepareFull(pctx, app)
if err != nil {
offboxRedirectTo(w, r, "/backups/restore", err.Error(), true)
return
}
http.Redirect(w, r, "/backups/restore?full_prep="+url.QueryEscape(app)+"&full_size="+url.QueryEscape(sizeHuman), http.StatusFound)
return
}
// Fast-path refuse a concurrent op, then run async on a BACKGROUND context (a proxy read-timeout on
// r.Context() would CANCEL the SFTP restore mid-flight — the F4 lesson).
if s.backupMgr.IsRunning() {
offboxRedirectTo(w, r, "/backups/restore", "Egy mentési/visszaállítási művelet már fut.", true)
return
}
dest := filepath.Join(s.cfg.Paths.DataDir, "offbox-restore", app)
full := mode == "full"
s.backupMgr.BeginRestoreOp("offbox-restore", app)
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if err := s.backupMgr.RestoreOffbox(ctx, app, dest); err != nil {
s.logger.Printf("[ERROR] [web] off-box restore %s (async): %v", app, err)
if err := s.backupMgr.RestoreOffboxScratch(ctx, app, full); err != nil {
s.logger.Printf("[ERROR] [web] off-box restore %s (full=%v, async): %v", app, full, err)
s.backupMgr.EndRestoreOp(false, "A visszaállítás sikertelen: "+err.Error())
return
}
s.logger.Printf("[INFO] [web] off-box restore %s completed (async) → %s", app, dest)
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" visszaállítva ide (ellenőrzésre): "+dest)
s.logger.Printf("[INFO] [web] off-box restore %s completed (full=%v, async)", app, full)
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" visszaállítva ellenőrző mappába a meghajtón (a meglévő adatok változatlanok).")
}()
offboxRedirectTo(w, r, "/backups/restore", "A távoli visszaállítás elindult — az állapot itt frissül.", false)
}
// offboxPlaceHandler places a COMPLETED full-restore scratch into the app's live locations via a
// missing-only merge (§7.3). Never overwrites existing files. Async on a background context.
func (s *Server) offboxPlaceHandler(w http.ResponseWriter, r *http.Request) {
if s.backupMgr == nil || !s.backupMgr.OffboxConfigured() {
offboxRedirectTo(w, r, "/backups/restore", "A távoli mentési cél nincs beállítva.", true)
return
}
_ = r.ParseForm()
app := strings.TrimSpace(r.FormValue("app"))
if app == "" {
offboxRedirectTo(w, r, "/backups/restore", "Hiányzó alkalmazás.", true)
return
}
if s.backupMgr.IsRunning() {
offboxRedirectTo(w, r, "/backups/restore", "Egy mentési/visszaállítási művelet már fut.", true)
return
}
s.backupMgr.BeginRestoreOp("offbox-place", app)
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if err := s.backupMgr.PlaceOffsiteRestore(ctx, app); err != nil {
s.logger.Printf("[ERROR] [web] off-box place %s (async): %v", app, err)
s.backupMgr.EndRestoreOp(false, "A helyreállítás sikertelen: "+err.Error())
return
}
s.logger.Printf("[INFO] [web] off-box place %s completed (async)", app)
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" hiányzó fájljai helyreállítva az élő adatok közé.")
}()
offboxRedirectTo(w, r, "/backups/restore", "A helyreállítás elindult — az állapot itt frissül.", false)
}
+2
View File
@@ -370,6 +370,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.offboxRunHandler(w, r)
case path == "/backup/offbox/restore" && r.Method == http.MethodPost:
s.offboxRestoreHandler(w, r)
case path == "/backup/offbox/place" && r.Method == http.MethodPost:
s.offboxPlaceHandler(w, r)
// Controller-driven escrow ceremony wizard (v0.127.0): the customer-facing R flow.
case path == "/backup/escrow" && r.Method == http.MethodGet:
s.escrowWizardPageHandler(w, r)
@@ -103,6 +103,9 @@
<input type="hidden" name="enabled" value="{{if .Enabled}}false{{else}}true{{end}}">
<button type="submit" class="btn btn-xs {{if .Enabled}}btn-outline{{else}}btn-primary{{end}}">{{if .Enabled}}Távoli mentés kikapcsolása{{else}}Távoli mentés bekapcsolása{{end}}</button>
</form>
{{if $.OffboxBlockedSet}}{{if index $.OffboxBlockedSet .Name}}
<span class="form-hint" style="display:block;margin-top:.25rem">A teljes mentés túllépné a tárhelykeretet — csak a konfiguráció és az adatbázis kerül mentésre.</span>
{{end}}{{end}}
{{template "app_list_row_end"}}
{{end}}
</div>
@@ -72,8 +72,29 @@
{{template "app_list_row" dict "Slug" .Slug "Name" .DisplayName}}
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
<input type="hidden" name="app" value="{{.Name}}">
<button type="submit" class="btn btn-xs btn-outline" data-confirm="Visszaállítja a(z) {{.DisplayName}} adatait a távoli tárolóról egy ellenőrző mappába? A meglévő adatok NEM íródnak felül.">Visszaállítás (ellenőrzéshez)</button>
<input type="hidden" name="mode" value="unit">
<button type="submit" class="btn btn-xs btn-outline">Visszaállítás ellenőrzéshez (konfiguráció + adatbázis)</button>
</form>
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
<input type="hidden" name="app" value="{{.Name}}">
<input type="hidden" name="mode" value="full">
<button type="submit" class="btn btn-xs btn-outline">Teljes visszaállítás előkészítése</button>
</form>
{{if $.FullPrepApp}}{{if eq $.FullPrepApp .Name}}
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
<input type="hidden" name="app" value="{{.Name}}">
<input type="hidden" name="mode" value="full">
<input type="hidden" name="confirm" value="1">
<button type="submit" class="btn btn-xs btn-primary">Teljes visszaállítás indítása (~{{$.FullPrepSize}})</button>
</form>
{{end}}{{end}}
{{if $.OffboxScratchReady}}{{if index $.OffboxScratchReady .Name}}
<form method="POST" action="/backup/offbox/place" style="display:inline">{{$.CSRFField}}
<input type="hidden" name="app" value="{{.Name}}">
<button type="submit" class="btn btn-xs btn-outline">Helyreállítás az élő adatok közé (csak a hiányzó fájlok)</button>
</form>
<span class="form-hint" style="display:block;margin-top:.25rem">A meglévő fájlokat nem írja felül.</span>
{{end}}{{end}}
{{template "app_list_row_end"}}
{{end}}
{{end}}