Offsite tier policy engine: mandatory userdata, raw-data quota, restore rework (Task 3a, v0.134.0)
Each toggled app's offsite push = one multi-path restic snapshot (recovery unit + TierOffsite mandatory userdata via ComputeCaptureSet); legacy/undeployed stay unit-only. Loud capture gaps (SP-3.4: restic 0.14.0 silently skips missing paths). Quota = stats --mode raw-data (SP-1; displayed size drops once). Pre-push enlargement gate blocks the userdata enlargement over-quota (unit-only push continues; EnlargedBlocked; edge-triggered notify). forget --group-by host,tags on both sites (SP-2). Restore reworked: scratch off the rootfs + headroom gate (F-A1), unit-only default via --include, size-first full, place-to-live missing-only merge (never --delete). UI: unit/full-two-step/place actions + per-app blocked note; route POST /backup/offbox/place. HUB FLAG: offbox_enlarge_blocked event needs hub allowlist for push delivery. +13 tests; all 10 §10 red-proofs verified. No tier-2/.fab/hub/agent changes.
This commit is contained in:
@@ -36,6 +36,21 @@ type Manager struct {
|
||||
offboxRunner offboxRunner
|
||||
offboxNotify func(dur time.Duration, snapshots int, err error)
|
||||
|
||||
// offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable
|
||||
// in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb).
|
||||
offboxSizer func(path string) int64
|
||||
// offboxEnlargeBlockedNotify (3a), if set, is called ONCE per app that NEWLY enters the
|
||||
// quota-blocked (enlargement-refused) state — edge-triggered against the persisted EnlargedBlocked
|
||||
// set so a nightly schedule can't re-notify a persistently-blocked app (the hub owns cooldown; the
|
||||
// controller must not add a timer). Wired in cmd/controller/main.go.
|
||||
offboxEnlargeBlockedNotify func(stack string, estBytes int64, usedGB, quotaGB int)
|
||||
// offboxPlaceCopier (3a) — the place-to-live missing-only merge seam (nil → rsyncRestoreMissing,
|
||||
// the `-a --ignore-existing` additive copy). Never rsyncMirror (--delete trap).
|
||||
offboxPlaceCopier func(src, dst string) (int, error)
|
||||
// offboxFreeFn (3a) — the free-space probe for the restore headroom gate, overridable in tests (the
|
||||
// Windows `go test` host has no `df`). Nil → the real diskFreeBytes (df --output=avail).
|
||||
offboxFreeFn func(path string) int64
|
||||
|
||||
// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
|
||||
// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
|
||||
discoverDBs func(ctx context.Context) ([]DiscoveredDB, error)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -52,6 +53,23 @@ func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err
|
||||
m.offboxNotify = fn
|
||||
}
|
||||
|
||||
// SetOffboxSizer overrides the mandatory-set byte estimator (tests). SetOffboxEnlargeBlockedNotifier
|
||||
// wires the edge-triggered enlargement-blocked notification (main.go). SetOffboxPlaceCopier overrides
|
||||
// the place-to-live missing-only merge (tests).
|
||||
func (m *Manager) SetOffboxSizer(fn func(path string) int64) { m.offboxSizer = fn }
|
||||
func (m *Manager) SetOffboxEnlargeBlockedNotifier(fn func(stack string, estBytes int64, usedGB, quotaGB int)) {
|
||||
m.offboxEnlargeBlockedNotify = fn
|
||||
}
|
||||
func (m *Manager) SetOffboxPlaceCopier(fn func(src, dst string) (int, error)) { m.offboxPlaceCopier = fn }
|
||||
|
||||
// offboxSize returns the mandatory-set byte estimator (nil seam → the real du -sb dirSizeBytes).
|
||||
func (m *Manager) offboxSize() func(string) int64 {
|
||||
if m.offboxSizer != nil {
|
||||
return m.offboxSizer
|
||||
}
|
||||
return dirSizeBytes
|
||||
}
|
||||
|
||||
func (m *Manager) runner() offboxRunner {
|
||||
if m.offboxRunner != nil {
|
||||
return m.offboxRunner
|
||||
@@ -395,6 +413,14 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
apps := m.settings.GetOffboxApps()
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
// Edge-trigger for the enlarge-blocked notification: capture the PRIOR blocked set so we notify only
|
||||
// apps that NEWLY cross into the blocked state (a persistently-blocked app doesn't re-notify nightly).
|
||||
priorBlocked := map[string]bool{}
|
||||
if t != nil {
|
||||
for _, s := range t.EnlargedBlocked {
|
||||
priorBlocked[s] = true
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
m.logger.Printf("[INFO] [offbox] backup run started (%d app(s) toggled)", len(apps))
|
||||
if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" }); err != nil {
|
||||
@@ -403,6 +429,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
|
||||
var backedUp int
|
||||
var missing []string
|
||||
var runResult offboxRunResult
|
||||
var runErr error
|
||||
if usedGB, quota, over := offboxQuotaState(t); over {
|
||||
// SLICE 4 soft-quota gate (pre-run): NEW backups are refused at ≥100% of the shared-model quota —
|
||||
@@ -414,8 +441,16 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
m.offboxRecordStats(ctx, base, env) // the prune may have brought the size back down — refresh
|
||||
runErr = fmt.Errorf("A távoli mentés túllépte a tárhelykeretet (%d/%d GB) — törölj régi mentéseket vagy kérj nagyobb keretet.", usedGB, quota)
|
||||
} else {
|
||||
backedUp, missing, runErr = m.runOffboxInternal(ctx, apps, base, env)
|
||||
runResult, runErr = m.runOffboxInternal(ctx, apps, base, env, t)
|
||||
backedUp = runResult.backedUp
|
||||
missing = runResult.missing
|
||||
}
|
||||
// Sorted names of apps whose enlargement was blocked this run (replaces the persisted set; empty clears).
|
||||
var blockedNames []string
|
||||
for _, b := range runResult.blocked {
|
||||
blockedNames = append(blockedNames, b.stack)
|
||||
}
|
||||
sort.Strings(blockedNames)
|
||||
|
||||
// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
|
||||
// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
|
||||
@@ -440,6 +475,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
o.LastStatus = "ok"
|
||||
o.LastError = ""
|
||||
o.SnapshotCount = snapshots
|
||||
o.EnlargedBlocked = blockedNames // replace each run (sorted); empty slice clears it
|
||||
var warns []string
|
||||
// Zero-toggle honesty (take-two obs.): a configured target with NOTHING selected reports
|
||||
// its emptiness instead of a bare success — the customer thinks offsite runs, but nothing
|
||||
@@ -451,6 +487,13 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
|
||||
len(missing), strings.Join(missing, ", ")))
|
||||
}
|
||||
// 3a: capture-gap warnings (structurally-refused / on-disk-missing mandatory paths, undeployed).
|
||||
warns = append(warns, runResult.warns...)
|
||||
// 3a: the pre-push enlargement gate blocked some apps' userdata — config+DB still saved.
|
||||
if len(blockedNames) > 0 {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: a tárhelykeret miatt %d alkalmazásnál csak konfiguráció- és adatbázis-mentés készült: %s.",
|
||||
len(blockedNames), strings.Join(blockedNames, ", ")))
|
||||
}
|
||||
// SLICE 4: approaching the soft quota (≥80%, <100%) — warn on an otherwise-OK run.
|
||||
if qw := offboxQuotaWarning(o); qw != "" {
|
||||
warns = append(warns, qw)
|
||||
@@ -463,6 +506,17 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
if m.offboxNotify != nil {
|
||||
m.offboxNotify(dur, snapshots, runErr)
|
||||
}
|
||||
// Edge-triggered enlarge-blocked notification: only apps that NEWLY crossed into the blocked state
|
||||
// (vs the prior persisted set) notify — a persistently-blocked app never re-notifies nightly. Uses
|
||||
// the pre-run last-known repo size (the same figure the gate used).
|
||||
if runErr == nil && m.offboxEnlargeBlockedNotify != nil && t != nil && t.QuotaGB > 0 {
|
||||
usedGB := int(t.RepoSizeBytes / offboxGiB)
|
||||
for _, b := range runResult.blocked {
|
||||
if !priorBlocked[b.stack] {
|
||||
m.offboxEnlargeBlockedNotify(b.stack, b.estBytes, usedGB, t.QuotaGB)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case runErr != nil:
|
||||
m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
|
||||
@@ -553,12 +607,24 @@ func offboxUnitTime(src, manifestPath string) time.Time {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
|
||||
// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
|
||||
// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
|
||||
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
|
||||
// offboxRunResult carries the outcome of a per-app offbox run: how many apps were backed up, which
|
||||
// had no discoverable unit (skipped), which had their enlargement quota-blocked (unit-only), and the
|
||||
// aggregated Hungarian customer warnings (capture gaps + undeployed).
|
||||
type offboxRunResult struct {
|
||||
backedUp int
|
||||
missing []string
|
||||
blocked []offboxBlocked
|
||||
warns []string
|
||||
}
|
||||
|
||||
// runOffboxInternal does the repo-ensure + per-app DISCOVER → capture-set → gate → multi-path backup +
|
||||
// prune. Caller holds the running flag. Each app's snapshot is ONE multi-path restic snapshot
|
||||
// (recovery unit + the app's MANDATORY offsite capture set, §6). The pre-push enlargement gate (§9,
|
||||
// decision #1) blocks only the ENLARGEMENT — the unit-only push always continues. Returns the result +
|
||||
// the first hard error (repo-ensure or a restic backup exec failure).
|
||||
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string, t *settings.OffboxTarget) (res offboxRunResult, err error) {
|
||||
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
|
||||
return 0, nil, rerr // fail fast (dead NAS surfaces here)
|
||||
return res, rerr // fail fast (dead NAS surfaces here)
|
||||
}
|
||||
// Pre-run hygiene: clear any lock restic can prove stale before we start (cheap; the --remove-all
|
||||
// crash-lock escalation lives in resticStep for the locks restic can't self-detect).
|
||||
@@ -568,11 +634,30 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
src, ok := m.discoverOffboxUnit(stack)
|
||||
if !ok {
|
||||
m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
|
||||
missing = append(missing, stack)
|
||||
res.missing = append(res.missing, stack)
|
||||
continue
|
||||
}
|
||||
// Task 3-core TierOffsite capture set: mandatory userdata paths added to the unit snapshot,
|
||||
// plus loud warnings for structurally-refused / on-disk-missing mandatory paths (SP-3.4).
|
||||
extra, capWarns := m.offboxCaptureSet(stack)
|
||||
res.warns = append(res.warns, capWarns...)
|
||||
// Pre-push enlargement gate (§9): if last-known repo raw-data bytes + the mandatory-set estimate
|
||||
// would cross the soft quota, push UNIT-ONLY (protection never regresses) and record the block.
|
||||
if len(extra) > 0 && t != nil && t.QuotaGB > 0 {
|
||||
var est int64
|
||||
for _, p := range extra {
|
||||
est += m.offboxSize()(p)
|
||||
}
|
||||
if t.RepoSizeBytes+est >= int64(t.QuotaGB)*offboxGiB {
|
||||
m.logger.Printf("[INFO] [offbox] %s: enlargement blocked by quota (est %s + repo %s ≥ %d GB) — unit-only push continues",
|
||||
stack, humanizeBytes(est), humanizeBytes(t.RepoSizeBytes), t.QuotaGB)
|
||||
res.blocked = append(res.blocked, offboxBlocked{stack: stack, estBytes: est})
|
||||
extra = nil
|
||||
}
|
||||
}
|
||||
args := append([]string{"backup", "--tag", "felhom-offbox", "--tag", stack, src}, extra...)
|
||||
bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, "backup", "--tag", "felhom-offbox", "--tag", stack, src)
|
||||
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, args...)
|
||||
cancel()
|
||||
if berr != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
|
||||
@@ -581,25 +666,29 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
}
|
||||
continue
|
||||
}
|
||||
backedUp++
|
||||
m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
|
||||
res.backedUp++
|
||||
m.logger.Printf("[INFO] [offbox] backed up %s (%s, %d mandatory path(s))", stack, src, len(extra))
|
||||
}
|
||||
if firstErr != nil {
|
||||
return backedUp, missing, firstErr
|
||||
return res, firstErr
|
||||
}
|
||||
// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
|
||||
// prune takes an EXCLUSIVE lock — the exact step whose crash left the C2 stale lock — so it goes
|
||||
// through resticStep for the --remove-all self-heal too.
|
||||
// Retention: keep a sane window, prune the rest. SP-2: `--group-by host,tags` so an app's OLD
|
||||
// unit-only-shape snapshots share a group with its NEW enlarged shape (same <stack> tag) and age
|
||||
// out naturally — the default host,paths grouping would strand old-shape snapshots in their own
|
||||
// permanently-retained group. prune takes an EXCLUSIVE lock (the C2 stale-lock step) → resticStep.
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
|
||||
m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
|
||||
}
|
||||
return backedUp, missing, nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic restore-size.
|
||||
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic REPO SIZE. Since
|
||||
// v0.134.0 the repo size is measured with `stats --mode raw-data` (actual deduplicated+compressed
|
||||
// bytes — what the customer's Storage Box really fills), NOT the old modeless restore-size which
|
||||
// multiplied by the retained-snapshot count (SP-1). The displayed size drops one-time after deploy.
|
||||
const offboxGiB = int64(1) << 30
|
||||
|
||||
// OffboxReportStatus is the NON-SECRET offsite summary carried on the hub report (SLICE 4) — the input
|
||||
@@ -673,7 +762,9 @@ func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
// SP-2: `--group-by host,tags` (mirrors runOffboxInternal's forget) so old unit-only-shape snapshots
|
||||
// age out with the enlarged shape instead of stranding in a permanently-retained host,paths group.
|
||||
fargs := append(append([]string{}, base...), "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
|
||||
} else {
|
||||
@@ -695,9 +786,12 @@ func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int
|
||||
if json.Unmarshal(out, &snaps) != nil {
|
||||
return 0
|
||||
}
|
||||
// Repo size (best-effort, restore-size). Bytes feed the soft-quota gate (SLICE 4); a failed stats
|
||||
// call keeps the last-known value (stale-but-safe).
|
||||
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--json")...); serr == nil {
|
||||
// Repo size (best-effort, RAW-DATA mode). SP-1: `--mode raw-data` reports the actual
|
||||
// deduplicated+compressed repo bytes (what the Storage Box really fills), not the modeless
|
||||
// restore-size that multiplies by the retained-snapshot count. Bytes feed the soft-quota gate
|
||||
// (SLICE 4); a failed stats call keeps the last-known value (stale-but-safe). RAW-DATA TRAP:
|
||||
// total_file_count is 0 in this mode — read total_size only.
|
||||
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--mode", "raw-data", "--json")...); serr == nil {
|
||||
var st struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,542 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// mandAbs builds the capture-set Abs the code produces: ComputeCaptureSet uses path.Join (slash) —
|
||||
// the 3-core separator rule — so on the Windows test host the mandatory path is drive + "/rel".
|
||||
func mandAbs(drive, rel string) string { return pathpkg.Join(drive, rel) }
|
||||
|
||||
// offbox3aProvider is a configurable StackDataProvider for the 3a capture-set tests: per-stack HDD
|
||||
// path + classified binds.
|
||||
type offbox3aProvider struct {
|
||||
hdd map[string]string
|
||||
binds map[string][]ClassifiedBind
|
||||
has map[string]bool
|
||||
}
|
||||
|
||||
func (p *offbox3aProvider) GetStackComposePath(string) (string, bool) { return "", false }
|
||||
func (p *offbox3aProvider) ListDeployedStacks() []StackSummary { return nil }
|
||||
func (p *offbox3aProvider) GetStackHDDMounts(string) []string { return nil }
|
||||
func (p *offbox3aProvider) GetStackHDDPath(n string) string { return p.hdd[n] }
|
||||
func (p *offbox3aProvider) GetDockerVolumes(string) []string { return nil }
|
||||
func (p *offbox3aProvider) StopStack(string) error { return nil }
|
||||
func (p *offbox3aProvider) StartStack(string) error { return nil }
|
||||
func (p *offbox3aProvider) RefreshAndIsRunning(string) bool { return false }
|
||||
func (p *offbox3aProvider) GetStackRecoveryInfo(string) (RecoveryInfo, bool) { return RecoveryInfo{}, false }
|
||||
func (p *offbox3aProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
|
||||
func (p *offbox3aProvider) RecreateStackFromUnit(_, _ string, _ map[string]string) error { return nil }
|
||||
func (p *offbox3aProvider) GetStackClassifiedBinds(n string) ([]ClassifiedBind, bool) {
|
||||
return p.binds[n], p.has[n]
|
||||
}
|
||||
|
||||
func mandatoryHDD(rel string) ClassifiedBind {
|
||||
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootHDD, RelPath: rel}, Class: appbackup.ClassMandatory}
|
||||
}
|
||||
func optionalUserdata(rel string) ClassifiedBind {
|
||||
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootUserdata, RelPath: rel, ReadOnly: true}, Class: appbackup.ClassOptional}
|
||||
}
|
||||
func excludedHDD(rel string) ClassifiedBind {
|
||||
return ClassifiedBind{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootHDD, RelPath: rel}, Class: appbackup.ClassExcluded}
|
||||
}
|
||||
|
||||
// classifiedOffboxManager: a configured offbox manager + a classified provider + the drive registered
|
||||
// as a schedulable storage path (so discoverOffboxUnit finds units on it).
|
||||
func classifiedOffboxManager(t *testing.T, drive string) (*Manager, *settings.Settings, *offbox3aProvider) {
|
||||
t.Helper()
|
||||
m, sett := newOffboxManager(t)
|
||||
prov := &offbox3aProvider{hdd: map[string]string{}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{}}
|
||||
m.SetStackProvider(prov)
|
||||
if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "USB", Schedulable: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, sett, prov
|
||||
}
|
||||
|
||||
// mkUnit lays down a discoverable recovery unit for stack on drive.
|
||||
func mkUnit(t *testing.T, drive, stack string) string {
|
||||
t.Helper()
|
||||
u := RecoveryUnitPath(drive, stack)
|
||||
if err := os.MkdirAll(u, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// captureBackupRunner records the FULL argv of each backup call (keyed by stack tag) + forget argv, and
|
||||
// answers the probes so RunOffboxBackup completes.
|
||||
type backupCapture struct {
|
||||
mu sync.Mutex
|
||||
byStack map[string][]string
|
||||
forgets [][]string
|
||||
backups int
|
||||
}
|
||||
|
||||
func (c *backupCapture) runner() offboxRunner {
|
||||
c.byStack = map[string][]string{}
|
||||
return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
switch {
|
||||
case contains(args, "cat") && contains(args, "config"):
|
||||
return []byte(`{"version":2}`), nil
|
||||
case contains(args, "backup"):
|
||||
c.backups++
|
||||
c.byStack[tagOf(args)] = append([]string{}, args...)
|
||||
return nil, nil
|
||||
case contains(args, "forget"):
|
||||
c.forgets = append(c.forgets, append([]string{}, args...))
|
||||
return nil, nil
|
||||
case contains(args, "snapshots"):
|
||||
return []byte(`[]`), nil
|
||||
case contains(args, "stats"):
|
||||
return []byte(`{"total_size":123}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario A: classified enlarged push (immich shape) ---
|
||||
|
||||
func TestOffbox3a_EnlargedPush_MandatoryOnly(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
unit := mkUnit(t, drive, "immich")
|
||||
if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The optional :ro library exists on disk — so if the tier filter ever leaked it, the stat-filter
|
||||
// would NOT hide it (this makes the RP-A tier-filter red-proof observable).
|
||||
if err := os.MkdirAll(filepath.Join(drive, "userdata", "media", "photos"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prov.hdd["immich"] = drive
|
||||
prov.has["immich"] = true
|
||||
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich"), optionalUserdata("media/photos")}
|
||||
_ = sett.SetAppOffbox("immich", true)
|
||||
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
if cap.backups != 1 {
|
||||
t.Fatalf("exactly ONE snapshot per app, got %d backup calls", cap.backups)
|
||||
}
|
||||
args := cap.byStack["immich"]
|
||||
wantMandatory := mandAbs(drive, "appdata/immich")
|
||||
if !contains(args, unit) {
|
||||
t.Errorf("backup argv missing the unit path %q: %v", unit, args)
|
||||
}
|
||||
if !contains(args, wantMandatory) {
|
||||
t.Errorf("backup argv missing the mandatory userdata path %q: %v", wantMandatory, args)
|
||||
}
|
||||
if contains(args, mandAbs(drive, "userdata/media/photos")) {
|
||||
t.Errorf("OPTIONAL :ro path must NOT ship offsite: %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario B: legacy / undeployed stay unit-only ---
|
||||
|
||||
func TestOffbox3a_LegacyUnitOnly(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
unit := mkUnit(t, drive, "sonarr")
|
||||
if err := os.MkdirAll(filepath.Join(drive, "appdata", "sonarr"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prov.hdd["sonarr"] = drive
|
||||
prov.has["sonarr"] = false // block REJECTED / absent → legacy (binds present but no class semantics)
|
||||
prov.binds["sonarr"] = []ClassifiedBind{mandatoryHDD("appdata/sonarr")}
|
||||
_ = sett.SetAppOffbox("sonarr", true)
|
||||
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
args := cap.byStack["sonarr"]
|
||||
// unit-only: exactly the base shape, last arg is the unit, no extra resolved paths.
|
||||
if args[len(args)-1] != unit {
|
||||
t.Errorf("legacy app argv must END at the unit (no resolved paths), got %v", args)
|
||||
}
|
||||
for _, a := range args {
|
||||
if strings.Contains(a, "appdata") || strings.Contains(a, "userdata") {
|
||||
t.Errorf("legacy app resolved a bind into offsite argv (SQ5 regression): %v", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOffbox3a_UndeployedUnitOnlyWithWarning(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
_ = mkUnit(t, drive, "immich")
|
||||
prov.hdd["immich"] = "" // undeployed → no live HDD_PATH
|
||||
prov.has["immich"] = true
|
||||
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
|
||||
_ = sett.SetAppOffbox("immich", true)
|
||||
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
args := cap.byStack["immich"]
|
||||
if strings.Contains(strings.Join(args, " "), "appdata") {
|
||||
t.Errorf("undeployed app must push unit-only: %v", args)
|
||||
}
|
||||
if w := sett.GetOffboxTarget().LastWarning; !strings.Contains(w, "nincs telepítve") {
|
||||
t.Errorf("undeployed warning missing from LastWarning: %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario C: pre-push enlargement gate ---
|
||||
|
||||
func TestOffbox3a_EnlargementGateBlocks(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
for _, app := range []string{"immich", "small"} {
|
||||
_ = mkUnit(t, drive, app)
|
||||
if err := os.MkdirAll(filepath.Join(drive, "appdata", app), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prov.hdd[app] = drive
|
||||
prov.has[app] = true
|
||||
prov.binds[app] = []ClassifiedBind{mandatoryHDD("appdata/" + app)}
|
||||
_ = sett.SetAppOffbox(app, true)
|
||||
}
|
||||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.QuotaGB = 50; o.RepoSizeBytes = 20 << 30 })
|
||||
// immich's mandatory set is 40 GiB (20+40 ≥ 50 → blocked); small's is 1 GiB (20+1 < 50 → fits).
|
||||
m.SetOffboxSizer(func(p string) int64 {
|
||||
if strings.Contains(p, "immich") {
|
||||
return 40 << 30
|
||||
}
|
||||
return 1 << 30
|
||||
})
|
||||
var noteMu sync.Mutex
|
||||
var notes []string
|
||||
m.SetOffboxEnlargeBlockedNotifier(func(stack string, _ int64, usedGB, quotaGB int) {
|
||||
noteMu.Lock()
|
||||
defer noteMu.Unlock()
|
||||
notes = append(notes, stack)
|
||||
if usedGB != 20 || quotaGB != 50 {
|
||||
t.Errorf("notifier numbers wrong: used=%d quota=%d", usedGB, quotaGB)
|
||||
}
|
||||
})
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run must be OK (a blocked enlargement is not a run failure): %v", err)
|
||||
}
|
||||
// immich → unit-only; small → enlarged.
|
||||
if strings.Contains(strings.Join(cap.byStack["immich"], " "), "appdata") {
|
||||
t.Errorf("blocked immich must be unit-only: %v", cap.byStack["immich"])
|
||||
}
|
||||
if !contains(cap.byStack["small"], mandAbs(drive, "appdata/small")) {
|
||||
t.Errorf("fitting 'small' must still push enlarged: %v", cap.byStack["small"])
|
||||
}
|
||||
tgt := sett.GetOffboxTarget()
|
||||
if len(tgt.EnlargedBlocked) != 1 || tgt.EnlargedBlocked[0] != "immich" {
|
||||
t.Errorf("EnlargedBlocked = %v, want [immich]", tgt.EnlargedBlocked)
|
||||
}
|
||||
if tgt.LastStatus != "ok" {
|
||||
t.Errorf("run status = %q, want ok", tgt.LastStatus)
|
||||
}
|
||||
if !strings.Contains(tgt.LastWarning, "tárhelykeret miatt") || !strings.Contains(tgt.LastWarning, "immich") {
|
||||
t.Errorf("blocked LastWarning missing: %q", tgt.LastWarning)
|
||||
}
|
||||
if len(notes) != 1 || notes[0] != "immich" {
|
||||
t.Errorf("notifier must fire ONCE for immich, got %v", notes)
|
||||
}
|
||||
|
||||
// EnlargedBlocked clears on a subsequent run where nothing is blocked.
|
||||
m.SetOffboxSizer(func(string) int64 { return 1 << 30 }) // now immich fits too
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := sett.GetOffboxTarget().EnlargedBlocked; len(b) != 0 {
|
||||
t.Errorf("EnlargedBlocked must clear when nothing is blocked, got %v", b)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario D: capture gaps are loud (SP-3.4) ---
|
||||
|
||||
func TestOffbox3a_CaptureGapsAreLoud(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
_ = mkUnit(t, drive, "app")
|
||||
if err := os.MkdirAll(filepath.Join(drive, "appdata", "good"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prov.hdd["app"] = drive
|
||||
prov.has["app"] = true
|
||||
prov.binds["app"] = []ClassifiedBind{
|
||||
mandatoryHDD("appdata/good"), // exists → captured
|
||||
mandatoryHDD("../evil"), // D1: traversal → Skipped
|
||||
mandatoryHDD("appdata/ghost"), // D2: passes guards but absent on disk → stat-filtered
|
||||
}
|
||||
_ = sett.SetAppOffbox("app", true)
|
||||
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
args := cap.byStack["app"]
|
||||
joined := strings.Join(args, " ")
|
||||
if !contains(args, mandAbs(drive, "appdata/good")) {
|
||||
t.Errorf("the valid mandatory path must still push: %v", args)
|
||||
}
|
||||
if strings.Contains(joined, "evil") {
|
||||
t.Errorf("traversal path escaped into argv: %v", args)
|
||||
}
|
||||
if strings.Contains(joined, "ghost") {
|
||||
t.Errorf("stat-missing mandatory path must NOT be in argv (SP-3.4 silent-skip): %v", args)
|
||||
}
|
||||
if w := sett.GetOffboxTarget().LastWarning; !strings.Contains(w, "nem kerültek a távoli mentésbe") {
|
||||
t.Errorf("capture-gap warning missing from LastWarning: %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
// --- §8 all-excluded row (radarr shape): unit-only, NO warning ---
|
||||
|
||||
func TestOffbox3a_AllExcludedUnitOnlyNoWarning(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
unit := mkUnit(t, drive, "radarr")
|
||||
prov.hdd["radarr"] = drive
|
||||
prov.has["radarr"] = true
|
||||
prov.binds["radarr"] = []ClassifiedBind{excludedHDD("appdata/radarr"), excludedHDD("downloads")}
|
||||
_ = sett.SetAppOffbox("radarr", true)
|
||||
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
if args := cap.byStack["radarr"]; args[len(args)-1] != unit {
|
||||
t.Errorf("all-excluded app must be unit-only: %v", args)
|
||||
}
|
||||
if w := sett.GetOffboxTarget().LastWarning; strings.Contains(w, "nem kerültek") {
|
||||
t.Errorf("all-excluded is correct, NOT a gap — no warning expected, got %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario E: raw-data stats mode ---
|
||||
|
||||
func TestOffbox3a_StatsRawDataMode(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
var statsArgs []string
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case contains(args, "snapshots"):
|
||||
return []byte(`[{"id":"a"}]`), nil
|
||||
case contains(args, "stats"):
|
||||
statsArgs = append([]string{}, args...)
|
||||
return []byte(`{"total_size":987654321}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
|
||||
m.offboxRecordStats(context.Background(), base, env)
|
||||
if !contains(statsArgs, "--mode") || valAfter(statsArgs, "--mode") != "raw-data" {
|
||||
t.Fatalf("stats must run in raw-data mode, got %v", statsArgs)
|
||||
}
|
||||
if got := sett.GetOffboxTarget().RepoSizeBytes; got != 987654321 {
|
||||
t.Errorf("RepoSizeBytes = %d, want 987654321 (parsed from raw-data total_size)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario F: both forget call sites carry --group-by host,tags ---
|
||||
|
||||
func TestOffbox3a_ForgetGrouping_MainRun(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, sett, prov := classifiedOffboxManager(t, drive)
|
||||
_ = mkUnit(t, drive, "app")
|
||||
prov.has["app"] = false
|
||||
_ = sett.SetAppOffbox("app", true)
|
||||
cap := &backupCapture{}
|
||||
m.SetOffboxRunner(cap.runner())
|
||||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cap.forgets) != 1 {
|
||||
t.Fatalf("expected one forget call, got %d", len(cap.forgets))
|
||||
}
|
||||
if valAfter(cap.forgets[0], "--group-by") != "host,tags" {
|
||||
t.Errorf("main-run forget missing --group-by host,tags: %v", cap.forgets[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOffbox3a_ForgetGrouping_OverQuotaPrune(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.QuotaGB = 50; o.RepoSizeBytes = 51 << 30 })
|
||||
var forgetArgs []string
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case contains(args, "cat") && contains(args, "config"):
|
||||
return []byte(`{}`), nil
|
||||
case contains(args, "forget"):
|
||||
forgetArgs = append([]string{}, args...)
|
||||
case contains(args, "snapshots"):
|
||||
return []byte(`[]`), nil
|
||||
case contains(args, "stats"):
|
||||
return []byte(`{"total_size":1}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
_ = m.RunOffboxBackup(context.Background()) // over-quota → prune-only path
|
||||
if valAfter(forgetArgs, "--group-by") != "host,tags" {
|
||||
t.Errorf("over-quota prune forget missing --group-by host,tags: %v", forgetArgs)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario E-restore: unit-only restore argv (ID-first + --include) + scratch OFF the rootfs ---
|
||||
|
||||
func TestOffbox3a_UnitOnlyRestoreArgv(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, _, prov := classifiedOffboxManager(t, drive)
|
||||
prov.hdd["immich"] = drive
|
||||
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 }) // plenty
|
||||
unitPath := filepath.ToSlash(filepath.Join(drive, "backups", "primary", "immich"))
|
||||
var restoreArgs []string
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case contains(args, "snapshots"):
|
||||
return []byte(`[{"short_id":"deadbeef","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(filepath.Join(drive, "appdata", "immich")) + `","` + unitPath + `"]}]`), nil
|
||||
case contains(args, "restore"):
|
||||
restoreArgs = append([]string{}, args...)
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err := m.RestoreOffboxScratch(context.Background(), "immich", false); err != nil {
|
||||
t.Fatalf("unit-only restore: %v", err)
|
||||
}
|
||||
if valAfter(restoreArgs, "restore") != "deadbeef" {
|
||||
t.Errorf("restore must be ID-first (deadbeef): %v", restoreArgs)
|
||||
}
|
||||
if valAfter(restoreArgs, "--include") != unitPath {
|
||||
t.Errorf("unit-only restore must --include the absolute unit path %q: %v", unitPath, restoreArgs)
|
||||
}
|
||||
target := valAfter(restoreArgs, "--target")
|
||||
if !strings.HasPrefix(target, drive) || strings.Contains(target, m.cfg.Paths.DataDir) {
|
||||
t.Errorf("scratch target must be on the data drive, never DataDir: %q", target)
|
||||
}
|
||||
}
|
||||
|
||||
// full restore refuses fail-closed when the snapshot size is unknown (no restore call made).
|
||||
func TestOffbox3a_FullRestoreRefusesOnSizeUnknown(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, _, prov := classifiedOffboxManager(t, drive)
|
||||
prov.hdd["immich"] = drive
|
||||
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
|
||||
unitPath := filepath.Join(drive, "backups", "primary", "immich")
|
||||
restoreCalled := false
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case contains(args, "snapshots"):
|
||||
return []byte(`[{"short_id":"a","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(unitPath) + `"]}]`), nil
|
||||
case contains(args, "stats"):
|
||||
return nil, context.DeadlineExceeded // size lookup fails → unknown
|
||||
case contains(args, "restore"):
|
||||
restoreCalled = true
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
err := m.RestoreOffboxScratch(context.Background(), "immich", true)
|
||||
if err == nil || !strings.Contains(err.Error(), "nem állapítható meg") {
|
||||
t.Fatalf("full restore must refuse fail-closed on unknown size, got err=%v", err)
|
||||
}
|
||||
if restoreCalled {
|
||||
t.Error("no restic restore call may run when the size is unknown")
|
||||
}
|
||||
}
|
||||
|
||||
// old rootfs scratch is cleaned up on a new restore.
|
||||
func TestOffbox3a_LegacyRootfsScratchCleanup(t *testing.T) {
|
||||
drive := t.TempDir()
|
||||
m, _, prov := classifiedOffboxManager(t, drive)
|
||||
prov.hdd["immich"] = drive
|
||||
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
|
||||
legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", "immich")
|
||||
if err := os.MkdirAll(legacy, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unitPath := filepath.Join(drive, "backups", "primary", "immich")
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
if contains(args, "snapshots") {
|
||||
return []byte(`[{"short_id":"a","time":"2026-07-14T00:00:00Z","paths":["` + filepath.ToSlash(unitPath) + `"]}]`), nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err := m.RestoreOffboxScratch(context.Background(), "immich", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(legacy); !os.IsNotExist(err) {
|
||||
t.Errorf("legacy rootfs scratch %s must be removed, stat err=%v", legacy, err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario G: place-to-live mapping (pure) + the wrong cases ---
|
||||
|
||||
func TestMapOffsiteRestorePaths(t *testing.T) {
|
||||
old := "/old/ns"
|
||||
newNs := "/new/ns"
|
||||
scratch := "/scratch"
|
||||
snap := []string{
|
||||
old + "/backups/primary/app",
|
||||
old + "/appdata/app",
|
||||
old + "/userdata/media/x",
|
||||
}
|
||||
got, err := mapOffsiteRestorePaths(snap, "app", scratch, newNs)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("got %d placements, want 3: %+v", len(got), got)
|
||||
}
|
||||
byDst := map[string]placement{}
|
||||
for _, pl := range got {
|
||||
byDst[pl.dst] = pl
|
||||
}
|
||||
// anchor derived by trimming backups/primary/app off the unit path → oldNs; dst = newNs/<rel>,
|
||||
// src = scratch/<abs-source> (SP-3.1). Built with filepath.Join to match the code (OS separators).
|
||||
check := func(snapPath, rel string, isUnit bool) {
|
||||
dst := filepath.Join(newNs, rel)
|
||||
pl, ok := byDst[dst]
|
||||
if !ok {
|
||||
t.Errorf("missing placement for dst %q", dst)
|
||||
return
|
||||
}
|
||||
if pl.src != filepath.Join(scratch, snapPath) {
|
||||
t.Errorf("src for %q = %q, want %q", snapPath, pl.src, filepath.Join(scratch, snapPath))
|
||||
}
|
||||
if pl.isUnit != isUnit {
|
||||
t.Errorf("isUnit for %q = %v, want %v", snapPath, pl.isUnit, isUnit)
|
||||
}
|
||||
}
|
||||
check(old+"/backups/primary/app", "backups/primary/app", true)
|
||||
check(old+"/appdata/app", "appdata/app", false)
|
||||
check(old+"/userdata/media/x", "userdata/media/x", false)
|
||||
|
||||
// Wrong cases — each REFUSES the whole placement.
|
||||
if _, err := mapOffsiteRestorePaths([]string{old + "/appdata/app"}, "app", scratch, newNs); err == nil {
|
||||
t.Error("no unit path → must refuse")
|
||||
}
|
||||
if _, err := mapOffsiteRestorePaths([]string{old + "/backups/primary/app", "/elsewhere/x"}, "app", scratch, newNs); err == nil {
|
||||
t.Error("a path outside the namespace → must refuse")
|
||||
}
|
||||
if _, err := mapOffsiteRestorePaths([]string{old + "/backups/primary/app", old + "/backups/secondary/y"}, "app", scratch, newNs); err == nil {
|
||||
t.Error("a non-unit path in the reserved backups/ zone → must refuse")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
)
|
||||
|
||||
// Offsite capture-set resolution (Task 3a, architecture doc §2/§6). Turns an app's Task-3-core
|
||||
// TierOffsite capture set (recovery unit + MANDATORY userdata only) into the extra absolute paths
|
||||
// appended to the app's restic snapshot, plus the Hungarian customer warnings for LOUD capture gaps.
|
||||
//
|
||||
// SP-3.4 is law here: restic 0.14.0 does NOT error on a missing source path — it skips with a warning,
|
||||
// exits 0, and silently writes a partial snapshot. So a skipped/missing MANDATORY path is detected in
|
||||
// THIS function (the structural-guard Skipped list + an os.Stat filter) and surfaced in BOTH the
|
||||
// English log and the Hungarian LastWarning. A restic exit code proves nothing about a missing path.
|
||||
|
||||
// offboxBlocked records an app whose enlarged (userdata-carrying) push was refused by the pre-push
|
||||
// quota gate. The unit-only push still proceeds (never a protection regression). estBytes is the
|
||||
// mandatory-set size estimate that would have been added.
|
||||
type offboxBlocked struct {
|
||||
stack string
|
||||
estBytes int64
|
||||
}
|
||||
|
||||
// offboxCaptureSet computes an app's OFFSITE mandatory capture paths to add to its recovery-unit
|
||||
// snapshot, plus any Hungarian warnings for capture gaps. It never returns optional/excluded paths
|
||||
// (the TierOffsite filter drops them — §2). Returns (nil, nil) for the legacy / no-provider / no-block
|
||||
// world: offsite stays UNIT-ONLY, byte-identical to pre-v0.134.0 (the SQ5 cost-regression guard).
|
||||
func (m *Manager) offboxCaptureSet(stack string) (extra []string, warns []string) {
|
||||
if m.stackProvider == nil {
|
||||
return nil, nil // no provider wired → legacy world → unit only
|
||||
}
|
||||
binds, has := m.stackProvider.GetStackClassifiedBinds(stack)
|
||||
if !has {
|
||||
return nil, nil // no backup block → legacy → unit only
|
||||
}
|
||||
// Resolve against the app's LIVE HDD_PATH (raw — NOT GetAppDrivePath, whose systemDataPath fallback
|
||||
// would resolve userdata onto the wrong drive). Empty ⇒ undeployed / no HDD (decision §2.4):
|
||||
// mandatory-path resolution needs the live HDD_PATH, so push unit-only + a loud WARN.
|
||||
hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack))
|
||||
if hdd == "" {
|
||||
m.logger.Printf("[WARN] [offbox] %s: not deployed — offsite push is unit-only (mandatory userdata not resolvable)", stack)
|
||||
return nil, []string{fmt.Sprintf("Figyelmeztetés: a(z) %s nincs telepítve — csak a mentési egység került a távoli mentésbe.", stack)}
|
||||
}
|
||||
nsRoot := m.namespaceRoot(hdd)
|
||||
cs := appbackup.ComputeCaptureSet(binds, has, appbackup.TierOffsite, nsRoot)
|
||||
|
||||
var gaps []string
|
||||
// Structurally-refused MANDATORY paths (traversal / bare drive-root / reserved backups/ zone) are
|
||||
// loud ERROR gaps — the path the customer thinks is protected is not in the snapshot.
|
||||
for _, sk := range cs.Skipped {
|
||||
if sk.Class == appbackup.ClassMandatory {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: mandatory path refused by a structural guard (%s): %s/%s — NOT in the offsite snapshot",
|
||||
stack, sk.Reason, sk.Root, sk.RelPath)
|
||||
gaps = append(gaps, sk.RelPath)
|
||||
}
|
||||
}
|
||||
// Stat-filter (§2.5): a declared mandatory path absent on disk. restic would skip it SILENTLY
|
||||
// (SP-3.4), so drop it from argv AND warn — never a silent "looks backed up but isn't".
|
||||
for _, p := range cs.Paths {
|
||||
if _, err := os.Stat(p.Abs); err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: mandatory data path missing on disk, skipped from offsite: %s", stack, p.Abs)
|
||||
gaps = append(gaps, p.RelPath)
|
||||
continue
|
||||
}
|
||||
extra = append(extra, p.Abs)
|
||||
}
|
||||
if len(gaps) > 0 {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: a(z) %s alkalmazás egyes adatmappái nem kerültek a távoli mentésbe: %s.",
|
||||
stack, strings.Join(gaps, ", ")))
|
||||
}
|
||||
return extra, warns
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Offsite restore rework (Task 3a §7). With mandatory userdata now in snapshots, restore needs three
|
||||
// changes over the old dump-to-rootfs-scratch:
|
||||
// 1. scratch relocated off the ~8 GB guest rootfs onto a data drive, behind a headroom gate (F-A1);
|
||||
// 2. a unit-only DEFAULT restore (`--include <absolute-unit-path>`, SP-3.2) — full is a deliberate,
|
||||
// size-gated second action;
|
||||
// 3. place-to-live = a missing-only merge (never --delete) so the SQ3 immich case is restorable
|
||||
// from offsite alone.
|
||||
// ID-first everywhere (§3): `restic stats --tag` is UNPROVEN on 0.14.0, so the size lookup resolves the
|
||||
// snapshot ID via `snapshots latest --tag` and calls `stats <ID>`.
|
||||
|
||||
const (
|
||||
// offboxUnitOnlyFreeFloor — a unit-only restore needs at least this much free on the scratch drive.
|
||||
// Catalog recovery units are MB–1 GB (SQ4); 2 GiB is a safe floor without a per-snapshot size probe.
|
||||
offboxUnitOnlyFreeFloor = int64(2) << 30
|
||||
)
|
||||
|
||||
// SetOffboxFreeFn overrides the restore free-space probe (tests; the Windows go-test host has no df).
|
||||
func (m *Manager) SetOffboxFreeFn(fn func(path string) int64) { m.offboxFreeFn = fn }
|
||||
|
||||
// offboxFree returns the free-space probe (nil seam → the real diskFreeBytes).
|
||||
func (m *Manager) offboxFree() func(string) int64 {
|
||||
if m.offboxFreeFn != nil {
|
||||
return m.offboxFreeFn
|
||||
}
|
||||
return diskFreeBytes
|
||||
}
|
||||
|
||||
// diskFreeBytes returns available bytes on the filesystem holding path (0 on any error). Mirrors
|
||||
// appexport.DiskFree; kept local so the backup package needs no cross-package dependency.
|
||||
func diskFreeBytes(path string) int64 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "df", "--output=avail", "-B1", path).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
if len(lines) < 2 {
|
||||
return 0
|
||||
}
|
||||
var size int64
|
||||
fmt.Sscanf(strings.TrimSpace(lines[1]), "%d", &size)
|
||||
return size
|
||||
}
|
||||
|
||||
// offboxUnitPathOf returns the snapshot path that is the recovery unit for stack (suffix
|
||||
// backups/primary/<stack>), or "" if none is present.
|
||||
func offboxUnitPathOf(paths []string, stack string) string {
|
||||
suffix := "/backups/primary/" + stack
|
||||
for _, p := range paths {
|
||||
if strings.HasSuffix(p, suffix) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// offboxLatestSnapshot resolves the newest snapshot for stack: its short ID + captured paths, via
|
||||
// `snapshots latest --tag <stack> --json`. When the tag spans more than one group (old unit-only shape
|
||||
// + new enlarged shape), it returns the newest by time.
|
||||
func (m *Manager) offboxLatestSnapshot(ctx context.Context, stack string) (id string, paths []string, err error) {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
out, serr := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "latest", "--tag", stack, "--json")...)
|
||||
if serr != nil {
|
||||
return "", nil, fmt.Errorf("offbox snapshots %s: %w: %s", stack, serr, truncate(out))
|
||||
}
|
||||
var snaps []struct {
|
||||
ShortID string `json:"short_id"`
|
||||
ID string `json:"id"`
|
||||
Time time.Time `json:"time"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
if json.Unmarshal(out, &snaps) != nil || len(snaps) == 0 {
|
||||
return "", nil, fmt.Errorf("offbox: nincs pillanatkép a(z) %s alkalmazáshoz", stack)
|
||||
}
|
||||
best := 0
|
||||
for i := 1; i < len(snaps); i++ {
|
||||
if snaps[i].Time.After(snaps[best].Time) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
id = snaps[best].ShortID
|
||||
if id == "" {
|
||||
id = snaps[best].ID
|
||||
}
|
||||
return id, snaps[best].Paths, nil
|
||||
}
|
||||
|
||||
// offboxSnapshotSize returns the restore-size (logical bytes) of ONE snapshot via `stats <ID> --json`
|
||||
// (default mode — for a single snapshot ID this is exactly that snapshot's on-disk-when-restored size,
|
||||
// the correct headroom meaning; SP-1). ID-first: never `stats --tag` (unproven on 0.14.0).
|
||||
func (m *Manager) offboxSnapshotSize(ctx context.Context, id string) (int64, error) {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
out, err := m.runner()(sctx, env, append(append([]string{}, base...), "stats", id, "--json")...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("offbox stats %s: %w: %s", id, err, truncate(out))
|
||||
}
|
||||
var st struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
if json.Unmarshal(out, &st) != nil || st.TotalSize <= 0 {
|
||||
return 0, fmt.Errorf("offbox: a(z) %s pillanatkép mérete ismeretlen", id)
|
||||
}
|
||||
return st.TotalSize, nil
|
||||
}
|
||||
|
||||
// offboxRestoreScratchDir returns the on-DATA-DRIVE scratch dir for an app's offsite restore
|
||||
// (<nsRoot>/backups/offsite-restore/<app>) plus the namespace root (an existing dir, for the free-space
|
||||
// probe). NEVER cfg.Paths.DataDir (the rootfs — the F-A1 filler). App's HDD drive first; else the first
|
||||
// schedulable storage path; else a Hungarian refusal.
|
||||
func (m *Manager) offboxRestoreScratchDir(stack string) (scratch, nsRoot string, err error) {
|
||||
if m.stackProvider != nil {
|
||||
if hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)); hdd != "" {
|
||||
nr := m.namespaceRoot(hdd)
|
||||
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
|
||||
}
|
||||
}
|
||||
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
||||
if strings.TrimSpace(sp.Path) != "" {
|
||||
nr := m.namespaceRoot(sp.Path)
|
||||
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
|
||||
}
|
||||
}
|
||||
return "", "", fmt.Errorf("nincs elérhető adatmeghajtó a visszaállításhoz")
|
||||
}
|
||||
|
||||
// RestoreOffboxScratch restores an app's latest offsite snapshot to an on-data-drive scratch dir
|
||||
// (non-destructive — never overwrites live data). full=false (the default) restores the recovery UNIT
|
||||
// only (`--include <absolute-unit-path>`, SP-3.2); full=true restores the whole snapshot (unit +
|
||||
// mandatory userdata) behind a size×1.1 headroom gate. Fail-closed: an unknown snapshot size refuses a
|
||||
// full restore.
|
||||
func (m *Manager) RestoreOffboxScratch(ctx context.Context, stack string, full bool) error {
|
||||
if !m.OffboxConfigured() {
|
||||
return fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return fmt.Errorf("invalid stack name")
|
||||
}
|
||||
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unitPath := offboxUnitPathOf(paths, stack)
|
||||
if unitPath == "" {
|
||||
return fmt.Errorf("a(z) %s pillanatképében nincs mentési egység — a visszaállítás nem indítható", stack)
|
||||
}
|
||||
scratch, nsRoot, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Headroom gate (F-A1) — probed on the namespace root (an existing dir).
|
||||
free := m.offboxFree()(nsRoot)
|
||||
if full {
|
||||
size, serr := m.offboxSnapshotSize(ctx, id)
|
||||
if serr != nil {
|
||||
// SizeUnknown never renders as fits — fail closed.
|
||||
return fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
|
||||
}
|
||||
need := size + size/10 // ×1.1
|
||||
if free < need {
|
||||
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
|
||||
}
|
||||
} else if free < offboxUnitOnlyFreeFloor {
|
||||
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(offboxUnitOnlyFreeFloor), humanizeBytes(free))
|
||||
}
|
||||
// F-A1 hygiene: drop the legacy rootfs scratch (DataDir/offbox-restore/<app>) best-effort.
|
||||
legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", stack)
|
||||
if _, sErr := os.Stat(legacy); sErr == nil {
|
||||
if rmErr := os.RemoveAll(legacy); rmErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] could not remove legacy rootfs restore scratch %s: %v", legacy, rmErr)
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] removed legacy rootfs restore scratch %s", legacy)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(scratch, 0o755); err != nil {
|
||||
return fmt.Errorf("restore dir: %w", err)
|
||||
}
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
m.unlockStale(rctx, base, env) // pre-restore hygiene
|
||||
args := []string{"restore", id, "--target", scratch}
|
||||
if !full {
|
||||
args = append(args, "--include", unitPath) // SP-3.2: absolute snapshot unit path = unit-only
|
||||
}
|
||||
out, rerr := m.resticStep(rctx, env, base, "restore:"+stack, args...)
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("offbox restore %s: %w: %s", stack, rerr, truncate(out))
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] restored %s (%s, full=%v) → %s", stack, id, full, scratch)
|
||||
return nil
|
||||
}
|
||||
|
||||
// OffboxRestorePrepareFull resolves the latest snapshot's restore-size and verifies scratch headroom
|
||||
// for a FULL restore WITHOUT starting it (the two-step size-first gate). Returns the human size on
|
||||
// success, or a Hungarian error to flash on refusal (size unknown / no headroom — fail-closed).
|
||||
func (m *Manager) OffboxRestorePrepareFull(ctx context.Context, stack string) (string, error) {
|
||||
if !m.OffboxConfigured() {
|
||||
return "", fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return "", fmt.Errorf("invalid stack name")
|
||||
}
|
||||
id, _, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
size, serr := m.offboxSnapshotSize(ctx, id)
|
||||
if serr != nil {
|
||||
return "", fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
|
||||
}
|
||||
_, nsRoot, derr := m.offboxRestoreScratchDir(stack)
|
||||
if derr != nil {
|
||||
return "", derr
|
||||
}
|
||||
need := size + size/10
|
||||
if free := m.offboxFree()(nsRoot); free < need {
|
||||
return "", fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
|
||||
}
|
||||
return humanizeBytes(size), nil
|
||||
}
|
||||
|
||||
// OffboxFullScratchReady reports whether a (non-empty) full-restore scratch exists for stack — the gate
|
||||
// for showing the place-to-live action. PlaceOffsiteRestore re-validates per-path completeness.
|
||||
func (m *Manager) OffboxFullScratchReady(stack string) bool {
|
||||
if !isSafeStackName(stack) {
|
||||
return false
|
||||
}
|
||||
scratch, _, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if fi, sErr := os.Stat(scratch); sErr != nil || !fi.IsDir() {
|
||||
return false
|
||||
}
|
||||
entries, _ := os.ReadDir(scratch)
|
||||
return len(entries) > 0
|
||||
}
|
||||
|
||||
// placement is one source→dest pair for place-to-live: src is the reconstructed absolute path under the
|
||||
// scratch (SP-3.1), dst is the live location under the app's current namespace root.
|
||||
type placement struct {
|
||||
src string
|
||||
dst string
|
||||
isUnit bool
|
||||
}
|
||||
|
||||
// mapOffsiteRestorePaths maps a completed full-scratch restore to live placements (pure). The anchor
|
||||
// oldNs is derived by trimming backups/primary/<stack> off the unit path (the snapshot may come from a
|
||||
// DIFFERENT drive after churn — liveNsRoot is where it goes). Refuses the WHOLE placement (no partial
|
||||
// writes) on: no unit path; a path outside oldNs (escape); a `..` segment; a non-unit path in the
|
||||
// reserved backups/ zone.
|
||||
func mapOffsiteRestorePaths(snapPaths []string, stack, scratch, liveNsRoot string) ([]placement, error) {
|
||||
unitSuffix := "/backups/primary/" + stack
|
||||
oldNs := ""
|
||||
for _, p := range snapPaths {
|
||||
if strings.HasSuffix(p, unitSuffix) {
|
||||
oldNs = strings.TrimSuffix(p, unitSuffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
if oldNs == "" {
|
||||
return nil, fmt.Errorf("a pillanatképben nincs mentési egység (backups/primary/%s)", stack)
|
||||
}
|
||||
out := make([]placement, 0, len(snapPaths))
|
||||
for _, p := range snapPaths {
|
||||
if p != oldNs && !strings.HasPrefix(p, oldNs+"/") {
|
||||
return nil, fmt.Errorf("a pillanatkép egy útvonala a névtéren kívülre mutat: %s", p)
|
||||
}
|
||||
rel := strings.TrimPrefix(p, oldNs+"/")
|
||||
for _, seg := range strings.Split(rel, "/") {
|
||||
if seg == ".." {
|
||||
return nil, fmt.Errorf("a pillanatkép egy útvonala érvénytelen (..): %s", p)
|
||||
}
|
||||
}
|
||||
isUnit := rel == "backups/primary/"+stack
|
||||
if !isUnit && (rel == "backups" || strings.HasPrefix(rel, "backups/")) {
|
||||
return nil, fmt.Errorf("nem-egység útvonal a fenntartott backups zónában: %s", p)
|
||||
}
|
||||
out = append(out, placement{
|
||||
src: filepath.Join(scratch, p), // SP-3.1: abs source reconstructed under the target
|
||||
dst: filepath.Join(liveNsRoot, rel),
|
||||
isUnit: isUnit,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// placeCopier returns the place-to-live missing-only merge (nil seam → rsyncRestoreMissing, the
|
||||
// `-a --ignore-existing` additive copy). NEVER rsyncMirror (--delete).
|
||||
func (m *Manager) placeCopier() func(src, dst string) (int, error) {
|
||||
if m.offboxPlaceCopier != nil {
|
||||
return m.offboxPlaceCopier
|
||||
}
|
||||
return rsyncRestoreMissing
|
||||
}
|
||||
|
||||
// PlaceOffsiteRestore places a COMPLETED full-scratch restore into the app's live locations via a
|
||||
// missing-only merge (§7.3), so the SQ3 immich case is restorable from offsite alone. The recovery
|
||||
// unit is placed ONLY if the live unit is ABSENT (never overwrites a local unit); every other path is
|
||||
// merged missing-only. Does NOT deploy/start anything — RecreateStackFromUnit / the restore flow owns
|
||||
// that. Single-flight. Requires a completed full scratch (deterministic path + existence check).
|
||||
func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error {
|
||||
if !m.OffboxConfigured() {
|
||||
return fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return fmt.Errorf("invalid stack name")
|
||||
}
|
||||
if err := m.acquireRunning(); err != nil {
|
||||
return fmt.Errorf("egy másik mentési/visszaállítási művelet már fut")
|
||||
}
|
||||
defer m.releaseRunning()
|
||||
|
||||
scratch, _, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, sErr := os.Stat(scratch); sErr != nil {
|
||||
return fmt.Errorf("nincs előkészített teljes visszaállítás — futtass előbb egy teljes visszaállítást")
|
||||
}
|
||||
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = id
|
||||
liveNs := m.AppNamespaceRoot(stack)
|
||||
if liveNs == "" {
|
||||
return fmt.Errorf("a(z) %s élő adatmeghajtója nem határozható meg", stack)
|
||||
}
|
||||
placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs)
|
||||
if err != nil {
|
||||
return err // whole-placement refusal (no partial writes)
|
||||
}
|
||||
copier := m.placeCopier()
|
||||
var placed int
|
||||
for _, pl := range placements {
|
||||
if _, sErr := os.Stat(pl.src); sErr != nil {
|
||||
// The full scratch is incomplete for this path (e.g. only a unit-only restore ran) — refuse
|
||||
// rather than place a partial set.
|
||||
return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src))
|
||||
}
|
||||
if pl.isUnit {
|
||||
if _, liveErr := os.Stat(pl.dst); liveErr == nil {
|
||||
m.logger.Printf("[INFO] [offbox] place %s: live recovery unit present — not overwriting", stack)
|
||||
continue // never overwrite a local unit
|
||||
}
|
||||
}
|
||||
n, cErr := copier(pl.src, pl.dst)
|
||||
if cErr != nil {
|
||||
return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr)
|
||||
}
|
||||
placed += n
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] placed %s from offsite scratch: %d file(s) merged (missing-only)", stack, placed)
|
||||
return nil
|
||||
}
|
||||
@@ -286,6 +286,15 @@ func (n *Notifier) NotifyBackupFailed(message, errMsg string) {
|
||||
n.PushEvent("backup_failed", "error", message, BackupDetails{Error: errMsg})
|
||||
}
|
||||
|
||||
// NotifyOffboxEnlargeBlocked sends a WARNING (not a failure) when an app's enlarged offsite push was
|
||||
// refused by the pre-push quota gate — its config+DB were still saved. Customer-facing (Hungarian
|
||||
// body). NOTE: the event type "offbox_enlarge_blocked" must be added to the hub's allowedEventTypes +
|
||||
// customerMessages for delivery (a hub-side task, flagged — until then the hub 400s/drops it and the
|
||||
// in-dashboard LastWarning + /backups/remote note carry the message).
|
||||
func (n *Notifier) NotifyOffboxEnlargeBlocked(message string) {
|
||||
n.PushEvent("offbox_enlarge_blocked", "warning", message, nil)
|
||||
}
|
||||
|
||||
// (NotifyBackupCompleted removed 2026-06-16 — the backup_completed event had no callers
|
||||
// since slice 8C moved whole-guest backup to the agent. The hub's backup-deadline check
|
||||
// now reads the agent host-report's PBS snapshots instead of this event. DB-dump events
|
||||
|
||||
@@ -147,6 +147,11 @@ type OffboxTarget struct {
|
||||
// LastWarning is a customer-visible notice set on an otherwise-OK run when SOME toggled apps had
|
||||
// no discoverable recovery unit (partial run). Empty on a fully-successful or failed run.
|
||||
LastWarning string `json:"last_warning,omitempty"`
|
||||
// EnlargedBlocked (3a) lists the apps whose ENLARGED (mandatory-userdata) offsite push was refused
|
||||
// by the pre-push quota gate on the last run — their unit-only push still succeeded. Replaced each
|
||||
// OK run (sorted; empty clears). Drives the per-app "config+DB only" note on /backups/remote and
|
||||
// the edge-triggered enlarge-blocked notification. Not a secret (app-name list).
|
||||
EnlargedBlocked []string `json:"enlarged_blocked,omitempty"`
|
||||
// EscrowState (fork-4) gates offsite RUNS on the repo password being escrowed under R: ""|"pending"
|
||||
// |"escrowed". Enabling offsite stages the password to the agent and sets "pending"; no offsite run
|
||||
// proceeds until an operator confirms the escrow ceremony ("escrowed") — so no un-recoverable
|
||||
|
||||
@@ -661,6 +661,14 @@ func (s *Server) backupsOffboxData(data map[string]interface{}) {
|
||||
}
|
||||
// SLICE 4 soft-quota usage bar (rendered only when a quota is set — shared model).
|
||||
data["OffboxQuotaPct"] = backup.OffboxQuotaPercent(offboxTgt)
|
||||
// 3a: per-app "config+DB only" note set — apps whose enlarged push the quota gate blocked last run.
|
||||
blocked := map[string]bool{}
|
||||
if offboxTgt != nil {
|
||||
for _, a := range offboxTgt.EnlargedBlocked {
|
||||
blocked[a] = true
|
||||
}
|
||||
}
|
||||
data["OffboxBlockedSet"] = blocked
|
||||
}
|
||||
|
||||
// offboxStaleWarningMarker is the substring the zero-toggled offbox run writes into
|
||||
@@ -757,6 +765,24 @@ func (s *Server) backupsAppsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *Server) backupsRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
data := s.backupsCommonData("backups-restore", "Biztonsági mentés — Visszaállítás", r)
|
||||
s.backupsOffboxData(data) // restore-to-verify lists the offbox-toggled apps
|
||||
// Full-restore two-step reveal (§7.2): after the size+headroom prepare step, offboxRestoreHandler
|
||||
// redirects here with the app + human size so the confirm section can show the size BEFORE starting.
|
||||
if fp := strings.TrimSpace(r.URL.Query().Get("full_prep")); fp != "" {
|
||||
data["FullPrepApp"] = fp
|
||||
data["FullPrepSize"] = r.URL.Query().Get("full_size")
|
||||
}
|
||||
// Per-app place-to-live availability (a completed full scratch exists → offer the merge action).
|
||||
ready := map[string]bool{}
|
||||
if s.backupMgr != nil {
|
||||
if apps, ok := data["OffboxApps"].([]OffboxAppRow); ok {
|
||||
for _, a := range apps {
|
||||
if a.Enabled && s.backupMgr.OffboxFullScratchReady(a.Name) {
|
||||
ready[a.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
data["OffboxScratchReady"] = ready
|
||||
s.executeTemplate(w, r, "backups_restore", data)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -88,6 +87,8 @@ func (s *Server) offboxConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
tgt.LastDuration, tgt.RepoSizeHuman, tgt.SnapshotCount = prev.LastDuration, prev.RepoSizeHuman, prev.SnapshotCount
|
||||
tgt.LastWarning = prev.LastWarning
|
||||
tgt.EscrowState = prev.EscrowState
|
||||
tgt.RepoSizeBytes = prev.RepoSizeBytes
|
||||
tgt.EnlargedBlocked = prev.EnlargedBlocked
|
||||
}
|
||||
// fork-4: enabling offsite stages the repo password to the agent for the R-escrow ceremony and marks
|
||||
// it PENDING — no offsite RUN proceeds until escrow is confirmed (atomicity). Re-editing an already
|
||||
@@ -213,8 +214,10 @@ func (s *Server) offboxRunHandler(w http.ResponseWriter, r *http.Request) {
|
||||
offboxRedirect(w, r, "A távoli mentés elindult (a futás után az állapot frissül).", false)
|
||||
}
|
||||
|
||||
// offboxRestoreHandler restores an app's off-box data to a scratch dir (non-destructive — does NOT
|
||||
// overwrite live data; the operator inspects the restored files).
|
||||
// offboxRestoreHandler restores an app's off-box data to an on-data-drive scratch dir (§7, F-A1;
|
||||
// non-destructive — does NOT overwrite live data). mode=unit (default) restores the recovery unit
|
||||
// only; mode=full is size-gated and two-step (first POST computes the size + headroom and redirects
|
||||
// with a reveal cue; the revealed confirm POSTs mode=full&confirm=1, re-checked at execution).
|
||||
func (s *Server) offboxRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if s.backupMgr == nil || !s.backupMgr.OffboxConfigured() {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "A távoli mentési cél nincs beállítva.", true)
|
||||
@@ -226,25 +229,73 @@ func (s *Server) offboxRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "Hiányzó alkalmazás.", true)
|
||||
return
|
||||
}
|
||||
// Part B: fast-path refuse a concurrent op, then run async on a BACKGROUND context. The old code
|
||||
// bounded on r.Context()+30m — a proxy read-timeout then CANCELED the SFTP restore mid-flight
|
||||
// (worse than F4: not just an error page, an aborted restore). Background ctx fixes that.
|
||||
mode := strings.TrimSpace(r.FormValue("mode"))
|
||||
if mode == "" {
|
||||
mode = "unit"
|
||||
}
|
||||
// Step 1 of the full two-step: compute size + headroom BEFORE any restic restore; on a refusal
|
||||
// flash the Hungarian reason, else redirect with the reveal params (size shown before starting).
|
||||
if mode == "full" && r.FormValue("confirm") != "1" {
|
||||
pctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
sizeHuman, err := s.backupMgr.OffboxRestorePrepareFull(pctx, app)
|
||||
if err != nil {
|
||||
offboxRedirectTo(w, r, "/backups/restore", err.Error(), true)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/backups/restore?full_prep="+url.QueryEscape(app)+"&full_size="+url.QueryEscape(sizeHuman), http.StatusFound)
|
||||
return
|
||||
}
|
||||
// Fast-path refuse a concurrent op, then run async on a BACKGROUND context (a proxy read-timeout on
|
||||
// r.Context() would CANCEL the SFTP restore mid-flight — the F4 lesson).
|
||||
if s.backupMgr.IsRunning() {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "Egy mentési/visszaállítási művelet már fut.", true)
|
||||
return
|
||||
}
|
||||
dest := filepath.Join(s.cfg.Paths.DataDir, "offbox-restore", app)
|
||||
full := mode == "full"
|
||||
s.backupMgr.BeginRestoreOp("offbox-restore", app)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.backupMgr.RestoreOffbox(ctx, app, dest); err != nil {
|
||||
s.logger.Printf("[ERROR] [web] off-box restore %s (async): %v", app, err)
|
||||
if err := s.backupMgr.RestoreOffboxScratch(ctx, app, full); err != nil {
|
||||
s.logger.Printf("[ERROR] [web] off-box restore %s (full=%v, async): %v", app, full, err)
|
||||
s.backupMgr.EndRestoreOp(false, "A visszaállítás sikertelen: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.logger.Printf("[INFO] [web] off-box restore %s completed (async) → %s", app, dest)
|
||||
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" visszaállítva ide (ellenőrzésre): "+dest)
|
||||
s.logger.Printf("[INFO] [web] off-box restore %s completed (full=%v, async)", app, full)
|
||||
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" visszaállítva ellenőrző mappába a meghajtón (a meglévő adatok változatlanok).")
|
||||
}()
|
||||
offboxRedirectTo(w, r, "/backups/restore", "A távoli visszaállítás elindult — az állapot itt frissül.", false)
|
||||
}
|
||||
|
||||
// offboxPlaceHandler places a COMPLETED full-restore scratch into the app's live locations via a
|
||||
// missing-only merge (§7.3). Never overwrites existing files. Async on a background context.
|
||||
func (s *Server) offboxPlaceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if s.backupMgr == nil || !s.backupMgr.OffboxConfigured() {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "A távoli mentési cél nincs beállítva.", true)
|
||||
return
|
||||
}
|
||||
_ = r.ParseForm()
|
||||
app := strings.TrimSpace(r.FormValue("app"))
|
||||
if app == "" {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "Hiányzó alkalmazás.", true)
|
||||
return
|
||||
}
|
||||
if s.backupMgr.IsRunning() {
|
||||
offboxRedirectTo(w, r, "/backups/restore", "Egy mentési/visszaállítási művelet már fut.", true)
|
||||
return
|
||||
}
|
||||
s.backupMgr.BeginRestoreOp("offbox-place", app)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.backupMgr.PlaceOffsiteRestore(ctx, app); err != nil {
|
||||
s.logger.Printf("[ERROR] [web] off-box place %s (async): %v", app, err)
|
||||
s.backupMgr.EndRestoreOp(false, "A helyreállítás sikertelen: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.logger.Printf("[INFO] [web] off-box place %s completed (async)", app)
|
||||
s.backupMgr.EndRestoreOp(true, "A(z) "+app+" hiányzó fájljai helyreállítva az élő adatok közé.")
|
||||
}()
|
||||
offboxRedirectTo(w, r, "/backups/restore", "A helyreállítás elindult — az állapot itt frissül.", false)
|
||||
}
|
||||
|
||||
@@ -370,6 +370,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.offboxRunHandler(w, r)
|
||||
case path == "/backup/offbox/restore" && r.Method == http.MethodPost:
|
||||
s.offboxRestoreHandler(w, r)
|
||||
case path == "/backup/offbox/place" && r.Method == http.MethodPost:
|
||||
s.offboxPlaceHandler(w, r)
|
||||
// Controller-driven escrow ceremony wizard (v0.127.0): the customer-facing R flow.
|
||||
case path == "/backup/escrow" && r.Method == http.MethodGet:
|
||||
s.escrowWizardPageHandler(w, r)
|
||||
|
||||
@@ -103,6 +103,9 @@
|
||||
<input type="hidden" name="enabled" value="{{if .Enabled}}false{{else}}true{{end}}">
|
||||
<button type="submit" class="btn btn-xs {{if .Enabled}}btn-outline{{else}}btn-primary{{end}}">{{if .Enabled}}Távoli mentés kikapcsolása{{else}}Távoli mentés bekapcsolása{{end}}</button>
|
||||
</form>
|
||||
{{if $.OffboxBlockedSet}}{{if index $.OffboxBlockedSet .Name}}
|
||||
<span class="form-hint" style="display:block;margin-top:.25rem">A teljes mentés túllépné a tárhelykeretet — csak a konfiguráció és az adatbázis kerül mentésre.</span>
|
||||
{{end}}{{end}}
|
||||
{{template "app_list_row_end"}}
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
@@ -72,8 +72,29 @@
|
||||
{{template "app_list_row" dict "Slug" .Slug "Name" .DisplayName}}
|
||||
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
|
||||
<input type="hidden" name="app" value="{{.Name}}">
|
||||
<button type="submit" class="btn btn-xs btn-outline" data-confirm="Visszaállítja a(z) {{.DisplayName}} adatait a távoli tárolóról egy ellenőrző mappába? A meglévő adatok NEM íródnak felül.">Visszaállítás (ellenőrzéshez)</button>
|
||||
<input type="hidden" name="mode" value="unit">
|
||||
<button type="submit" class="btn btn-xs btn-outline">Visszaállítás ellenőrzéshez (konfiguráció + adatbázis)</button>
|
||||
</form>
|
||||
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
|
||||
<input type="hidden" name="app" value="{{.Name}}">
|
||||
<input type="hidden" name="mode" value="full">
|
||||
<button type="submit" class="btn btn-xs btn-outline">Teljes visszaállítás előkészítése</button>
|
||||
</form>
|
||||
{{if $.FullPrepApp}}{{if eq $.FullPrepApp .Name}}
|
||||
<form method="POST" action="/backup/offbox/restore" style="display:inline">{{$.CSRFField}}
|
||||
<input type="hidden" name="app" value="{{.Name}}">
|
||||
<input type="hidden" name="mode" value="full">
|
||||
<input type="hidden" name="confirm" value="1">
|
||||
<button type="submit" class="btn btn-xs btn-primary">Teljes visszaállítás indítása (~{{$.FullPrepSize}})</button>
|
||||
</form>
|
||||
{{end}}{{end}}
|
||||
{{if $.OffboxScratchReady}}{{if index $.OffboxScratchReady .Name}}
|
||||
<form method="POST" action="/backup/offbox/place" style="display:inline">{{$.CSRFField}}
|
||||
<input type="hidden" name="app" value="{{.Name}}">
|
||||
<button type="submit" class="btn btn-xs btn-outline">Helyreállítás az élő adatok közé (csak a hiányzó fájlok)</button>
|
||||
</form>
|
||||
<span class="form-hint" style="display:block;margin-top:.25rem">A meglévő fájlokat nem írja felül.</span>
|
||||
{{end}}{{end}}
|
||||
{{template "app_list_row_end"}}
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
Reference in New Issue
Block a user