Offsite tier policy engine: mandatory userdata, raw-data quota, restore rework (Task 3a, v0.134.0)
Each toggled app's offsite push = one multi-path restic snapshot (recovery unit + TierOffsite mandatory userdata via ComputeCaptureSet); legacy/undeployed stay unit-only. Loud capture gaps (SP-3.4: restic 0.14.0 silently skips missing paths). Quota = stats --mode raw-data (SP-1; displayed size drops once). Pre-push enlargement gate blocks the userdata enlargement over-quota (unit-only push continues; EnlargedBlocked; edge-triggered notify). forget --group-by host,tags on both sites (SP-2). Restore reworked: scratch off the rootfs + headroom gate (F-A1), unit-only default via --include, size-first full, place-to-live missing-only merge (never --delete). UI: unit/full-two-step/place actions + per-app blocked note; route POST /backup/offbox/place. HUB FLAG: offbox_enlarge_blocked event needs hub allowlist for push delivery. +13 tests; all 10 §10 red-proofs verified. No tier-2/.fab/hub/agent changes.
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -52,6 +53,23 @@ func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err
|
||||
m.offboxNotify = fn
|
||||
}
|
||||
|
||||
// SetOffboxSizer overrides the mandatory-set byte estimator (tests). SetOffboxEnlargeBlockedNotifier
|
||||
// wires the edge-triggered enlargement-blocked notification (main.go). SetOffboxPlaceCopier overrides
|
||||
// the place-to-live missing-only merge (tests).
|
||||
func (m *Manager) SetOffboxSizer(fn func(path string) int64) { m.offboxSizer = fn }
|
||||
func (m *Manager) SetOffboxEnlargeBlockedNotifier(fn func(stack string, estBytes int64, usedGB, quotaGB int)) {
|
||||
m.offboxEnlargeBlockedNotify = fn
|
||||
}
|
||||
func (m *Manager) SetOffboxPlaceCopier(fn func(src, dst string) (int, error)) { m.offboxPlaceCopier = fn }
|
||||
|
||||
// offboxSize returns the mandatory-set byte estimator (nil seam → the real du -sb dirSizeBytes).
|
||||
func (m *Manager) offboxSize() func(string) int64 {
|
||||
if m.offboxSizer != nil {
|
||||
return m.offboxSizer
|
||||
}
|
||||
return dirSizeBytes
|
||||
}
|
||||
|
||||
func (m *Manager) runner() offboxRunner {
|
||||
if m.offboxRunner != nil {
|
||||
return m.offboxRunner
|
||||
@@ -395,6 +413,14 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
apps := m.settings.GetOffboxApps()
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
// Edge-trigger for the enlarge-blocked notification: capture the PRIOR blocked set so we notify only
|
||||
// apps that NEWLY cross into the blocked state (a persistently-blocked app doesn't re-notify nightly).
|
||||
priorBlocked := map[string]bool{}
|
||||
if t != nil {
|
||||
for _, s := range t.EnlargedBlocked {
|
||||
priorBlocked[s] = true
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
m.logger.Printf("[INFO] [offbox] backup run started (%d app(s) toggled)", len(apps))
|
||||
if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" }); err != nil {
|
||||
@@ -403,6 +429,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
|
||||
var backedUp int
|
||||
var missing []string
|
||||
var runResult offboxRunResult
|
||||
var runErr error
|
||||
if usedGB, quota, over := offboxQuotaState(t); over {
|
||||
// SLICE 4 soft-quota gate (pre-run): NEW backups are refused at ≥100% of the shared-model quota —
|
||||
@@ -414,8 +441,16 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
m.offboxRecordStats(ctx, base, env) // the prune may have brought the size back down — refresh
|
||||
runErr = fmt.Errorf("A távoli mentés túllépte a tárhelykeretet (%d/%d GB) — törölj régi mentéseket vagy kérj nagyobb keretet.", usedGB, quota)
|
||||
} else {
|
||||
backedUp, missing, runErr = m.runOffboxInternal(ctx, apps, base, env)
|
||||
runResult, runErr = m.runOffboxInternal(ctx, apps, base, env, t)
|
||||
backedUp = runResult.backedUp
|
||||
missing = runResult.missing
|
||||
}
|
||||
// Sorted names of apps whose enlargement was blocked this run (replaces the persisted set; empty clears).
|
||||
var blockedNames []string
|
||||
for _, b := range runResult.blocked {
|
||||
blockedNames = append(blockedNames, b.stack)
|
||||
}
|
||||
sort.Strings(blockedNames)
|
||||
|
||||
// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
|
||||
// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
|
||||
@@ -440,6 +475,7 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
o.LastStatus = "ok"
|
||||
o.LastError = ""
|
||||
o.SnapshotCount = snapshots
|
||||
o.EnlargedBlocked = blockedNames // replace each run (sorted); empty slice clears it
|
||||
var warns []string
|
||||
// Zero-toggle honesty (take-two obs.): a configured target with NOTHING selected reports
|
||||
// its emptiness instead of a bare success — the customer thinks offsite runs, but nothing
|
||||
@@ -451,6 +487,13 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
|
||||
len(missing), strings.Join(missing, ", ")))
|
||||
}
|
||||
// 3a: capture-gap warnings (structurally-refused / on-disk-missing mandatory paths, undeployed).
|
||||
warns = append(warns, runResult.warns...)
|
||||
// 3a: the pre-push enlargement gate blocked some apps' userdata — config+DB still saved.
|
||||
if len(blockedNames) > 0 {
|
||||
warns = append(warns, fmt.Sprintf("Figyelmeztetés: a tárhelykeret miatt %d alkalmazásnál csak konfiguráció- és adatbázis-mentés készült: %s.",
|
||||
len(blockedNames), strings.Join(blockedNames, ", ")))
|
||||
}
|
||||
// SLICE 4: approaching the soft quota (≥80%, <100%) — warn on an otherwise-OK run.
|
||||
if qw := offboxQuotaWarning(o); qw != "" {
|
||||
warns = append(warns, qw)
|
||||
@@ -463,6 +506,17 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
|
||||
if m.offboxNotify != nil {
|
||||
m.offboxNotify(dur, snapshots, runErr)
|
||||
}
|
||||
// Edge-triggered enlarge-blocked notification: only apps that NEWLY crossed into the blocked state
|
||||
// (vs the prior persisted set) notify — a persistently-blocked app never re-notifies nightly. Uses
|
||||
// the pre-run last-known repo size (the same figure the gate used).
|
||||
if runErr == nil && m.offboxEnlargeBlockedNotify != nil && t != nil && t.QuotaGB > 0 {
|
||||
usedGB := int(t.RepoSizeBytes / offboxGiB)
|
||||
for _, b := range runResult.blocked {
|
||||
if !priorBlocked[b.stack] {
|
||||
m.offboxEnlargeBlockedNotify(b.stack, b.estBytes, usedGB, t.QuotaGB)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case runErr != nil:
|
||||
m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
|
||||
@@ -553,12 +607,24 @@ func offboxUnitTime(src, manifestPath string) time.Time {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
|
||||
// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
|
||||
// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
|
||||
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
|
||||
// offboxRunResult carries the outcome of a per-app offbox run: how many apps were backed up, which
|
||||
// had no discoverable unit (skipped), which had their enlargement quota-blocked (unit-only), and the
|
||||
// aggregated Hungarian customer warnings (capture gaps + undeployed).
|
||||
type offboxRunResult struct {
|
||||
backedUp int
|
||||
missing []string
|
||||
blocked []offboxBlocked
|
||||
warns []string
|
||||
}
|
||||
|
||||
// runOffboxInternal does the repo-ensure + per-app DISCOVER → capture-set → gate → multi-path backup +
|
||||
// prune. Caller holds the running flag. Each app's snapshot is ONE multi-path restic snapshot
|
||||
// (recovery unit + the app's MANDATORY offsite capture set, §6). The pre-push enlargement gate (§9,
|
||||
// decision #1) blocks only the ENLARGEMENT — the unit-only push always continues. Returns the result +
|
||||
// the first hard error (repo-ensure or a restic backup exec failure).
|
||||
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string, t *settings.OffboxTarget) (res offboxRunResult, err error) {
|
||||
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
|
||||
return 0, nil, rerr // fail fast (dead NAS surfaces here)
|
||||
return res, rerr // fail fast (dead NAS surfaces here)
|
||||
}
|
||||
// Pre-run hygiene: clear any lock restic can prove stale before we start (cheap; the --remove-all
|
||||
// crash-lock escalation lives in resticStep for the locks restic can't self-detect).
|
||||
@@ -568,11 +634,30 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
src, ok := m.discoverOffboxUnit(stack)
|
||||
if !ok {
|
||||
m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
|
||||
missing = append(missing, stack)
|
||||
res.missing = append(res.missing, stack)
|
||||
continue
|
||||
}
|
||||
// Task 3-core TierOffsite capture set: mandatory userdata paths added to the unit snapshot,
|
||||
// plus loud warnings for structurally-refused / on-disk-missing mandatory paths (SP-3.4).
|
||||
extra, capWarns := m.offboxCaptureSet(stack)
|
||||
res.warns = append(res.warns, capWarns...)
|
||||
// Pre-push enlargement gate (§9): if last-known repo raw-data bytes + the mandatory-set estimate
|
||||
// would cross the soft quota, push UNIT-ONLY (protection never regresses) and record the block.
|
||||
if len(extra) > 0 && t != nil && t.QuotaGB > 0 {
|
||||
var est int64
|
||||
for _, p := range extra {
|
||||
est += m.offboxSize()(p)
|
||||
}
|
||||
if t.RepoSizeBytes+est >= int64(t.QuotaGB)*offboxGiB {
|
||||
m.logger.Printf("[INFO] [offbox] %s: enlargement blocked by quota (est %s + repo %s ≥ %d GB) — unit-only push continues",
|
||||
stack, humanizeBytes(est), humanizeBytes(t.RepoSizeBytes), t.QuotaGB)
|
||||
res.blocked = append(res.blocked, offboxBlocked{stack: stack, estBytes: est})
|
||||
extra = nil
|
||||
}
|
||||
}
|
||||
args := append([]string{"backup", "--tag", "felhom-offbox", "--tag", stack, src}, extra...)
|
||||
bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, "backup", "--tag", "felhom-offbox", "--tag", stack, src)
|
||||
out, berr := m.resticStep(bctx, env, base, "backup:"+stack, args...)
|
||||
cancel()
|
||||
if berr != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
|
||||
@@ -581,25 +666,29 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
}
|
||||
continue
|
||||
}
|
||||
backedUp++
|
||||
m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
|
||||
res.backedUp++
|
||||
m.logger.Printf("[INFO] [offbox] backed up %s (%s, %d mandatory path(s))", stack, src, len(extra))
|
||||
}
|
||||
if firstErr != nil {
|
||||
return backedUp, missing, firstErr
|
||||
return res, firstErr
|
||||
}
|
||||
// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
|
||||
// prune takes an EXCLUSIVE lock — the exact step whose crash left the C2 stale lock — so it goes
|
||||
// through resticStep for the --remove-all self-heal too.
|
||||
// Retention: keep a sane window, prune the rest. SP-2: `--group-by host,tags` so an app's OLD
|
||||
// unit-only-shape snapshots share a group with its NEW enlarged shape (same <stack> tag) and age
|
||||
// out naturally — the default host,paths grouping would strand old-shape snapshots in their own
|
||||
// permanently-retained group. prune takes an EXCLUSIVE lock (the C2 stale-lock step) → resticStep.
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
|
||||
m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
|
||||
}
|
||||
return backedUp, missing, nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic restore-size.
|
||||
// offboxGiB is the soft-quota unit: QuotaGB counts binary gigabytes (GiB) of restic REPO SIZE. Since
|
||||
// v0.134.0 the repo size is measured with `stats --mode raw-data` (actual deduplicated+compressed
|
||||
// bytes — what the customer's Storage Box really fills), NOT the old modeless restore-size which
|
||||
// multiplied by the retained-snapshot count (SP-1). The displayed size drops one-time after deploy.
|
||||
const offboxGiB = int64(1) << 30
|
||||
|
||||
// OffboxReportStatus is the NON-SECRET offsite summary carried on the hub report (SLICE 4) — the input
|
||||
@@ -673,7 +762,9 @@ func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
// SP-2: `--group-by host,tags` (mirrors runOffboxInternal's forget) so old unit-only-shape snapshots
|
||||
// age out with the enlarged shape instead of stranding in a permanently-retained host,paths group.
|
||||
fargs := append(append([]string{}, base...), "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
|
||||
} else {
|
||||
@@ -695,9 +786,12 @@ func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int
|
||||
if json.Unmarshal(out, &snaps) != nil {
|
||||
return 0
|
||||
}
|
||||
// Repo size (best-effort, restore-size). Bytes feed the soft-quota gate (SLICE 4); a failed stats
|
||||
// call keeps the last-known value (stale-but-safe).
|
||||
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--json")...); serr == nil {
|
||||
// Repo size (best-effort, RAW-DATA mode). SP-1: `--mode raw-data` reports the actual
|
||||
// deduplicated+compressed repo bytes (what the Storage Box really fills), not the modeless
|
||||
// restore-size that multiplies by the retained-snapshot count. Bytes feed the soft-quota gate
|
||||
// (SLICE 4); a failed stats call keeps the last-known value (stale-but-safe). RAW-DATA TRAP:
|
||||
// total_file_count is 0 in this mode — read total_size only.
|
||||
if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--mode", "raw-data", "--json")...); serr == nil {
|
||||
var st struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user