Offsite tier policy engine: mandatory userdata, raw-data quota, restore rework (Task 3a, v0.134.0)
Each toggled app's offsite push = one multi-path restic snapshot (recovery unit + TierOffsite mandatory userdata via ComputeCaptureSet); legacy/undeployed stay unit-only. Loud capture gaps (SP-3.4: restic 0.14.0 silently skips missing paths). Quota = stats --mode raw-data (SP-1; displayed size drops once). Pre-push enlargement gate blocks the userdata enlargement over-quota (unit-only push continues; EnlargedBlocked; edge-triggered notify). forget --group-by host,tags on both sites (SP-2). Restore reworked: scratch off the rootfs + headroom gate (F-A1), unit-only default via --include, size-first full, place-to-live missing-only merge (never --delete). UI: unit/full-two-step/place actions + per-app blocked note; route POST /backup/offbox/place. HUB FLAG: offbox_enlarge_blocked event needs hub allowlist for push delivery. +13 tests; all 10 §10 red-proofs verified. No tier-2/.fab/hub/agent changes.
This commit is contained in:
@@ -0,0 +1,377 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Offsite restore rework (Task 3a §7). With mandatory userdata now in snapshots, restore needs three
|
||||
// changes over the old dump-to-rootfs-scratch:
|
||||
// 1. scratch relocated off the ~8 GB guest rootfs onto a data drive, behind a headroom gate (F-A1);
|
||||
// 2. a unit-only DEFAULT restore (`--include <absolute-unit-path>`, SP-3.2) — full is a deliberate,
|
||||
// size-gated second action;
|
||||
// 3. place-to-live = a missing-only merge (never --delete) so the SQ3 immich case is restorable
|
||||
// from offsite alone.
|
||||
// ID-first everywhere (§3): `restic stats --tag` is UNPROVEN on 0.14.0, so the size lookup resolves the
|
||||
// snapshot ID via `snapshots latest --tag` and calls `stats <ID>`.
|
||||
|
||||
const (
|
||||
// offboxUnitOnlyFreeFloor — a unit-only restore needs at least this much free on the scratch drive.
|
||||
// Catalog recovery units are MB–1 GB (SQ4); 2 GiB is a safe floor without a per-snapshot size probe.
|
||||
offboxUnitOnlyFreeFloor = int64(2) << 30
|
||||
)
|
||||
|
||||
// SetOffboxFreeFn overrides the restore free-space probe (tests; the Windows go-test host has no df).
|
||||
func (m *Manager) SetOffboxFreeFn(fn func(path string) int64) { m.offboxFreeFn = fn }
|
||||
|
||||
// offboxFree returns the free-space probe (nil seam → the real diskFreeBytes).
|
||||
func (m *Manager) offboxFree() func(string) int64 {
|
||||
if m.offboxFreeFn != nil {
|
||||
return m.offboxFreeFn
|
||||
}
|
||||
return diskFreeBytes
|
||||
}
|
||||
|
||||
// diskFreeBytes returns available bytes on the filesystem holding path (0 on any error). Mirrors
|
||||
// appexport.DiskFree; kept local so the backup package needs no cross-package dependency.
|
||||
func diskFreeBytes(path string) int64 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "df", "--output=avail", "-B1", path).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
if len(lines) < 2 {
|
||||
return 0
|
||||
}
|
||||
var size int64
|
||||
fmt.Sscanf(strings.TrimSpace(lines[1]), "%d", &size)
|
||||
return size
|
||||
}
|
||||
|
||||
// offboxUnitPathOf returns the snapshot path that is the recovery unit for stack (suffix
|
||||
// backups/primary/<stack>), or "" if none is present.
|
||||
func offboxUnitPathOf(paths []string, stack string) string {
|
||||
suffix := "/backups/primary/" + stack
|
||||
for _, p := range paths {
|
||||
if strings.HasSuffix(p, suffix) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// offboxLatestSnapshot resolves the newest snapshot for stack: its short ID + captured paths, via
|
||||
// `snapshots latest --tag <stack> --json`. When the tag spans more than one group (old unit-only shape
|
||||
// + new enlarged shape), it returns the newest by time.
|
||||
func (m *Manager) offboxLatestSnapshot(ctx context.Context, stack string) (id string, paths []string, err error) {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
out, serr := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "latest", "--tag", stack, "--json")...)
|
||||
if serr != nil {
|
||||
return "", nil, fmt.Errorf("offbox snapshots %s: %w: %s", stack, serr, truncate(out))
|
||||
}
|
||||
var snaps []struct {
|
||||
ShortID string `json:"short_id"`
|
||||
ID string `json:"id"`
|
||||
Time time.Time `json:"time"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
if json.Unmarshal(out, &snaps) != nil || len(snaps) == 0 {
|
||||
return "", nil, fmt.Errorf("offbox: nincs pillanatkép a(z) %s alkalmazáshoz", stack)
|
||||
}
|
||||
best := 0
|
||||
for i := 1; i < len(snaps); i++ {
|
||||
if snaps[i].Time.After(snaps[best].Time) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
id = snaps[best].ShortID
|
||||
if id == "" {
|
||||
id = snaps[best].ID
|
||||
}
|
||||
return id, snaps[best].Paths, nil
|
||||
}
|
||||
|
||||
// offboxSnapshotSize returns the restore-size (logical bytes) of ONE snapshot via `stats <ID> --json`
|
||||
// (default mode — for a single snapshot ID this is exactly that snapshot's on-disk-when-restored size,
|
||||
// the correct headroom meaning; SP-1). ID-first: never `stats --tag` (unproven on 0.14.0).
|
||||
func (m *Manager) offboxSnapshotSize(ctx context.Context, id string) (int64, error) {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
out, err := m.runner()(sctx, env, append(append([]string{}, base...), "stats", id, "--json")...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("offbox stats %s: %w: %s", id, err, truncate(out))
|
||||
}
|
||||
var st struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
if json.Unmarshal(out, &st) != nil || st.TotalSize <= 0 {
|
||||
return 0, fmt.Errorf("offbox: a(z) %s pillanatkép mérete ismeretlen", id)
|
||||
}
|
||||
return st.TotalSize, nil
|
||||
}
|
||||
|
||||
// offboxRestoreScratchDir returns the on-DATA-DRIVE scratch dir for an app's offsite restore
|
||||
// (<nsRoot>/backups/offsite-restore/<app>) plus the namespace root (an existing dir, for the free-space
|
||||
// probe). NEVER cfg.Paths.DataDir (the rootfs — the F-A1 filler). App's HDD drive first; else the first
|
||||
// schedulable storage path; else a Hungarian refusal.
|
||||
func (m *Manager) offboxRestoreScratchDir(stack string) (scratch, nsRoot string, err error) {
|
||||
if m.stackProvider != nil {
|
||||
if hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)); hdd != "" {
|
||||
nr := m.namespaceRoot(hdd)
|
||||
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
|
||||
}
|
||||
}
|
||||
for _, sp := range m.settings.GetSchedulableStoragePaths() {
|
||||
if strings.TrimSpace(sp.Path) != "" {
|
||||
nr := m.namespaceRoot(sp.Path)
|
||||
return filepath.Join(nr, "backups", "offsite-restore", stack), nr, nil
|
||||
}
|
||||
}
|
||||
return "", "", fmt.Errorf("nincs elérhető adatmeghajtó a visszaállításhoz")
|
||||
}
|
||||
|
||||
// RestoreOffboxScratch restores an app's latest offsite snapshot to an on-data-drive scratch dir
|
||||
// (non-destructive — never overwrites live data). full=false (the default) restores the recovery UNIT
|
||||
// only (`--include <absolute-unit-path>`, SP-3.2); full=true restores the whole snapshot (unit +
|
||||
// mandatory userdata) behind a size×1.1 headroom gate. Fail-closed: an unknown snapshot size refuses a
|
||||
// full restore.
|
||||
func (m *Manager) RestoreOffboxScratch(ctx context.Context, stack string, full bool) error {
|
||||
if !m.OffboxConfigured() {
|
||||
return fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return fmt.Errorf("invalid stack name")
|
||||
}
|
||||
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unitPath := offboxUnitPathOf(paths, stack)
|
||||
if unitPath == "" {
|
||||
return fmt.Errorf("a(z) %s pillanatképében nincs mentési egység — a visszaállítás nem indítható", stack)
|
||||
}
|
||||
scratch, nsRoot, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Headroom gate (F-A1) — probed on the namespace root (an existing dir).
|
||||
free := m.offboxFree()(nsRoot)
|
||||
if full {
|
||||
size, serr := m.offboxSnapshotSize(ctx, id)
|
||||
if serr != nil {
|
||||
// SizeUnknown never renders as fits — fail closed.
|
||||
return fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
|
||||
}
|
||||
need := size + size/10 // ×1.1
|
||||
if free < need {
|
||||
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
|
||||
}
|
||||
} else if free < offboxUnitOnlyFreeFloor {
|
||||
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(offboxUnitOnlyFreeFloor), humanizeBytes(free))
|
||||
}
|
||||
// F-A1 hygiene: drop the legacy rootfs scratch (DataDir/offbox-restore/<app>) best-effort.
|
||||
legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", stack)
|
||||
if _, sErr := os.Stat(legacy); sErr == nil {
|
||||
if rmErr := os.RemoveAll(legacy); rmErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] could not remove legacy rootfs restore scratch %s: %v", legacy, rmErr)
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] removed legacy rootfs restore scratch %s", legacy)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(scratch, 0o755); err != nil {
|
||||
return fmt.Errorf("restore dir: %w", err)
|
||||
}
|
||||
t := m.settings.GetOffboxTarget()
|
||||
base, env := m.offboxBaseArgs(t)
|
||||
rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
m.unlockStale(rctx, base, env) // pre-restore hygiene
|
||||
args := []string{"restore", id, "--target", scratch}
|
||||
if !full {
|
||||
args = append(args, "--include", unitPath) // SP-3.2: absolute snapshot unit path = unit-only
|
||||
}
|
||||
out, rerr := m.resticStep(rctx, env, base, "restore:"+stack, args...)
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("offbox restore %s: %w: %s", stack, rerr, truncate(out))
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] restored %s (%s, full=%v) → %s", stack, id, full, scratch)
|
||||
return nil
|
||||
}
|
||||
|
||||
// OffboxRestorePrepareFull resolves the latest snapshot's restore-size and verifies scratch headroom
|
||||
// for a FULL restore WITHOUT starting it (the two-step size-first gate). Returns the human size on
|
||||
// success, or a Hungarian error to flash on refusal (size unknown / no headroom — fail-closed).
|
||||
func (m *Manager) OffboxRestorePrepareFull(ctx context.Context, stack string) (string, error) {
|
||||
if !m.OffboxConfigured() {
|
||||
return "", fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return "", fmt.Errorf("invalid stack name")
|
||||
}
|
||||
id, _, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
size, serr := m.offboxSnapshotSize(ctx, id)
|
||||
if serr != nil {
|
||||
return "", fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.")
|
||||
}
|
||||
_, nsRoot, derr := m.offboxRestoreScratchDir(stack)
|
||||
if derr != nil {
|
||||
return "", derr
|
||||
}
|
||||
need := size + size/10
|
||||
if free := m.offboxFree()(nsRoot); free < need {
|
||||
return "", fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
|
||||
}
|
||||
return humanizeBytes(size), nil
|
||||
}
|
||||
|
||||
// OffboxFullScratchReady reports whether a (non-empty) full-restore scratch exists for stack — the gate
|
||||
// for showing the place-to-live action. PlaceOffsiteRestore re-validates per-path completeness.
|
||||
func (m *Manager) OffboxFullScratchReady(stack string) bool {
|
||||
if !isSafeStackName(stack) {
|
||||
return false
|
||||
}
|
||||
scratch, _, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if fi, sErr := os.Stat(scratch); sErr != nil || !fi.IsDir() {
|
||||
return false
|
||||
}
|
||||
entries, _ := os.ReadDir(scratch)
|
||||
return len(entries) > 0
|
||||
}
|
||||
|
||||
// placement is one source→dest pair for place-to-live: src is the reconstructed absolute path under the
|
||||
// scratch (SP-3.1), dst is the live location under the app's current namespace root.
|
||||
type placement struct {
|
||||
src string
|
||||
dst string
|
||||
isUnit bool
|
||||
}
|
||||
|
||||
// mapOffsiteRestorePaths maps a completed full-scratch restore to live placements (pure). The anchor
|
||||
// oldNs is derived by trimming backups/primary/<stack> off the unit path (the snapshot may come from a
|
||||
// DIFFERENT drive after churn — liveNsRoot is where it goes). Refuses the WHOLE placement (no partial
|
||||
// writes) on: no unit path; a path outside oldNs (escape); a `..` segment; a non-unit path in the
|
||||
// reserved backups/ zone.
|
||||
func mapOffsiteRestorePaths(snapPaths []string, stack, scratch, liveNsRoot string) ([]placement, error) {
|
||||
unitSuffix := "/backups/primary/" + stack
|
||||
oldNs := ""
|
||||
for _, p := range snapPaths {
|
||||
if strings.HasSuffix(p, unitSuffix) {
|
||||
oldNs = strings.TrimSuffix(p, unitSuffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
if oldNs == "" {
|
||||
return nil, fmt.Errorf("a pillanatképben nincs mentési egység (backups/primary/%s)", stack)
|
||||
}
|
||||
out := make([]placement, 0, len(snapPaths))
|
||||
for _, p := range snapPaths {
|
||||
if p != oldNs && !strings.HasPrefix(p, oldNs+"/") {
|
||||
return nil, fmt.Errorf("a pillanatkép egy útvonala a névtéren kívülre mutat: %s", p)
|
||||
}
|
||||
rel := strings.TrimPrefix(p, oldNs+"/")
|
||||
for _, seg := range strings.Split(rel, "/") {
|
||||
if seg == ".." {
|
||||
return nil, fmt.Errorf("a pillanatkép egy útvonala érvénytelen (..): %s", p)
|
||||
}
|
||||
}
|
||||
isUnit := rel == "backups/primary/"+stack
|
||||
if !isUnit && (rel == "backups" || strings.HasPrefix(rel, "backups/")) {
|
||||
return nil, fmt.Errorf("nem-egység útvonal a fenntartott backups zónában: %s", p)
|
||||
}
|
||||
out = append(out, placement{
|
||||
src: filepath.Join(scratch, p), // SP-3.1: abs source reconstructed under the target
|
||||
dst: filepath.Join(liveNsRoot, rel),
|
||||
isUnit: isUnit,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// placeCopier returns the place-to-live missing-only merge (nil seam → rsyncRestoreMissing, the
|
||||
// `-a --ignore-existing` additive copy). NEVER rsyncMirror (--delete).
|
||||
func (m *Manager) placeCopier() func(src, dst string) (int, error) {
|
||||
if m.offboxPlaceCopier != nil {
|
||||
return m.offboxPlaceCopier
|
||||
}
|
||||
return rsyncRestoreMissing
|
||||
}
|
||||
|
||||
// PlaceOffsiteRestore places a COMPLETED full-scratch restore into the app's live locations via a
|
||||
// missing-only merge (§7.3), so the SQ3 immich case is restorable from offsite alone. The recovery
|
||||
// unit is placed ONLY if the live unit is ABSENT (never overwrites a local unit); every other path is
|
||||
// merged missing-only. Does NOT deploy/start anything — RecreateStackFromUnit / the restore flow owns
|
||||
// that. Single-flight. Requires a completed full scratch (deterministic path + existence check).
|
||||
func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error {
|
||||
if !m.OffboxConfigured() {
|
||||
return fmt.Errorf("off-box backup not configured")
|
||||
}
|
||||
if !isSafeStackName(stack) {
|
||||
return fmt.Errorf("invalid stack name")
|
||||
}
|
||||
if err := m.acquireRunning(); err != nil {
|
||||
return fmt.Errorf("egy másik mentési/visszaállítási művelet már fut")
|
||||
}
|
||||
defer m.releaseRunning()
|
||||
|
||||
scratch, _, err := m.offboxRestoreScratchDir(stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, sErr := os.Stat(scratch); sErr != nil {
|
||||
return fmt.Errorf("nincs előkészített teljes visszaállítás — futtass előbb egy teljes visszaállítást")
|
||||
}
|
||||
id, paths, err := m.offboxLatestSnapshot(ctx, stack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = id
|
||||
liveNs := m.AppNamespaceRoot(stack)
|
||||
if liveNs == "" {
|
||||
return fmt.Errorf("a(z) %s élő adatmeghajtója nem határozható meg", stack)
|
||||
}
|
||||
placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs)
|
||||
if err != nil {
|
||||
return err // whole-placement refusal (no partial writes)
|
||||
}
|
||||
copier := m.placeCopier()
|
||||
var placed int
|
||||
for _, pl := range placements {
|
||||
if _, sErr := os.Stat(pl.src); sErr != nil {
|
||||
// The full scratch is incomplete for this path (e.g. only a unit-only restore ran) — refuse
|
||||
// rather than place a partial set.
|
||||
return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src))
|
||||
}
|
||||
if pl.isUnit {
|
||||
if _, liveErr := os.Stat(pl.dst); liveErr == nil {
|
||||
m.logger.Printf("[INFO] [offbox] place %s: live recovery unit present — not overwriting", stack)
|
||||
continue // never overwrite a local unit
|
||||
}
|
||||
}
|
||||
n, cErr := copier(pl.src, pl.dst)
|
||||
if cErr != nil {
|
||||
return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr)
|
||||
}
|
||||
placed += n
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] placed %s from offsite scratch: %d file(s) merged (missing-only)", stack, placed)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user