package backup import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "strings" "time" ) // Offsite restore rework (Task 3a §7). With mandatory userdata now in snapshots, restore needs three // changes over the old dump-to-rootfs-scratch: // 1. scratch relocated off the ~8 GB guest rootfs onto a data drive, behind a headroom gate (F-A1); // 2. a unit-only DEFAULT restore (`--include `, SP-3.2) — full is a deliberate, // size-gated second action; // 3. place-to-live = a missing-only merge (never --delete) so the SQ3 immich case is restorable // from offsite alone. // ID-first everywhere (§3): `restic stats --tag` is UNPROVEN on 0.14.0, so the size lookup resolves the // snapshot ID via `snapshots latest --tag` and calls `stats `. const ( // offboxUnitOnlyFreeFloor — a unit-only restore needs at least this much free on the scratch drive. // Catalog recovery units are MB–1 GB (SQ4); 2 GiB is a safe floor without a per-snapshot size probe. offboxUnitOnlyFreeFloor = int64(2) << 30 ) // SetOffboxFreeFn overrides the restore free-space probe (tests; the Windows go-test host has no df). func (m *Manager) SetOffboxFreeFn(fn func(path string) int64) { m.offboxFreeFn = fn } // offboxFree returns the free-space probe (nil seam → the real diskFreeBytes). func (m *Manager) offboxFree() func(string) int64 { if m.offboxFreeFn != nil { return m.offboxFreeFn } return diskFreeBytes } // diskFreeBytes returns available bytes on the filesystem holding path (0 on any error). Mirrors // appexport.DiskFree; kept local so the backup package needs no cross-package dependency. func diskFreeBytes(path string) int64 { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out, err := exec.CommandContext(ctx, "df", "--output=avail", "-B1", path).Output() if err != nil { return 0 } lines := strings.Split(strings.TrimSpace(string(out)), "\n") if len(lines) < 2 { return 0 } var size int64 fmt.Sscanf(strings.TrimSpace(lines[1]), "%d", &size) return size } // offboxUnitPathOf returns the snapshot path that is the recovery unit for stack (suffix // backups/primary/), or "" if none is present. func offboxUnitPathOf(paths []string, stack string) string { suffix := "/backups/primary/" + stack for _, p := range paths { if strings.HasSuffix(p, suffix) { return p } } return "" } // offboxLatestSnapshot resolves the newest snapshot for stack: its short ID + captured paths, via // `snapshots latest --tag --json`. When the tag spans more than one group (old unit-only shape // + new enlarged shape), it returns the newest by time. func (m *Manager) offboxLatestSnapshot(ctx context.Context, stack string) (id string, paths []string, err error) { t := m.settings.GetOffboxTarget() base, env := m.offboxBaseArgs(t) sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout) defer cancel() out, serr := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "latest", "--tag", stack, "--json")...) if serr != nil { return "", nil, fmt.Errorf("offbox snapshots %s: %w: %s", stack, serr, truncate(out)) } var snaps []struct { ShortID string `json:"short_id"` ID string `json:"id"` Time time.Time `json:"time"` Paths []string `json:"paths"` } if json.Unmarshal(out, &snaps) != nil || len(snaps) == 0 { return "", nil, fmt.Errorf("offbox: nincs pillanatkép a(z) %s alkalmazáshoz", stack) } best := 0 for i := 1; i < len(snaps); i++ { if snaps[i].Time.After(snaps[best].Time) { best = i } } id = snaps[best].ShortID if id == "" { id = snaps[best].ID } return id, snaps[best].Paths, nil } // offboxSnapshotSize returns the restore-size (logical bytes) of ONE snapshot via `stats --json` // (default mode — for a single snapshot ID this is exactly that snapshot's on-disk-when-restored size, // the correct headroom meaning; SP-1). ID-first: never `stats --tag` (unproven on 0.14.0). func (m *Manager) offboxSnapshotSize(ctx context.Context, id string) (int64, error) { t := m.settings.GetOffboxTarget() base, env := m.offboxBaseArgs(t) sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout) defer cancel() out, err := m.runner()(sctx, env, append(append([]string{}, base...), "stats", id, "--json")...) if err != nil { return 0, fmt.Errorf("offbox stats %s: %w: %s", id, err, truncate(out)) } var st struct { TotalSize int64 `json:"total_size"` } if json.Unmarshal(out, &st) != nil || st.TotalSize <= 0 { return 0, fmt.Errorf("offbox: a(z) %s pillanatkép mérete ismeretlen", id) } return st.TotalSize, nil } // offboxRestoreScratchDir returns the on-DATA-DRIVE scratch dir for an app's offsite restore // (/backups/offsite-restore/) plus the namespace root (an existing dir, for the free-space // probe). NEVER cfg.Paths.DataDir (the rootfs — the F-A1 filler). App's HDD drive first; else the first // schedulable storage path; else a Hungarian refusal. func (m *Manager) offboxRestoreScratchDir(stack string) (scratch, nsRoot string, err error) { // offsiteRestoreRootFor is THE place `backups/offsite-restore` is spelled (offbox_verify_copies.go) // — the listing/delete surface must resolve byte-identical paths to the ones written here. scratchFor := func(root string) (string, string) { return filepath.Join(m.offsiteRestoreRootFor(root), stack), m.namespaceRoot(root) } isNet := func(path string) bool { return m.settings != nil && m.settings.IsNetworkStoragePath(path) } // (1) the app's own drive — preferred, but ONLY if it is not NETWORK storage (F-3afix-1). restic // restores uid/gid/setgid fully onto a LOCAL fs (SP-3.3); a squashed network scratch would feed // PlaceOffsiteRestore wrong-owner files — the F-6C-1 silently-broken-restore class, offsite-side. if m.stackProvider != nil { if hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)); hdd != "" && !isNet(hdd) { s, nr := scratchFor(hdd) return s, nr, nil } } // (2) the first NON-network schedulable path. if m.settings != nil { for _, sp := range m.settings.GetSchedulableStoragePaths() { if strings.TrimSpace(sp.Path) != "" && !sp.IsNetwork() { s, nr := scratchFor(sp.Path) return s, nr, nil } } // (3) last resort ONLY: any schedulable path, with a loud WARN — a network scratch cannot // guarantee ownership fidelity under root_squash. for _, sp := range m.settings.GetSchedulableStoragePaths() { if strings.TrimSpace(sp.Path) != "" { m.logger.Printf("[WARN] [offbox] %s: restore scratch on network storage %s — ownership fidelity not guaranteed under squash", stack, sp.Path) s, nr := scratchFor(sp.Path) return s, nr, nil } } } return "", "", fmt.Errorf("nincs elérhető adatmeghajtó a visszaállításhoz") } // RestoreOffboxScratch restores an app's latest offsite snapshot to an on-data-drive scratch dir // (non-destructive — never overwrites live data). full=false (the default) restores the recovery UNIT // only (`--include `, SP-3.2); full=true restores the whole snapshot (unit + // mandatory userdata) behind a size×1.1 headroom gate. Fail-closed: an unknown snapshot size refuses a // full restore. func (m *Manager) RestoreOffboxScratch(ctx context.Context, stack string, full bool) error { if !m.OffboxConfigured() { return fmt.Errorf("off-box backup not configured") } if !isSafeStackName(stack) { return fmt.Errorf("invalid stack name") } id, paths, err := m.offboxLatestSnapshot(ctx, stack) if err != nil { return err } unitPath := offboxUnitPathOf(paths, stack) if unitPath == "" { return fmt.Errorf("a(z) %s pillanatképében nincs mentési egység — a visszaállítás nem indítható", stack) } scratch, nsRoot, err := m.offboxRestoreScratchDir(stack) if err != nil { return err } // Headroom gate (F-A1) — probed on the namespace root (an existing dir). free := m.offboxFree()(nsRoot) if full { size, serr := m.offboxSnapshotSize(ctx, id) if serr != nil { // SizeUnknown never renders as fits — fail closed. return fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.") } need := size + size/10 // ×1.1 if free < need { return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free)) } } else if free < offboxUnitOnlyFreeFloor { return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(offboxUnitOnlyFreeFloor), humanizeBytes(free)) } // F-A1 hygiene: drop the legacy rootfs scratch (DataDir/offbox-restore/) best-effort. legacy := filepath.Join(m.cfg.Paths.DataDir, "offbox-restore", stack) if _, sErr := os.Stat(legacy); sErr == nil { if rmErr := os.RemoveAll(legacy); rmErr != nil { m.logger.Printf("[WARN] [offbox] could not remove legacy rootfs restore scratch %s: %v", legacy, rmErr) } else { m.logger.Printf("[INFO] [offbox] removed legacy rootfs restore scratch %s", legacy) } } if err := os.MkdirAll(scratch, 0o755); err != nil { return fmt.Errorf("restore dir: %w", err) } t := m.settings.GetOffboxTarget() base, env := m.offboxBaseArgs(t) rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout) defer cancel() m.unlockStale(rctx, base, env) // pre-restore hygiene args := []string{"restore", id, "--target", scratch} if !full { args = append(args, "--include", unitPath) // SP-3.2: absolute snapshot unit path = unit-only } out, rerr := m.resticStep(rctx, env, base, "restore:"+stack, args...) if rerr != nil { return fmt.Errorf("offbox restore %s: %w: %s", stack, rerr, truncate(out)) } m.logger.Printf("[INFO] [offbox] restored %s (%s, full=%v) → %s", stack, id, full, scratch) return nil } // OffboxRestorePrepareFull resolves the latest snapshot's restore-size and verifies scratch headroom // for a FULL restore WITHOUT starting it (the two-step size-first gate). Returns the human size on // success, or a Hungarian error to flash on refusal (size unknown / no headroom — fail-closed). func (m *Manager) OffboxRestorePrepareFull(ctx context.Context, stack string) (string, error) { if !m.OffboxConfigured() { return "", fmt.Errorf("off-box backup not configured") } if !isSafeStackName(stack) { return "", fmt.Errorf("invalid stack name") } id, _, err := m.offboxLatestSnapshot(ctx, stack) if err != nil { return "", err } size, serr := m.offboxSnapshotSize(ctx, id) if serr != nil { return "", fmt.Errorf("A mentés mérete nem állapítható meg — a teljes visszaállítás biztonsági okból nem indítható.") } _, nsRoot, derr := m.offboxRestoreScratchDir(stack) if derr != nil { return "", derr } need := size + size/10 if free := m.offboxFree()(nsRoot); free < need { return "", fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free)) } return humanizeBytes(size), nil } // OffboxFullScratchReady reports whether a (non-empty) full-restore scratch exists for stack — the gate // for showing the place-to-live action. PlaceOffsiteRestore re-validates per-path completeness. func (m *Manager) OffboxFullScratchReady(stack string) bool { if !isSafeStackName(stack) { return false } scratch, _, err := m.offboxRestoreScratchDir(stack) if err != nil { return false } if fi, sErr := os.Stat(scratch); sErr != nil || !fi.IsDir() { return false } entries, _ := os.ReadDir(scratch) return len(entries) > 0 } // placement is one source→dest pair for place-to-live: src is the reconstructed absolute path under the // scratch (SP-3.1), dst is the live location under the app's current namespace root. type placement struct { src string dst string isUnit bool } // mapOffsiteRestorePaths maps a completed full-scratch restore to live placements (pure). The anchor // oldNs is derived by trimming backups/primary/ off the unit path (the snapshot may come from a // DIFFERENT drive after churn — liveNsRoot is where it goes). Refuses the WHOLE placement (no partial // writes) on: no unit path; a path outside oldNs (escape); a `..` segment; a non-unit path in the // reserved backups/ zone. func mapOffsiteRestorePaths(snapPaths []string, stack, scratch, liveNsRoot string) ([]placement, error) { unitSuffix := "/backups/primary/" + stack oldNs := "" for _, p := range snapPaths { if strings.HasSuffix(p, unitSuffix) { oldNs = strings.TrimSuffix(p, unitSuffix) break } } if oldNs == "" { return nil, fmt.Errorf("a pillanatképben nincs mentési egység (backups/primary/%s)", stack) } out := make([]placement, 0, len(snapPaths)) for _, p := range snapPaths { // Every captured path must be a STRICT descendant of oldNs. Requiring the trailing "/" also // catches p == oldNs (the namespace root itself — F-3a-3), which would otherwise map to a junk // placement nesting the whole old namespace under the live root. if !strings.HasPrefix(p, oldNs+"/") { return nil, fmt.Errorf("a pillanatkép egy útvonala a névtéren kívülre mutat: %s", p) } rel := strings.TrimPrefix(p, oldNs+"/") for _, seg := range strings.Split(rel, "/") { if seg == ".." { return nil, fmt.Errorf("a pillanatkép egy útvonala érvénytelen (..): %s", p) } } isUnit := rel == "backups/primary/"+stack if !isUnit && (rel == "backups" || strings.HasPrefix(rel, "backups/")) { return nil, fmt.Errorf("nem-egység útvonal a fenntartott backups zónában: %s", p) } out = append(out, placement{ src: filepath.Join(scratch, p), // SP-3.1: abs source reconstructed under the target dst: filepath.Join(liveNsRoot, rel), isUnit: isUnit, }) } return out, nil } // placeCopier returns the place-to-live missing-only merge (nil seam → rsyncRestoreMissing, the // `-a --ignore-existing` additive copy). NEVER rsyncMirror (--delete). func (m *Manager) placeCopier() func(src, dst string) (int, error) { if m.offboxPlaceCopier != nil { return m.offboxPlaceCopier } return rsyncRestoreMissing } // PlaceOffsiteRestore places a COMPLETED full-scratch restore into the app's live locations via a // missing-only merge (§7.3), so the SQ3 immich case is restorable from offsite alone. The recovery // unit is placed ONLY if the live unit is ABSENT (never overwrites a local unit); every other path is // merged missing-only. Does NOT deploy/start anything — RecreateStackFromUnit / the restore flow owns // that. Single-flight. Requires a completed full scratch (deterministic path + existence check). func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error { if !m.OffboxConfigured() { return fmt.Errorf("off-box backup not configured") } if !isSafeStackName(stack) { return fmt.Errorf("invalid stack name") } if err := m.acquireRunning(); err != nil { return fmt.Errorf("egy másik mentési/visszaállítási művelet már fut") } defer m.releaseRunning() scratch, _, err := m.offboxRestoreScratchDir(stack) if err != nil { return err } if _, sErr := os.Stat(scratch); sErr != nil { return fmt.Errorf("nincs előkészített teljes visszaállítás — futtass előbb egy teljes visszaállítást") } id, paths, err := m.offboxLatestSnapshot(ctx, stack) if err != nil { return err } _ = id // F-3a-1a: the live target uses the RAW HDD path (mirrors offboxCaptureSet). NOT AppNamespaceRoot — // its systemDataPath fallback would merge userdata onto the SSD system namespace. Empty HDD ⇒ // undeployed ⇒ refuse: the app must be restored first, then its data placed under its live drive. hdd := "" if m.stackProvider != nil { hdd = strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)) } if hdd == "" { return fmt.Errorf("a(z) %s nincs telepítve — előbb állítsd helyre az alkalmazást, utána az adatokat", stack) } liveNs := m.namespaceRoot(hdd) // F-3a-1b: headroom gate — a missing-only merge copies at most the scratch size; refuse before any // copy if the live drive lacks that (conservative — scratch and live often share a drive). if free, need := m.offboxFree()(liveNs), m.offboxSize()(scratch); free < need { return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free)) } placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs) if err != nil { return err // whole-placement refusal (no partial writes) } // F-3a-4: stat pre-pass over EVERY placement BEFORE the first copy — an incomplete scratch (e.g. a // unit-only restore, userdata srcs absent) refuses with ZERO copies, making "no partial writes" true. for _, pl := range placements { if _, sErr := os.Stat(pl.src); sErr != nil { return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src)) } } copier := m.placeCopier() var placed int for _, pl := range placements { if pl.isUnit { if _, liveErr := os.Stat(pl.dst); liveErr == nil { m.logger.Printf("[INFO] [offbox] place %s: live recovery unit present — not overwriting", stack) continue // never overwrite a local unit } } n, cErr := copier(pl.src, pl.dst) if cErr != nil { return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr) // scratch KEPT for retry } placed += n } // F-3a-2: on FULL success, remove the scratch best-effort (OffboxFullScratchReady then turns false → // the place button disappears). A failed placement returned above, keeping the scratch for a retry. if rmErr := os.RemoveAll(scratch); rmErr != nil { m.logger.Printf("[WARN] [offbox] place %s: scratch cleanup failed (harmless): %v", stack, rmErr) } else { m.logger.Printf("[INFO] [offbox] place %s: scratch removed after successful placement", stack) } m.logger.Printf("[INFO] [offbox] placed %s from offsite scratch: %d file(s) merged (missing-only)", stack, placed) return nil }