package backup import ( "context" "fmt" "os" "os/exec" "path/filepath" "strings" "time" ) // Offsite reconstitution (R-43, v0.148.0) — the leg that was missing. // // Until v0.148.0 NO offsite path could restore a database. The two „visszaállítás" buttons staged // files into a scratch folder and never touched postgres; the place-to-live button merged only the // files MISSING from the live tree (`rsync --ignore-existing`) and never replayed a dump. For a // DB-indexed app — most of the catalog — that combination cannot bring content back: the bytes // return and the application still cannot see them, because its index lives in the database. // Measured live on 2026-07-19 (DIAG-immich-restore-2026-07-19): 11 photos, files intact on disk, // timeline empty, two "successful" restores that merged 0 files. // // ReconstituteFromOffsite is the honest version of that operation: it takes the CHOSEN snapshot's // coherent pair and makes the live app equal to it — files overwritten to the snapshot's version, // database replayed from the same snapshot's dump, app restarted. It is deliberately a different // function from PlaceOffsiteRestore rather than a flag on it, because the two have opposite file // semantics and conflating them is exactly how the missing-only merge came to be presented as a // restore. // // Two invariants hold throughout: // // - NOTHING IS EVER DELETED. The file copy overwrites and adds; it never carries `--delete`. A // file the customer created after the snapshot survives the restore as an extra. That is the // house boundary — a restore that silently removed newer work would be a data-loss event // wearing a recovery button's label. // - THE UNDO EXISTS BEFORE THE ACT. A safety dump of the live database is written, and verified // present on disk, BEFORE anything is stopped, overwritten or replayed. If that dump cannot be // taken, the whole operation refuses with zero changes — a replay whose previous state was not // captured is not a restore, it is an overwrite with no way back. // offsitePreDump runs the coherence pre-phase's dump leg (nil seam → runDBDumpsInternal, which also // refreshes the recovery units so the manifests enumerate the dumps just written). Extracted as a // seam because the ORDER — dumps strictly before the restic capture — is the entire mechanism of // R-44, and an ordering guarantee that no test can observe is one refactor away from silently // reverting to the behaviour that produced DIAG-immich-restore-2026-07-19. func (m *Manager) offsitePreDump(ctx context.Context) error { if m.offsitePreDumpFn != nil { return m.offsitePreDumpFn(ctx) } return m.runDBDumpsInternal(ctx) } // SetOffsitePreDumpFn overrides the offsite dump pre-phase (tests; no Docker needed). func (m *Manager) SetOffsitePreDumpFn(fn func(ctx context.Context) error) { m.offsitePreDumpFn = fn } // preRestoreDumpPrefix marks the safety dumps taken immediately before a reconstitution. They live // in the app's own unit db-dumps dir so `ListDumpFiles` surfaces them beside the regular dumps — // they ARE the undo, and an undo the customer cannot see is not much of one. The regular replay // loop matches `-.sql` exactly, so a prefixed file is never mistaken for a source. const preRestoreDumpPrefix = "pre-restore-" // OffsiteReconstituteResult reports what a reconstitution actually did, so the flash can state an // OUTCOME instead of a mechanism. Every field here exists because the v0.147 flash could not say it. type OffsiteReconstituteResult struct { SnapshotID string FilesPlaced int DBsReplayed int SafetyDump string // path of the pre-restore dump (the undo), "" when the app has no DB DumpsAt time.Time // when the snapshot's DB half was taken (zero = unknown/legacy unit) OffsiteRunID string // "" for a pre-v0.148 snapshot — an unverified pair Skewed bool // the snapshot carries no coherence stamp: files and DB may differ in age LooksEmpty bool // R-44 sniff on the dump about to be replayed } // fullPlaceCopier returns the FULL-restore file copier (nil seam → rsyncRestoreOverwrite). // Deliberately NOT placeCopier(): that one is `--ignore-existing`, whose whole purpose is to leave // live files alone, which is precisely what a full restore must not do. func (m *Manager) fullPlaceCopier() func(src, dst string) (int, error) { if m.offboxFullPlaceCopier != nil { return m.offboxFullPlaceCopier } return rsyncRestoreOverwrite } // rsyncRestoreOverwrite copies src over dst: `rsync -a --itemize-changes`, with NO // `--ignore-existing` (a changed file becomes the snapshot's version) and NO `--delete` (an extra // file at dst survives). Returns the number of regular files transferred. func rsyncRestoreOverwrite(src, dst string) (int, error) { if err := os.MkdirAll(dst, 0755); err != nil { return 0, fmt.Errorf("mkdir %s: %w", dst, err) } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, "rsync", "-a", "--itemize-changes", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/") out, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) } return countRestoredFiles(string(out)), nil } // writeSafetyDump dumps every live database of stack into the app's unit db-dumps dir under the // `pre-restore-` prefix, and returns the first dump's path. Returns ("", nil) when the app has no // database at all — a no-DB app has nothing to undo and must flow exactly as it did before // v0.148.0 (no dump, no replay, no behaviour change). // // A discovered database that CANNOT be dumped is a hard error: it means the undo would not exist. func (m *Manager) writeSafetyDump(ctx context.Context, stackName, nsRoot string) (string, error) { discover := m.discoverDBs if discover == nil { discover = func(ctx context.Context) ([]DiscoveredDB, error) { return DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()) } } dbs, err := discover(ctx) if err != nil { return "", fmt.Errorf("a biztonsági mentés előtt nem sikerült felderíteni az adatbázisokat: %w", err) } var mine []DiscoveredDB for _, db := range dbs { if db.StackName == stackName { mine = append(mine, db) } } if len(mine) == 0 { return "", nil // no DB → nothing to undo → scenario E flows unchanged } dumpDir := AppDBDumpPath(nsRoot, stackName) if err := os.MkdirAll(dumpDir, 0755); err != nil { return "", fmt.Errorf("a biztonsági mentés könyvtára nem hozható létre: %w", err) } stamp := time.Now().UTC().Format("20060102T150405Z") first := "" for _, db := range mine { res := m.dumpForSafety(ctx, db, dumpDir) if res.Error != nil { return "", fmt.Errorf("a jelenlegi adatbázis biztonsági mentése sikertelen (%s): %w — a visszaállítás nem indult el", db.ContainerName, res.Error) } // DumpOne writes `-.sql`; rename it under the safety prefix so it can never be // picked up as a replay SOURCE and can never overwrite the app's real dump. safe := filepath.Join(dumpDir, fmt.Sprintf("%s%s-%s-%s.sql", preRestoreDumpPrefix, stamp, stackName, db.DBType)) if res.FilePath != safe { if err := os.Rename(res.FilePath, safe); err != nil { return "", fmt.Errorf("a biztonsági mentés véglegesítése sikertelen: %w", err) } } if first == "" { first = safe } m.logger.Printf("[INFO] [offbox] %s: pre-restore safety dump written → %s (%s)", stackName, filepath.Base(safe), humanizeBytes(res.Size)) } return first, nil } // dumpForSafety is the DumpOne seam for the safety dump (tests inject; nil → the real DumpOne). func (m *Manager) dumpForSafety(ctx context.Context, db DiscoveredDB, dumpDir string) DumpResult { if m.safetyDumpFn != nil { return m.safetyDumpFn(ctx, db, dumpDir) } return DumpOne(ctx, db, dumpDir, m.logger, m.isDebug()) } // ReconstituteFromOffsite makes the live app equal to a restored full-scratch snapshot: files // overwritten to the snapshot's version (extras survive, nothing deleted), then the snapshot's own // DB dump replayed, with a safety dump of the current database taken first. Requires a completed // FULL scratch restore (RestoreOffboxScratch with full=true). Single-flight. func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string) (OffsiteReconstituteResult, error) { var res OffsiteReconstituteResult if !m.OffboxConfigured() { return res, fmt.Errorf("off-box backup not configured") } if !isSafeStackName(stack) { return res, fmt.Errorf("invalid stack name") } if m.stackProvider == nil { return res, fmt.Errorf("stack provider not configured") } if err := m.acquireRunning(); err != nil { return res, fmt.Errorf("egy másik mentési/visszaállítási művelet már fut") } defer m.releaseRunning() scratch, _, err := m.offboxRestoreScratchDir(stack) if err != nil { return res, err } if _, sErr := os.Stat(scratch); sErr != nil { return res, fmt.Errorf("nincs előkészített teljes visszaállítás — futtass előbb egy teljes visszaállítást") } id, paths, err := m.offboxLatestSnapshot(ctx, stack) if err != nil { return res, err } res.SnapshotID = id hdd := strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack)) if hdd == "" { return res, fmt.Errorf("a(z) %s nincs telepítve — előbb állítsd helyre az alkalmazást, utána az adatokat", stack) } liveNs := m.namespaceRoot(hdd) placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs) if err != nil { return res, err // whole-placement refusal (no partial writes) } // Stat pre-pass over EVERY placement before the first copy — an incomplete scratch (e.g. only a // unit-only restore was run) refuses with ZERO copies. for _, pl := range placements { if _, sErr := os.Stat(pl.src); sErr != nil { return res, fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src)) } } // The snapshot's coherence stamp, read from the RESTORED unit manifest (not the live one). scratchUnit := "" for _, pl := range placements { if pl.isUnit { scratchUnit = pl.src break } } if scratchUnit == "" { return res, fmt.Errorf("a pillanatképben nincs mentési egység — a visszaállítás nem indítható") } scratchDumpDir := filepath.Join(scratchUnit, "db-dumps") if man := readManifest(filepath.Join(scratchUnit, "manifest.json")); man != nil { res.OffsiteRunID = man.OffsiteRunID if man.DumpsAt != "" { if t, pErr := time.Parse(time.RFC3339, man.DumpsAt); pErr == nil { res.DumpsAt = t } } } // A pre-v0.148 snapshot carries no stamp: its dump was whatever the 02:30 local run left behind, // so the pair's two halves may be hours or days apart. Surfaced, never blocked — the confirm // dialog says so and the safety dump makes it reversible. res.Skewed = res.OffsiteRunID == "" res.LooksEmpty = m.sniffScratchDump(scratchDumpDir, stack) // --- WHICH SERVICE HOLDS THE DATABASE (R-47) ------------------------------------------------ // Read from the LIVE compose, not the scratch one: reconstitution never overwrites the stack dir, // so the live file is what `docker compose up` will actually act on. Resolved BEFORE the first // mutation so the refusal below costs nothing. var dbServices []string if composePath, cOK := m.stackProvider.GetStackComposePath(stack); cOK && composePath != "" { svcs, dsErr := DBServiceNames(composePath) if dsErr != nil { // "cannot tell" is not "no database" — leave dbServices empty and let the gate refuse. m.logger.Printf("[WARN] [offbox] %s: could not read the live compose services: %v", stack, dsErr) } dbServices = svcs } // --- THE UNDO, BEFORE THE ACT --------------------------------------------------------------- // Taken while the stack is still UP (a stopped database cannot be dumped) and before a single // byte is overwritten, so a failure here aborts with the live app completely untouched. safety, err := m.writeSafetyDump(ctx, stack, liveNs) if err != nil { return res, err } res.SafetyDump = safety hasDB := safety != "" if hasDB { if _, sErr := os.Stat(safety); sErr != nil { // Fail-closed: never replay when the undo is not verifiably on disk. return res, fmt.Errorf("a biztonsági mentés nem található a lemezen — a visszaállítás biztonsági okból nem indult el") } // Fail-closed (R-47): the app HAS a database but no compose service can be identified to // start alone for the replay. The only alternative would be to start everything and replay // into the race that produced H4 — refusing with the live app untouched is the better outcome. if len(dbServices) == 0 { return res, fmt.Errorf("Az adatbázis-szolgáltatás nem azonosítható a(z) %s alkalmazásban — a visszaállítás biztonsági okból nem indult el.", stack) } } // --- FILES ---------------------------------------------------------------------------------- // R-166: mark the stop→restore→start window BEFORE stopping. A controller killed anywhere inside // it used to leave the app down with nothing on disk recording that it was owed a restart — and a // full offsite restore is a LONG window, so this is the shape most likely to be interrupted. if err := m.appStop.Begin("offbox-reconstitute:"+stack, ReasonOffboxReconstitute, []string{stack}); err != nil { return res, fmt.Errorf("a(z) %s leállítása előtti jelölő nem menthető: %w", stack, err) } // restartStack starts the app and clears the marker ONLY when the start actually succeeded — a // failed start leaves the marker so the next startup retries. Every bring-up below goes through // it; a bare StartStack here would clear nothing and strand the marker on the success path. restartStack := func() error { err := m.stackProvider.StartStack(stack) if err == nil { m.appStop.End() } return err } if err := m.stackProvider.StopStack(stack); err != nil { m.logger.Printf("[WARN] [offbox] could not stop %s before reconstitution: %v (continuing)", stack, err) } copier := m.fullPlaceCopier() for _, pl := range placements { if pl.isUnit { // The live recovery unit is still never overwritten — it is the LOCAL restore path's // source and clobbering it would trade one recovery route for another. The snapshot's // dump is replayed from the scratch unit instead, so nothing is lost by skipping it. continue } n, cErr := copier(pl.src, pl.dst) if cErr != nil { // Best-effort bring-up: leaving the app stopped after a partial copy would turn a failed // restore into an outage. if sErr := restartStack(); sErr != nil { m.logger.Printf("[WARN] [offbox] %s: restart after failed placement also failed: %v", stack, sErr) } return res, fmt.Errorf("a(z) %s fájljainak visszaállítása sikertelen: %w", stack, cErr) } res.FilesPlaced += n } // --- DATABASE ------------------------------------------------------------------------------- // The DB container must be UP for the replay (ImportDump talks to it with its own discovered // credentials), but NOTHING ELSE may be — R-47. Until v0.153.0 this was a full StartStack, which // gave the application a window to rebuild the very schema objects the dump was about to create: // measured at 2 s on 2026-07-19, and the replay aborted `relation "clip_index" already exists` // under ON_ERROR_STOP=1 (H4). Starting only the database service closes that window entirely. if hasDB { if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil { // Best-effort bring-up: a failed restore must not also be an outage. if sErr := restartStack(); sErr != nil { m.logger.Printf("[WARN] [offbox] %s: full start after failed DB-only start also failed: %v", stack, sErr) } return res, fmt.Errorf("a(z) %s adatbázis-szolgáltatásának indítása sikertelen: %w", stack, err) } n, iErr := m.reimportDBDumpsFrom(ctx, stack, scratchDumpDir) res.DBsReplayed = n if iErr != nil { if sErr := restartStack(); sErr != nil { m.logger.Printf("[WARN] [offbox] %s: full start after failed replay also failed: %v", stack, sErr) } return res, fmt.Errorf("az adatbázis visszaállítása sikertelen: %w — a korábbi állapot mentése megvan: %s", iErr, filepath.Base(safety)) } } if err := restartStack(); err != nil { return res, fmt.Errorf("a(z) %s újraindítása sikertelen a fájlok visszaállítása után: %w", stack, err) } if err := m.waitForHealthy(stack, 90*time.Second); err != nil { m.logger.Printf("[WARN] [offbox] %s reconstituted but health check failed: %v", stack, err) } m.logger.Printf("[INFO] [offbox] reconstituted %s from snapshot %s: %d file(s) placed, %d DB dump(s) replayed, safety dump=%s, skewed=%v", stack, id, res.FilesPlaced, res.DBsReplayed, filepath.Base(safety), res.Skewed) return res, nil } // OffsitePairInfo describes the {DB, files} pair sitting in a prepared full-restore scratch, so the // confirm dialog can tell the customer what they are about to restore BEFORE they commit to it. // Everything here is honesty-surface: none of it blocks the operation. type OffsitePairInfo struct { Ready bool DumpsAt time.Time // when the DB half was taken (zero = legacy unit, age unknown) Skewed bool // no coherence stamp → the two halves may be from different times LooksEmpty bool // R-44 sniff: the dump has an accounts table with no rows HasDump bool } // OffsiteScratchPair reads the prepared scratch's unit manifest and reports what the pair looks // like. Cheap and read-only — safe to call from a page render. func (m *Manager) OffsiteScratchPair(stack string) OffsitePairInfo { var info OffsitePairInfo if !isSafeStackName(stack) { return info } scratch, _, err := m.offboxRestoreScratchDir(stack) if err != nil { return info } // The unit sits at //backups/primary/; the old namespace is unknown here, // so find it rather than reconstructing it. unit := findScratchUnitDir(scratch, stack) if unit == "" { return info } info.Ready = true dumpDir := filepath.Join(unit, "db-dumps") if entries, rErr := os.ReadDir(dumpDir); rErr == nil { for _, e := range entries { if !e.IsDir() && filepath.Ext(e.Name()) == ".sql" && !strings.HasPrefix(e.Name(), preRestoreDumpPrefix) { info.HasDump = true break } } } if man := readManifest(filepath.Join(unit, "manifest.json")); man != nil { if man.DumpsAt != "" { if t, pErr := time.Parse(time.RFC3339, man.DumpsAt); pErr == nil { info.DumpsAt = t } } info.Skewed = man.OffsiteRunID == "" } else { info.Skewed = true } if info.HasDump { info.LooksEmpty = m.sniffScratchDump(dumpDir, stack) } return info } // findScratchUnitDir locates `backups/primary/` anywhere under a restored scratch. restic // rebuilds absolute source paths under the target, and the snapshot may have come from a drive that // no longer exists on this box, so the prefix cannot be assumed. func findScratchUnitDir(scratch, stack string) string { found := "" suffix := filepath.Join("backups", "primary", stack) _ = filepath.Walk(scratch, func(path string, fi os.FileInfo, err error) error { if err != nil || found != "" { return nil //nolint:nilerr // a walk error on one branch must not abort the search } if fi.IsDir() && strings.HasSuffix(path, suffix) { found = path } return nil }) return found } // sniffScratchDump runs the R-44 content sniff over the dump about to be replayed. Best-effort and // warn-level: any failure to read simply reports "no warning", because a sniff that blocks a // restore is worse than the skew it describes. func (m *Manager) sniffScratchDump(dumpDir, stack string) bool { entries, err := os.ReadDir(dumpDir) if err != nil { return false } for _, e := range entries { name := e.Name() if e.IsDir() || filepath.Ext(name) != ".sql" || strings.HasPrefix(name, preRestoreDumpPrefix) { continue } dbType := DBTypePostgres if strings.Contains(name, string(DBTypeMariaDB)) { dbType = DBTypeMariaDB } if v := ValidateDump(filepath.Join(dumpDir, name), dbType); v.LooksEmpty { m.logger.Printf("[WARN] [offbox] %s: the snapshot dump %s has no account rows — it may predate the customer's data", stack, name) return true } } return false }