package backup import ( "context" "fmt" "os" "path/filepath" "sort" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" "gitea.dooplex.hu/admin/felhom-controller/internal/util" ) // R-893 option A (operator decision D20 = A, 2026-10-20; `09` §3 decision 192, second half; `07` §6.3): // a failed off-site restore of one app puts the app back EXACTLY as it was and starts it. // // The off-site restore (ReconstituteFromOffsite) replaces the app's named volumes with the snapshot's BEFORE it // replays the snapshot's database copy. Until now a failed replay put only the database ROWS back (the undo dump) // on top of the snapshot's volume — measured 2026-10-19 on 9202: the newest rows over the backup's older volume, // a table only the backup had left behind — and the app was held stopped for support. // // Now, for every volume the snapshot will replace: // - BEFORE THE STOP the box checks that a copy of those volumes fits beside the restore (the same exact-size // rule the restore's own gate uses, R-357) — not enough → refused, nothing stopped; // - RIGHT AFTER THE STOP it tars each of them (the nightly volume-dump helper, tmp + fsync + rename) into // /backups/restore-undo// — outside every capture set (captureset.go skips backups/); // - on ANY failure after the first volume moved it stops the app, pours the copies back (the restore's own // volume helper), removes a volume the snapshot added, writes the live definition back and starts the app; // - a put-back that fails keeps the copy on disk and HOLDS the app (the R-893 option C hold, unchanged); // - on success the copy is removed. // // Pinned by r893_putback_test.go. // liveVolumeCopy is the pre-restore copy of the volumes an off-site restore will replace. type liveVolumeCopy struct { dir string // where the tars are; removed on success copied []string // volumes that existed and were copied (each /.tar) absent []string // volumes the snapshot adds that did not exist before (removed on a put-back) keep bool // set when a put-back could not be done: the copy is the support path back, never removed } // restoreUndoDir is where the live-volume copy of one restore lives. Under backups/ on purpose: no capture set // includes it (appbackup.captureset skips backups/), and backups/primary — the recovery unit — is never written. func restoreUndoDir(nsRoot, stack string) string { return filepath.Join(nsRoot, "backups", "restore-undo", stack) } // snapshotVolumeNames lists the volumes the snapshot's unit will replace (one .tar each), sorted. func snapshotVolumeNames(unitVolumeDir string) []string { entries, err := os.ReadDir(unitVolumeDir) if err != nil { return nil } var out []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(e.Name(), ".tar") { out = append(out, strings.TrimSuffix(e.Name(), ".tar")) } } sort.Strings(out) return out } // undoVolumeProbe measures a live volume: its byte size and whether it exists at all. func (m *Manager) undoVolumeProbe(vol string) (int64, bool, error) { if m.undoVolumeProbeFn != nil { return m.undoVolumeProbeFn(vol) } if err := dockerexec.Command("docker", "volume", "inspect", vol).Run(); err != nil { return 0, false, nil // `docker volume inspect` fails only for a volume that does not exist } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() out, err := dockerexec.CommandContext(ctx, "docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "du", "-sb", "/v").Output() if err != nil { return 0, true, fmt.Errorf("measuring volume %s: %w", vol, err) } f := strings.Fields(string(out)) if len(f) == 0 { return 0, true, fmt.Errorf("measuring volume %s: du printed nothing", vol) } n, err := strconv.ParseInt(f[0], 10, 64) return n, true, err } // undoVolumeTar writes /.tar with the nightly helper's crash-safe shape (tmp, fsync, rename). func (m *Manager) undoVolumeTar(vol, dir string) error { if m.undoVolumeTarFn != nil { return m.undoVolumeTarFn(vol, dir) } tmp := filepath.Join(dir, vol+".tar.tmp") if out, err := m.tarVolumeOrDefault(vol, dir); err != nil { os.Remove(tmp) return fmt.Errorf("copying volume %s: %s: %w", vol, strings.TrimSpace(string(out)), err) } if err := atomicPromoteTar(tmp, filepath.Join(dir, vol+".tar")); err != nil { os.Remove(tmp) return fmt.Errorf("copying volume %s: %w", vol, err) } return nil } func (m *Manager) undoVolumeRemove(vol string) error { if m.undoVolumeRemoveFn != nil { return m.undoVolumeRemoveFn(vol) } if out, err := dockerexec.Command("docker", "volume", "rm", "-f", vol).CombinedOutput(); err != nil { return fmt.Errorf("removing volume %s: %s: %w", vol, strings.TrimSpace(string(out)), err) } return nil } // planLiveVolumeCopy measures the volumes the snapshot will replace — BEFORE anything is stopped. It returns // the bytes the copy needs and which of the volumes exist. A volume that cannot be measured refuses the // restore (fail closed, the R-357 direction: an unmeasurable copy must not sail into the destructive phase). func (m *Manager) planLiveVolumeCopy(vols []string) (need int64, exists map[string]bool, err error) { exists = map[string]bool{} for _, v := range vols { n, ok, pErr := m.undoVolumeProbe(v) if pErr != nil { return 0, nil, pErr } exists[v] = ok if ok { need += n } } return need, exists, nil } // copyLiveVolumes tars every existing volume of the plan into a fresh restoreUndoDir. Runs with the app // STOPPED, so each copy is consistent. On any error the partial copy is removed and nothing has moved yet. func (m *Manager) copyLiveVolumes(stack, nsRoot string, vols []string, exists map[string]bool) (*liveVolumeCopy, error) { dir := filepath.Join(restoreUndoDir(nsRoot, stack), time.Now().UTC().Format("20060102T150405Z")) if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } c := &liveVolumeCopy{dir: dir} start := time.Now() for _, v := range vols { if !exists[v] { c.absent = append(c.absent, v) continue } if err := m.undoVolumeTar(v, dir); err != nil { os.RemoveAll(dir) return nil, err } c.copied = append(c.copied, v) } m.logger.Printf("[INFO] [offbox] %s: the live volumes are copied before the restore (%d copied, %d new from the snapshot) in %s — %s (R-893)", stack, len(c.copied), len(c.absent), time.Since(start).Round(time.Millisecond), dir) return c, nil } // drop removes the copy (deferred by the restore). A kept copy (a put-back that failed) stays for support. func (c *liveVolumeCopy) drop(m *Manager, stack string) { if c == nil || c.dir == "" { return } if c.keep { m.logger.Printf("[WARN] [offbox] %s: the restore's volume copy is KEPT for support at %s", stack, c.dir) return } if err := os.RemoveAll(c.dir); err != nil { m.logger.Printf("[WARN] [offbox] %s: could not remove the restore's volume copy %s: %v", stack, c.dir, err) return } _ = os.Remove(filepath.Dir(c.dir)) // the per-app folder, when it is now empty } // putBack pours the copy back: every copied volume exactly as it was, every volume the snapshot added removed. // The app must be stopped. An error means the volumes are NOT known to be as before. func (m *Manager) putBack(stack string, c *liveVolumeCopy) error { if c == nil { return nil } volReplay := m.volumeReplayFrom if volReplay == nil { volReplay = m.restoreDockerVolumesFrom } if len(c.copied) > 0 { n, err := volReplay(stack, c.dir) if err != nil { return err } if n != len(c.copied) { return fmt.Errorf("put back %d of %d volume(s)", n, len(c.copied)) } } for _, v := range c.absent { if err := m.undoVolumeRemove(v); err != nil { return err } } return nil } // putBackAfterFailedRestore is option A's failure branch: stop, put the volumes and the definition back, start. // When the put-back cannot be done exactly, the app is HELD (holdAppAfterMixedRestore, unchanged; holdKey is the // failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can // put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for // option C, a file mix does not hold), so the sentence then says so instead of „nothing changed". func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool, filesPlaced int, restart func() error, holdKey string) error { if volumesMoved && c == nil { m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack) m.holdAppAfterMixedRestore(stack, cause, live) return util.MsgError(holdKey, stack) } if err := m.stackProvider.StopStack(stack); err != nil { m.logger.Printf("[WARN] [offbox] %s: stopping the app before the put-back failed: %v", stack, err) } if volumesMoved { if err := m.putBack(stack, c); err != nil { c.keep = true m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir) m.holdAppAfterMixedRestore(stack, cause, live) return util.MsgError(holdKey, stack) } } if live != nil { if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil { m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — holding the app", stack, err) m.holdAppAfterMixedRestore(stack, cause, live) return util.MsgError(holdKey, stack) } } m.logger.Printf("[WARN] [offbox] %s: the restore failed (%v) — the live volumes and definition are put back; starting the app (R-893)", stack, cause) if m.restorePutBackNotify != nil { m.restorePutBackNotify(stack, cause) } if err := restart(); err != nil { m.logger.Printf("[ERROR] [offbox] %s: start after the put-back failed: %v (the copy stays at %s)", stack, err, c.dirOrNone()) return util.MsgError("err.backup.restore_failed_put_back_not_started", stack) } return putBackMessage(stack, filesPlaced) } func (c *liveVolumeCopy) dirOrNone() string { if c == nil { return "(no volume copy)" } return c.dir } // putBackMessage is the household's sentence after a failed restore whose data is back as it was before. // Files the restore already copied are not taken back, so with placed files the sentence says that too. func putBackMessage(stack string, filesPlaced int) error { if filesPlaced > 0 { return util.MsgError("err.backup.restore_failed_put_back_files", stack) } return util.MsgError("err.backup.restore_failed_put_back", stack) } // SetRestorePutBackNotify wires the operator notification for a failed restore that was put back (main.go). func (m *Manager) SetRestorePutBackNotify(fn func(stack string, replayErr error)) { m.restorePutBackNotify = fn }