263 lines
11 KiB
Go
263 lines
11 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
)
|
|
|
|
// R-893 option A (operator decision D20 = A, 2026-10-20; `09` §3 decision 192, second half; `07` §6.3):
|
|
// a failed off-site restore of one app puts the app back EXACTLY as it was and starts it.
|
|
//
|
|
// The off-site restore (ReconstituteFromOffsite) replaces the app's named volumes with the snapshot's BEFORE it
|
|
// replays the snapshot's database copy. Until now a failed replay put only the database ROWS back (the undo dump)
|
|
// on top of the snapshot's volume — measured 2026-10-19 on 9202: the newest rows over the backup's older volume,
|
|
// a table only the backup had left behind — and the app was held stopped for support.
|
|
//
|
|
// Now, for every volume the snapshot will replace:
|
|
// - BEFORE THE STOP the box checks that a copy of those volumes fits beside the restore (the same exact-size
|
|
// rule the restore's own gate uses, R-357) — not enough → refused, nothing stopped;
|
|
// - RIGHT AFTER THE STOP it tars each of them (the nightly volume-dump helper, tmp + fsync + rename) into
|
|
// <namespace>/backups/restore-undo/<app>/ — outside every capture set (captureset.go skips backups/);
|
|
// - on ANY failure after the first volume moved it stops the app, pours the copies back (the restore's own
|
|
// volume helper), removes a volume the snapshot added, writes the live definition back and starts the app;
|
|
// - a put-back that fails keeps the copy on disk and HOLDS the app (the R-893 option C hold, unchanged);
|
|
// - on success the copy is removed.
|
|
//
|
|
// Pinned by r893_putback_test.go.
|
|
|
|
// liveVolumeCopy is the pre-restore copy of the volumes an off-site restore will replace.
|
|
type liveVolumeCopy struct {
|
|
dir string // where the tars are; removed on success
|
|
copied []string // volumes that existed and were copied (each <dir>/<vol>.tar)
|
|
absent []string // volumes the snapshot adds that did not exist before (removed on a put-back)
|
|
keep bool // set when a put-back could not be done: the copy is the support path back, never removed
|
|
}
|
|
|
|
// restoreUndoDir is where the live-volume copy of one restore lives. Under backups/ on purpose: no capture set
|
|
// includes it (appbackup.captureset skips backups/), and backups/primary — the recovery unit — is never written.
|
|
func restoreUndoDir(nsRoot, stack string) string {
|
|
return filepath.Join(nsRoot, "backups", "restore-undo", stack)
|
|
}
|
|
|
|
// snapshotVolumeNames lists the volumes the snapshot's unit will replace (one <vol>.tar each), sorted.
|
|
func snapshotVolumeNames(unitVolumeDir string) []string {
|
|
entries, err := os.ReadDir(unitVolumeDir)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for _, e := range entries {
|
|
if !e.IsDir() && strings.HasSuffix(e.Name(), ".tar") {
|
|
out = append(out, strings.TrimSuffix(e.Name(), ".tar"))
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// undoVolumeProbe measures a live volume: its byte size and whether it exists at all.
|
|
func (m *Manager) undoVolumeProbe(vol string) (int64, bool, error) {
|
|
if m.undoVolumeProbeFn != nil {
|
|
return m.undoVolumeProbeFn(vol)
|
|
}
|
|
if err := dockerexec.Command("docker", "volume", "inspect", vol).Run(); err != nil {
|
|
return 0, false, nil // `docker volume inspect` fails only for a volume that does not exist
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
out, err := dockerexec.CommandContext(ctx, "docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "du", "-sb", "/v").Output()
|
|
if err != nil {
|
|
return 0, true, fmt.Errorf("measuring volume %s: %w", vol, err)
|
|
}
|
|
f := strings.Fields(string(out))
|
|
if len(f) == 0 {
|
|
return 0, true, fmt.Errorf("measuring volume %s: du printed nothing", vol)
|
|
}
|
|
n, err := strconv.ParseInt(f[0], 10, 64)
|
|
return n, true, err
|
|
}
|
|
|
|
// undoVolumeTar writes <dir>/<vol>.tar with the nightly helper's crash-safe shape (tmp, fsync, rename).
|
|
func (m *Manager) undoVolumeTar(vol, dir string) error {
|
|
if m.undoVolumeTarFn != nil {
|
|
return m.undoVolumeTarFn(vol, dir)
|
|
}
|
|
tmp := filepath.Join(dir, vol+".tar.tmp")
|
|
if out, err := m.tarVolumeOrDefault(vol, dir); err != nil {
|
|
os.Remove(tmp)
|
|
return fmt.Errorf("copying volume %s: %s: %w", vol, strings.TrimSpace(string(out)), err)
|
|
}
|
|
if err := atomicPromoteTar(tmp, filepath.Join(dir, vol+".tar")); err != nil {
|
|
os.Remove(tmp)
|
|
return fmt.Errorf("copying volume %s: %w", vol, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) undoVolumeRemove(vol string) error {
|
|
if m.undoVolumeRemoveFn != nil {
|
|
return m.undoVolumeRemoveFn(vol)
|
|
}
|
|
if out, err := dockerexec.Command("docker", "volume", "rm", "-f", vol).CombinedOutput(); err != nil {
|
|
return fmt.Errorf("removing volume %s: %s: %w", vol, strings.TrimSpace(string(out)), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// planLiveVolumeCopy measures the volumes the snapshot will replace — BEFORE anything is stopped. It returns
|
|
// the bytes the copy needs and which of the volumes exist. A volume that cannot be measured refuses the
|
|
// restore (fail closed, the R-357 direction: an unmeasurable copy must not sail into the destructive phase).
|
|
func (m *Manager) planLiveVolumeCopy(vols []string) (need int64, exists map[string]bool, err error) {
|
|
exists = map[string]bool{}
|
|
for _, v := range vols {
|
|
n, ok, pErr := m.undoVolumeProbe(v)
|
|
if pErr != nil {
|
|
return 0, nil, pErr
|
|
}
|
|
exists[v] = ok
|
|
if ok {
|
|
need += n
|
|
}
|
|
}
|
|
return need, exists, nil
|
|
}
|
|
|
|
// copyLiveVolumes tars every existing volume of the plan into a fresh restoreUndoDir. Runs with the app
|
|
// STOPPED, so each copy is consistent. On any error the partial copy is removed and nothing has moved yet.
|
|
func (m *Manager) copyLiveVolumes(stack, nsRoot string, vols []string, exists map[string]bool) (*liveVolumeCopy, error) {
|
|
dir := filepath.Join(restoreUndoDir(nsRoot, stack), time.Now().UTC().Format("20060102T150405Z"))
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
c := &liveVolumeCopy{dir: dir}
|
|
start := time.Now()
|
|
for _, v := range vols {
|
|
if !exists[v] {
|
|
c.absent = append(c.absent, v)
|
|
continue
|
|
}
|
|
if err := m.undoVolumeTar(v, dir); err != nil {
|
|
os.RemoveAll(dir)
|
|
return nil, err
|
|
}
|
|
c.copied = append(c.copied, v)
|
|
}
|
|
m.logger.Printf("[INFO] [offbox] %s: the live volumes are copied before the restore (%d copied, %d new from the snapshot) in %s — %s (R-893)",
|
|
stack, len(c.copied), len(c.absent), time.Since(start).Round(time.Millisecond), dir)
|
|
return c, nil
|
|
}
|
|
|
|
// drop removes the copy (deferred by the restore). A kept copy (a put-back that failed) stays for support.
|
|
func (c *liveVolumeCopy) drop(m *Manager, stack string) {
|
|
if c == nil || c.dir == "" {
|
|
return
|
|
}
|
|
if c.keep {
|
|
m.logger.Printf("[WARN] [offbox] %s: the restore's volume copy is KEPT for support at %s", stack, c.dir)
|
|
return
|
|
}
|
|
if err := os.RemoveAll(c.dir); err != nil {
|
|
m.logger.Printf("[WARN] [offbox] %s: could not remove the restore's volume copy %s: %v", stack, c.dir, err)
|
|
return
|
|
}
|
|
_ = os.Remove(filepath.Dir(c.dir)) // the per-app folder, when it is now empty
|
|
}
|
|
|
|
// putBack pours the copy back: every copied volume exactly as it was, every volume the snapshot added removed.
|
|
// The app must be stopped. An error means the volumes are NOT known to be as before.
|
|
func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
volReplay := m.volumeReplayFrom
|
|
if volReplay == nil {
|
|
volReplay = m.restoreDockerVolumesFrom
|
|
}
|
|
if len(c.copied) > 0 {
|
|
n, err := volReplay(stack, c.dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != len(c.copied) {
|
|
return fmt.Errorf("put back %d of %d volume(s)", n, len(c.copied))
|
|
}
|
|
}
|
|
for _, v := range c.absent {
|
|
if err := m.undoVolumeRemove(v); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// putBackAfterFailedRestore is option A's failure branch: stop, put the volumes and the definition back, start.
|
|
// When the put-back cannot be done exactly, the app is HELD (holdAppAfterMixedRestore, unchanged; holdKey is the
|
|
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
|
|
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
|
|
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
|
|
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
|
|
filesPlaced int, restart func() error, holdKey string) error {
|
|
if volumesMoved && c == nil {
|
|
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
|
|
m.holdAppAfterMixedRestore(stack, cause, live)
|
|
return util.MsgError(holdKey, stack)
|
|
}
|
|
if err := m.stackProvider.StopStack(stack); err != nil {
|
|
m.logger.Printf("[WARN] [offbox] %s: stopping the app before the put-back failed: %v", stack, err)
|
|
}
|
|
if volumesMoved {
|
|
if err := m.putBack(stack, c); err != nil {
|
|
c.keep = true
|
|
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
|
|
m.holdAppAfterMixedRestore(stack, cause, live)
|
|
return util.MsgError(holdKey, stack)
|
|
}
|
|
}
|
|
if live != nil {
|
|
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
|
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — holding the app", stack, err)
|
|
m.holdAppAfterMixedRestore(stack, cause, live)
|
|
return util.MsgError(holdKey, stack)
|
|
}
|
|
}
|
|
m.logger.Printf("[WARN] [offbox] %s: the restore failed (%v) — the live volumes and definition are put back; starting the app (R-893)", stack, cause)
|
|
if m.restorePutBackNotify != nil {
|
|
m.restorePutBackNotify(stack, cause)
|
|
}
|
|
if err := restart(); err != nil {
|
|
m.logger.Printf("[ERROR] [offbox] %s: start after the put-back failed: %v (the copy stays at %s)", stack, err, c.dirOrNone())
|
|
return util.MsgError("err.backup.restore_failed_put_back_not_started", stack)
|
|
}
|
|
return putBackMessage(stack, filesPlaced)
|
|
}
|
|
|
|
func (c *liveVolumeCopy) dirOrNone() string {
|
|
if c == nil {
|
|
return "(no volume copy)"
|
|
}
|
|
return c.dir
|
|
}
|
|
|
|
// putBackMessage is the household's sentence after a failed restore whose data is back as it was before.
|
|
// Files the restore already copied are not taken back, so with placed files the sentence says that too.
|
|
func putBackMessage(stack string, filesPlaced int) error {
|
|
if filesPlaced > 0 {
|
|
return util.MsgError("err.backup.restore_failed_put_back_files", stack)
|
|
}
|
|
return util.MsgError("err.backup.restore_failed_put_back", stack)
|
|
}
|
|
|
|
// SetRestorePutBackNotify wires the operator notification for a failed restore that was put back (main.go).
|
|
func (m *Manager) SetRestorePutBackNotify(fn func(stack string, replayErr error)) {
|
|
m.restorePutBackNotify = fn
|
|
}
|