285 lines
11 KiB
Go
285 lines
11 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// R-893 option A (operator decision D20 = A, 2026-10-20): a failed off-site restore puts the app's live volumes
|
|
// back EXACTLY and starts the app. offbox_restore_undo.go. These tests pin the CONSEQUENCE — what is poured back,
|
|
// what is removed, what runs, what stays on disk — not only that a function was called.
|
|
|
|
const r893LiveVol, r893NewVol = "immich_pgdata", "immich_newvol"
|
|
|
|
// putBackRig records the volume legs: what the snapshot replay saw, and what the put-back poured, BYTE FOR BYTE.
|
|
type putBackRig struct {
|
|
scratchVolDir string
|
|
undoRoot string
|
|
snapReplayErr error // the snapshot's own volume replay
|
|
putBackErr error // the put-back from the copy
|
|
snapReplays int
|
|
poured map[string]string // volume → the bytes poured back from the copy
|
|
removed []string
|
|
tarred []string
|
|
tarErr error
|
|
probeErr error
|
|
}
|
|
|
|
func r893PutBackFixture(t *testing.T) (*Manager, *r893Provider, *putBackRig, *int, *int, *int) {
|
|
t.Helper()
|
|
m, vp, rolled, notified, putBack := r893FixtureA(t, false, 0)
|
|
unit := vtSnapshotUnit(t, m, "16", true)
|
|
rig := &putBackRig{scratchVolDir: filepath.Join(unit, "volume-dumps"), poured: map[string]string{}}
|
|
// The snapshot holds two volumes: one the box has today, one it does not (the snapshot adds it).
|
|
mustWrite(t, filepath.Join(rig.scratchVolDir, r893LiveVol+".tar"), "SNAPSHOT bytes")
|
|
mustWrite(t, filepath.Join(rig.scratchVolDir, r893NewVol+".tar"), "SNAPSHOT new")
|
|
_, liveNs, err := m.offboxRestoreScratchDir("immich")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rig.undoRoot = restoreUndoDir(liveNs, "immich")
|
|
|
|
m.undoVolumeProbeFn = func(vol string) (int64, bool, error) {
|
|
if rig.probeErr != nil {
|
|
return 0, false, rig.probeErr
|
|
}
|
|
if vol == r893LiveVol {
|
|
return 1000, true, nil
|
|
}
|
|
return 0, false, nil
|
|
}
|
|
m.undoVolumeTarFn = func(vol, dir string) error {
|
|
if rig.tarErr != nil {
|
|
return rig.tarErr
|
|
}
|
|
rig.tarred = append(rig.tarred, vol)
|
|
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("LIVE bytes of "+vol), 0o600)
|
|
}
|
|
m.undoVolumeRemoveFn = func(vol string) error { rig.removed = append(rig.removed, vol); return nil }
|
|
m.volumeReplayFrom = func(_ string, dir string) (int, error) {
|
|
entries, _ := os.ReadDir(dir)
|
|
n := 0
|
|
if dir == rig.scratchVolDir {
|
|
rig.snapReplays++
|
|
for _, e := range entries {
|
|
if strings.HasSuffix(e.Name(), ".tar") {
|
|
n++
|
|
}
|
|
}
|
|
if rig.snapReplayErr != nil {
|
|
return 0, rig.snapReplayErr
|
|
}
|
|
return n, nil
|
|
}
|
|
if !strings.HasPrefix(dir, rig.undoRoot) {
|
|
t.Errorf("a volume replay from an unexpected folder %q", dir)
|
|
return 0, errors.New("unexpected folder")
|
|
}
|
|
if rig.putBackErr != nil {
|
|
return 0, rig.putBackErr
|
|
}
|
|
for _, e := range entries {
|
|
if strings.HasSuffix(e.Name(), ".tar") {
|
|
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
|
|
rig.poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
|
|
n++
|
|
}
|
|
}
|
|
return n, nil
|
|
}
|
|
return m, vp, rig, rolled, notified, putBack
|
|
}
|
|
|
|
func r893Stops(vp *r893Provider) int {
|
|
n := 0
|
|
for _, c := range vp.calls {
|
|
if c == "stop" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func undoCopies(t *testing.T, root string) []string {
|
|
t.Helper()
|
|
var out []string
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, _ error) error {
|
|
if fi != nil && !fi.IsDir() {
|
|
out = append(out, p)
|
|
}
|
|
return nil
|
|
})
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// The whole point of option A, in one run: the replay fails after the snapshot's volumes replaced the live ones;
|
|
// the live volume comes back with EXACTLY the bytes copied before the restore, the volume the snapshot added is
|
|
// removed, the app starts, nothing is held, the copy is gone, the operator is told once.
|
|
// RED-PROOF: make putBackAfterFailedRestore skip m.putBack (volumesMoved ignored) → poured is empty → FAILS.
|
|
func TestR893A_AFailedReplayPutsTheLiveVolumesBackExactly(t *testing.T) {
|
|
m, vp, rig, rolled, notified, putBack := r893PutBackFixture(t)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed replay must be surfaced")
|
|
}
|
|
if rig.snapReplays != 1 {
|
|
t.Fatalf("the snapshot's volumes must have been replayed once before the failure, got %d", rig.snapReplays)
|
|
}
|
|
if got := rig.poured[r893LiveVol]; got != "LIVE bytes of "+r893LiveVol {
|
|
t.Fatalf("the live volume was poured back as %q — want the exact pre-restore copy", got)
|
|
}
|
|
if len(rig.poured) != 1 {
|
|
t.Errorf("only the volume that existed before may be poured back, got %v", rig.poured)
|
|
}
|
|
if len(rig.removed) != 1 || rig.removed[0] != r893NewVol {
|
|
t.Errorf("the volume the snapshot added must be removed, got %v", rig.removed)
|
|
}
|
|
if !vp.fullStarted {
|
|
t.Fatalf("the app must run again after the put-back — calls %v", vp.calls)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); held {
|
|
t.Fatal("a put-back app must not be held")
|
|
}
|
|
if *rolled != 1 || *notified != 0 || *putBack != 1 {
|
|
t.Errorf("rollback %d (want 1), hold notices %d (want 0), put-back notices %d (want 1)", *rolled, *notified, *putBack)
|
|
}
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
|
t.Errorf("the copy must be removed after a put-back that worked, left %v", left)
|
|
}
|
|
if !strings.Contains(err.Error(), "visszaker") {
|
|
t.Errorf("the sentence must say the data is back: %v", err)
|
|
}
|
|
}
|
|
|
|
// RED-PROOF: drop `|| undo != nil` from the volume-replay failure branch → the app starts with nothing poured back
|
|
// (a volume the replay removed stays removed) → FAILS.
|
|
func TestR893A_AFailedVolumeReplayStillPutsBackWhenItCountedNothing(t *testing.T) {
|
|
m, vp, rig, _, _, putBack := r893PutBackFixture(t)
|
|
rig.snapReplayErr = errors.New("volume create failed")
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed volume replay must be surfaced")
|
|
}
|
|
if got := rig.poured[r893LiveVol]; got != "LIVE bytes of "+r893LiveVol {
|
|
t.Fatalf("the replay removes a volume before it recreates it; the copy must be poured back even at a count of 0, got %q", got)
|
|
}
|
|
if !vp.fullStarted || *putBack != 1 {
|
|
t.Fatalf("started=%v put-back notices=%d — want the app running and one notice", vp.fullStarted, *putBack)
|
|
}
|
|
}
|
|
|
|
// The copy is checked BEFORE the stop. One byte short → refused, nothing stopped, nothing copied.
|
|
// RED-PROOF: remove volNeed from the comparison (`free < need`) → the restore runs → FAILS.
|
|
func TestR893A_TheCopyMustFitBeforeAnythingStops(t *testing.T) {
|
|
m, vp, rig, _, _, _ := r893PutBackFixture(t)
|
|
m.SetOffboxFreeFn(func(string) int64 { return (1 << 20) + 999 }) // the restore needs 1 MiB + the copy 1000 B
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a copy that does not fit must refuse the restore")
|
|
}
|
|
if n := r893Stops(vp); n != 0 {
|
|
t.Fatalf("the refusal must come before the stop — StopStack ran %d time(s)", n)
|
|
}
|
|
if len(rig.tarred) != 0 || rig.snapReplays != 0 {
|
|
t.Errorf("nothing may be copied or replayed on a refusal: tarred %v, replays %d", rig.tarred, rig.snapReplays)
|
|
}
|
|
if !strings.Contains(err.Error(), "szabad") {
|
|
t.Errorf("the refusal must name the free space: %v", err)
|
|
}
|
|
// The boundary: exactly enough room → the restore proceeds (and fails later on the fixture's replay).
|
|
m2, vp2, _, _, _, _ := r893PutBackFixture(t)
|
|
m2.SetOffboxFreeFn(func(string) int64 { return (1 << 20) + 1000 })
|
|
_, _ = m2.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if r893Stops(vp2) == 0 {
|
|
t.Error("with exactly enough room the restore must run")
|
|
}
|
|
}
|
|
|
|
// RED-PROOF: let planLiveVolumeCopy skip a probe error → the restore runs on an unmeasured copy → FAILS.
|
|
func TestR893A_AnUnmeasurableVolumeRefusesBeforeAnythingStops(t *testing.T) {
|
|
m, vp, rig, _, _, _ := r893PutBackFixture(t)
|
|
rig.probeErr = errors.New("du failed")
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("an unmeasurable volume must refuse")
|
|
}
|
|
if n := r893Stops(vp); n != 0 {
|
|
t.Fatalf("StopStack ran %d time(s) on an unmeasured copy", n)
|
|
}
|
|
}
|
|
|
|
// A restore that WORKS removes the copy and never pours it back.
|
|
// RED-PROOF: remove `defer undo.drop(m, stack)` → the copy stays → FAILS.
|
|
func TestR893A_ASuccessfulRestoreRemovesTheCopy(t *testing.T) {
|
|
m, vp, rig, _, _, putBack := r893PutBackFixture(t)
|
|
m.importDBDump = func(context.Context, DiscoveredDB, string) error { return nil }
|
|
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
|
|
t.Fatalf("restore: %v", err)
|
|
}
|
|
if len(rig.tarred) != 1 {
|
|
t.Fatalf("the live volume must have been copied before the restore, tarred %v", rig.tarred)
|
|
}
|
|
if len(rig.poured) != 0 || *putBack != 0 {
|
|
t.Errorf("nothing may be poured back after a restore that worked: poured %v, notices %d", rig.poured, *putBack)
|
|
}
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
|
t.Errorf("the copy must be removed after a restore that worked, left %v", left)
|
|
}
|
|
if !vp.fullStarted {
|
|
t.Error("the restored app must be started")
|
|
}
|
|
}
|
|
|
|
// A put-back that fails cannot promise anything: the app is HELD (option C) and the copy STAYS for support.
|
|
// RED-PROOF: remove `c.keep = true` → the deferred drop deletes the only copy → FAILS.
|
|
func TestR893A_AFailedPutBackHoldsTheAppAndKeepsTheCopy(t *testing.T) {
|
|
m, vp, rig, _, notified, putBack := r893PutBackFixture(t)
|
|
rig.putBackErr = errors.New("tar xf failed")
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed put-back must be surfaced")
|
|
}
|
|
if vp.fullStarted {
|
|
t.Fatalf("the app must NOT start on volumes that could not be put back — calls %v", vp.calls)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); !held {
|
|
t.Fatal("a failed put-back must leave the hold")
|
|
}
|
|
if *notified != 1 || *putBack != 0 {
|
|
t.Errorf("hold notices %d (want 1), put-back notices %d (want 0)", *notified, *putBack)
|
|
}
|
|
left := undoCopies(t, rig.undoRoot)
|
|
if len(left) != 1 || filepath.Base(left[0]) != r893LiveVol+".tar" {
|
|
t.Fatalf("the copy must stay on disk for support, found %v", left)
|
|
}
|
|
if low := strings.ToLower(err.Error()); strings.Contains(low, "visszaker") || !strings.Contains(low, "kapcsolatot") {
|
|
t.Errorf("the held sentence must not claim the data is back, and must send to support: %q", err.Error())
|
|
}
|
|
}
|
|
|
|
// A copy that fails touches nothing: no replay, no definition write, the app starts as it was.
|
|
// RED-PROOF: ignore the copy error (carry on with undo == nil) → the snapshot replay runs → FAILS.
|
|
func TestR893A_AFailedCopyRestoresNothingAndStartsTheApp(t *testing.T) {
|
|
m, vp, rig, rolled, _, _ := r893PutBackFixture(t)
|
|
rig.tarErr = errors.New("disk full")
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed copy must be surfaced")
|
|
}
|
|
if rig.snapReplays != 0 || *rolled != 0 || len(vp.defs) != 0 {
|
|
t.Fatalf("nothing may move after a failed copy: replays %d, rollbacks %d, definition writes %v", rig.snapReplays, *rolled, vp.defs)
|
|
}
|
|
if !vp.fullStarted {
|
|
t.Fatal("the app must be started again as it was")
|
|
}
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
|
t.Errorf("a failed copy must leave nothing behind, found %v", left)
|
|
}
|
|
}
|