package backup import ( "context" "errors" "os" "path/filepath" "sort" "strings" "testing" ) // R-893 option A (operator decision D20 = A, 2026-10-20): a failed off-site restore puts the app's live volumes // back EXACTLY and starts the app. offbox_restore_undo.go. These tests pin the CONSEQUENCE — what is poured back, // what is removed, what runs, what stays on disk — not only that a function was called. const r893LiveVol, r893NewVol = "immich_pgdata", "immich_newvol" // putBackRig records the volume legs: what the snapshot replay saw, and what the put-back poured, BYTE FOR BYTE. type putBackRig struct { scratchVolDir string undoRoot string snapReplayErr error // the snapshot's own volume replay putBackErr error // the put-back from the copy snapReplays int poured map[string]string // volume → the bytes poured back from the copy removed []string tarred []string tarErr error probeErr error } func r893PutBackFixture(t *testing.T) (*Manager, *r893Provider, *putBackRig, *int, *int, *int) { t.Helper() m, vp, rolled, notified, putBack := r893FixtureA(t, false, 0) unit := vtSnapshotUnit(t, m, "16", true) rig := &putBackRig{scratchVolDir: filepath.Join(unit, "volume-dumps"), poured: map[string]string{}} // The snapshot holds two volumes: one the box has today, one it does not (the snapshot adds it). mustWrite(t, filepath.Join(rig.scratchVolDir, r893LiveVol+".tar"), "SNAPSHOT bytes") mustWrite(t, filepath.Join(rig.scratchVolDir, r893NewVol+".tar"), "SNAPSHOT new") _, liveNs, err := m.offboxRestoreScratchDir("immich") if err != nil { t.Fatal(err) } rig.undoRoot = restoreUndoDir(liveNs, "immich") m.undoVolumeProbeFn = func(vol string) (int64, bool, error) { if rig.probeErr != nil { return 0, false, rig.probeErr } if vol == r893LiveVol { return 1000, true, nil } return 0, false, nil } m.undoVolumeTarFn = func(vol, dir string) error { if rig.tarErr != nil { return rig.tarErr } rig.tarred = append(rig.tarred, vol) return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("LIVE bytes of "+vol), 0o600) } m.undoVolumeRemoveFn = func(vol string) error { rig.removed = append(rig.removed, vol); return nil } m.volumeReplayFrom = func(_ string, dir string) (int, error) { entries, _ := os.ReadDir(dir) n := 0 if dir == rig.scratchVolDir { rig.snapReplays++ for _, e := range entries { if strings.HasSuffix(e.Name(), ".tar") { n++ } } if rig.snapReplayErr != nil { return 0, rig.snapReplayErr } return n, nil } if !strings.HasPrefix(dir, rig.undoRoot) { t.Errorf("a volume replay from an unexpected folder %q", dir) return 0, errors.New("unexpected folder") } if rig.putBackErr != nil { return 0, rig.putBackErr } for _, e := range entries { if strings.HasSuffix(e.Name(), ".tar") { b, _ := os.ReadFile(filepath.Join(dir, e.Name())) rig.poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b) n++ } } return n, nil } return m, vp, rig, rolled, notified, putBack } func r893Stops(vp *r893Provider) int { n := 0 for _, c := range vp.calls { if c == "stop" { n++ } } return n } func undoCopies(t *testing.T, root string) []string { t.Helper() var out []string _ = filepath.Walk(root, func(p string, fi os.FileInfo, _ error) error { if fi != nil && !fi.IsDir() { out = append(out, p) } return nil }) sort.Strings(out) return out } // The whole point of option A, in one run: the replay fails after the snapshot's volumes replaced the live ones; // the live volume comes back with EXACTLY the bytes copied before the restore, the volume the snapshot added is // removed, the app starts, nothing is held, the copy is gone, the operator is told once. // RED-PROOF: make putBackAfterFailedRestore skip m.putBack (volumesMoved ignored) → poured is empty → FAILS. func TestR893A_AFailedReplayPutsTheLiveVolumesBackExactly(t *testing.T) { m, vp, rig, rolled, notified, putBack := r893PutBackFixture(t) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed replay must be surfaced") } if rig.snapReplays != 1 { t.Fatalf("the snapshot's volumes must have been replayed once before the failure, got %d", rig.snapReplays) } if got := rig.poured[r893LiveVol]; got != "LIVE bytes of "+r893LiveVol { t.Fatalf("the live volume was poured back as %q — want the exact pre-restore copy", got) } if len(rig.poured) != 1 { t.Errorf("only the volume that existed before may be poured back, got %v", rig.poured) } if len(rig.removed) != 1 || rig.removed[0] != r893NewVol { t.Errorf("the volume the snapshot added must be removed, got %v", rig.removed) } if !vp.fullStarted { t.Fatalf("the app must run again after the put-back — calls %v", vp.calls) } if held, _ := m.RestoreHoldFor("immich"); held { t.Fatal("a put-back app must not be held") } if *rolled != 1 || *notified != 0 || *putBack != 1 { t.Errorf("rollback %d (want 1), hold notices %d (want 0), put-back notices %d (want 1)", *rolled, *notified, *putBack) } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Errorf("the copy must be removed after a put-back that worked, left %v", left) } if !strings.Contains(err.Error(), "visszaker") { t.Errorf("the sentence must say the data is back: %v", err) } } // RED-PROOF: drop `|| undo != nil` from the volume-replay failure branch → the app starts with nothing poured back // (a volume the replay removed stays removed) → FAILS. func TestR893A_AFailedVolumeReplayStillPutsBackWhenItCountedNothing(t *testing.T) { m, vp, rig, _, _, putBack := r893PutBackFixture(t) rig.snapReplayErr = errors.New("volume create failed") _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed volume replay must be surfaced") } if got := rig.poured[r893LiveVol]; got != "LIVE bytes of "+r893LiveVol { t.Fatalf("the replay removes a volume before it recreates it; the copy must be poured back even at a count of 0, got %q", got) } if !vp.fullStarted || *putBack != 1 { t.Fatalf("started=%v put-back notices=%d — want the app running and one notice", vp.fullStarted, *putBack) } } // The copy is checked BEFORE the stop. One byte short → refused, nothing stopped, nothing copied. // RED-PROOF: remove volNeed from the comparison (`free < need`) → the restore runs → FAILS. func TestR893A_TheCopyMustFitBeforeAnythingStops(t *testing.T) { m, vp, rig, _, _, _ := r893PutBackFixture(t) m.SetOffboxFreeFn(func(string) int64 { return (1 << 20) + 999 }) // the restore needs 1 MiB + the copy 1000 B _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a copy that does not fit must refuse the restore") } if n := r893Stops(vp); n != 0 { t.Fatalf("the refusal must come before the stop — StopStack ran %d time(s)", n) } if len(rig.tarred) != 0 || rig.snapReplays != 0 { t.Errorf("nothing may be copied or replayed on a refusal: tarred %v, replays %d", rig.tarred, rig.snapReplays) } if !strings.Contains(err.Error(), "szabad") { t.Errorf("the refusal must name the free space: %v", err) } // The boundary: exactly enough room → the restore proceeds (and fails later on the fixture's replay). m2, vp2, _, _, _, _ := r893PutBackFixture(t) m2.SetOffboxFreeFn(func(string) int64 { return (1 << 20) + 1000 }) _, _ = m2.ReconstituteFromOffsite(context.Background(), "immich", false) if r893Stops(vp2) == 0 { t.Error("with exactly enough room the restore must run") } } // RED-PROOF: let planLiveVolumeCopy skip a probe error → the restore runs on an unmeasured copy → FAILS. func TestR893A_AnUnmeasurableVolumeRefusesBeforeAnythingStops(t *testing.T) { m, vp, rig, _, _, _ := r893PutBackFixture(t) rig.probeErr = errors.New("du failed") _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("an unmeasurable volume must refuse") } if n := r893Stops(vp); n != 0 { t.Fatalf("StopStack ran %d time(s) on an unmeasured copy", n) } } // A restore that WORKS removes the copy and never pours it back. // RED-PROOF: remove `defer undo.drop(m, stack)` → the copy stays → FAILS. func TestR893A_ASuccessfulRestoreRemovesTheCopy(t *testing.T) { m, vp, rig, _, _, putBack := r893PutBackFixture(t) m.importDBDump = func(context.Context, DiscoveredDB, string) error { return nil } if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil { t.Fatalf("restore: %v", err) } if len(rig.tarred) != 1 { t.Fatalf("the live volume must have been copied before the restore, tarred %v", rig.tarred) } if len(rig.poured) != 0 || *putBack != 0 { t.Errorf("nothing may be poured back after a restore that worked: poured %v, notices %d", rig.poured, *putBack) } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Errorf("the copy must be removed after a restore that worked, left %v", left) } if !vp.fullStarted { t.Error("the restored app must be started") } } // A put-back that fails cannot promise anything: the app is HELD (option C) and the copy STAYS for support. // RED-PROOF: remove `c.keep = true` → the deferred drop deletes the only copy → FAILS. func TestR893A_AFailedPutBackHoldsTheAppAndKeepsTheCopy(t *testing.T) { m, vp, rig, _, notified, putBack := r893PutBackFixture(t) rig.putBackErr = errors.New("tar xf failed") _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed put-back must be surfaced") } if vp.fullStarted { t.Fatalf("the app must NOT start on volumes that could not be put back — calls %v", vp.calls) } if held, _ := m.RestoreHoldFor("immich"); !held { t.Fatal("a failed put-back must leave the hold") } if *notified != 1 || *putBack != 0 { t.Errorf("hold notices %d (want 1), put-back notices %d (want 0)", *notified, *putBack) } left := undoCopies(t, rig.undoRoot) if len(left) != 1 || filepath.Base(left[0]) != r893LiveVol+".tar" { t.Fatalf("the copy must stay on disk for support, found %v", left) } if low := strings.ToLower(err.Error()); strings.Contains(low, "visszaker") || !strings.Contains(low, "kapcsolatot") { t.Errorf("the held sentence must not claim the data is back, and must send to support: %q", err.Error()) } } // A copy that fails touches nothing: no replay, no definition write, the app starts as it was. // RED-PROOF: ignore the copy error (carry on with undo == nil) → the snapshot replay runs → FAILS. func TestR893A_AFailedCopyRestoresNothingAndStartsTheApp(t *testing.T) { m, vp, rig, rolled, _, _ := r893PutBackFixture(t) rig.tarErr = errors.New("disk full") _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed copy must be surfaced") } if rig.snapReplays != 0 || *rolled != 0 || len(vp.defs) != 0 { t.Fatalf("nothing may move after a failed copy: replays %d, rollbacks %d, definition writes %v", rig.snapReplays, *rolled, vp.defs) } if !vp.fullStarted { t.Fatal("the app must be started again as it was") } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Errorf("a failed copy must leave nothing behind, found %v", left) } }