D22: every restore puts the data back exactly on failure and after a power cut (R-1177, R-1195); whole restore asks the unit first (R-1178); no mirror during a restore (R-1180); W22 restore refusals in Hungarian; unreleased
gates / gates (push) Successful in 1m40s
gates / gates (push) Successful in 1m40s
This commit is contained in:
@@ -0,0 +1,391 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
// D22 (operator ruling 2026-10-21 20:05): the off-site restore's „put back exactly as it was" (R-893 option A) now
|
||||
// covers the LOCAL unit restore (R-1177) — and a restore cut by a power loss is put back on start (R-1195,
|
||||
// restore_crash_putback.go). These tests pin the CONSEQUENCE: which bytes are poured back, what is removed, which
|
||||
// definition is written last, whether the app runs, what stays on disk.
|
||||
|
||||
const r1177Vol, r1177NewVol = "app_data", "app_newvol"
|
||||
|
||||
type r1177Rig struct {
|
||||
unitVolDir string
|
||||
undoRoot string
|
||||
poured map[string]string
|
||||
removed []string
|
||||
unitErr error // the unit's own volume replay
|
||||
putBackErr error
|
||||
defsWrote []string // RecreateStackDefinitionFromUnit's folders, in order
|
||||
liveDir string
|
||||
}
|
||||
|
||||
// r1177Provider records which definition folder was written, so a test can tell the unit's from the live one.
|
||||
type r1177Provider struct {
|
||||
*fakeRecoveryProvider
|
||||
rig *r1177Rig
|
||||
}
|
||||
|
||||
func (p *r1177Provider) RecreateStackDefinitionFromUnit(name, dir string, env map[string]string) error {
|
||||
p.rig.defsWrote = append(p.rig.defsWrote, dir)
|
||||
return p.fakeRecoveryProvider.RecreateStackDefinitionFromUnit(name, dir, env)
|
||||
}
|
||||
|
||||
// r1177Fixture: the r47 unit for "app" (Postgres dump), plus a volume archive in the unit, a live definition on disk,
|
||||
// one live volume the box has (app_data) and one only the unit has (app_newvol). Docker is never reached.
|
||||
func r1177Fixture(t *testing.T) (*Manager, *r1177Provider, *r1177Rig, *[]string) {
|
||||
t.Helper()
|
||||
m, prov, imported := r47UnitFixture(t, immichLikeCompose, true)
|
||||
rig := &r1177Rig{poured: map[string]string{}, liveDir: t.TempDir()}
|
||||
prov.info.StackDir = rig.liveDir
|
||||
mustWrite(t, filepath.Join(rig.liveDir, "docker-compose.yml"), "LIVE compose\n")
|
||||
mustWrite(t, filepath.Join(rig.liveDir, "app.yaml"), "deployed: true\nenv:\n SUBDOMAIN: app\n")
|
||||
p := &r1177Provider{fakeRecoveryProvider: prov, rig: rig}
|
||||
m.stackProvider = p
|
||||
unit := m.primaryUnitDirFor("app")
|
||||
rig.unitVolDir = UnitVolumeDumpDir(unit)
|
||||
mustWrite(t, filepath.Join(rig.unitVolDir, r1177Vol+".tar"), "UNIT bytes")
|
||||
mustWrite(t, filepath.Join(rig.unitVolDir, r1177NewVol+".tar"), "UNIT new")
|
||||
rig.undoRoot = restoreUndoDir(m.namespaceRoot(m.GetAppDrivePath("app")), "app")
|
||||
|
||||
m.undoVolumeProbeFn = func(vol string) (int64, bool, error) { return 1000, vol == r1177Vol, nil }
|
||||
m.undoVolumeTarFn = func(vol, dir string) error {
|
||||
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("LIVE bytes of "+vol), 0o600)
|
||||
}
|
||||
m.undoVolumeRemoveFn = func(vol string) error { rig.removed = append(rig.removed, vol); return nil }
|
||||
m.volumeReplayFrom = func(_ string, dir string) (int, error) {
|
||||
if dir == rig.unitVolDir {
|
||||
if rig.unitErr != nil {
|
||||
return 0, rig.unitErr
|
||||
}
|
||||
return 2, nil
|
||||
}
|
||||
if !strings.HasPrefix(dir, rig.undoRoot) {
|
||||
t.Errorf("a volume replay from an unexpected folder %q", dir)
|
||||
return 0, errors.New("unexpected folder")
|
||||
}
|
||||
if rig.putBackErr != nil {
|
||||
return 0, rig.putBackErr
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
n := 0
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".tar") {
|
||||
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
rig.poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
return m, p, rig, imported
|
||||
}
|
||||
|
||||
// assertPutBack is the consequence D22 promises: the live bytes are back, the unit's extra volume is gone, the LIVE
|
||||
// definition was written last, the app runs, the copy is removed, and the household reads „nothing changed".
|
||||
func assertPutBack(t *testing.T, err error, p *r1177Provider, rig *r1177Rig) {
|
||||
t.Helper()
|
||||
if want := util.MsgError("err.backup.restore_failed_put_back", "app").Error(); err == nil || err.Error() != want {
|
||||
t.Fatalf("the household must read the put-back sentence, got: %v", err)
|
||||
}
|
||||
if rig.poured[r1177Vol] != "LIVE bytes of "+r1177Vol {
|
||||
t.Fatalf("the live volume was not poured back byte for byte: %v", rig.poured)
|
||||
}
|
||||
if len(rig.removed) != 1 || rig.removed[0] != r1177NewVol {
|
||||
t.Fatalf("the volume only the unit has must be removed, removed %v", rig.removed)
|
||||
}
|
||||
if n := len(rig.defsWrote); n == 0 || !strings.Contains(rig.defsWrote[n-1], "felhom-livedef-") {
|
||||
t.Fatalf("the LIVE definition must be written back last, wrote %v", rig.defsWrote)
|
||||
}
|
||||
if !p.fullStarted || p.calls[len(p.calls)-1] != "start" {
|
||||
t.Fatalf("the app must run again, calls %v", p.calls)
|
||||
}
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
||||
t.Fatalf("the copy must be removed once it is poured back, found %v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1177's measured case (C3, 9202): a damaged volume archive in the unit. Before D22 the live volume was gone, the
|
||||
// database replay skipped and the app started on nothing (Postgres crash-looped).
|
||||
// RED-PROOF: on the pre-D22 restore_unit.go this fails on the error text („completed with data errors") and on the
|
||||
// live bytes never poured back.
|
||||
func TestR1177_ADamagedVolumePutsTheLiveDataBackAndStartsTheApp(t *testing.T) {
|
||||
m, p, rig, imported := r1177Fixture(t)
|
||||
rig.unitErr = errors.New("failed to restore 1 volume(s): [app_data]")
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
assertPutBack(t, err, p, rig)
|
||||
if len(*imported) != 0 {
|
||||
t.Fatalf("the database copy was replayed over a volume that failed (R-638): %v", *imported)
|
||||
}
|
||||
}
|
||||
|
||||
// A database replay that fails half-way: the live volumes (the database's own among them) go back.
|
||||
func TestR1177_AFailedDatabaseReplayPutsTheLiveDataBack(t *testing.T) {
|
||||
m, p, rig, _ := r1177Fixture(t)
|
||||
m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("ERROR: relation exists") }
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
assertPutBack(t, err, p, rig)
|
||||
}
|
||||
|
||||
// A put-back that fails: the app is HELD and the copy stays on disk — never started on data nobody knows.
|
||||
func TestR1177_AFailedPutBackHoldsTheAppAndKeepsTheCopy(t *testing.T) {
|
||||
m, p, rig, _ := r1177Fixture(t)
|
||||
rig.unitErr = errors.New("tar: short read")
|
||||
rig.putBackErr = errors.New("tar xf failed")
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
if want := util.MsgError("err.backup.restore_failed_held_mixed", "app").Error(); err == nil || err.Error() != want {
|
||||
t.Fatalf("a failed put-back must hold, got: %v", err)
|
||||
}
|
||||
if p.fullStarted {
|
||||
t.Fatalf("the app must NOT start on volumes that could not be put back — calls %v", p.calls)
|
||||
}
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 1 || filepath.Base(left[0]) != r1177Vol+".tar" {
|
||||
t.Fatalf("the copy must stay on disk for support, found %v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// Not enough room for the copy: refused BEFORE the stop, nothing touched.
|
||||
func TestR1177_NoRoomForTheCopyRefusesBeforeTheStop(t *testing.T) {
|
||||
m, p, rig, _ := r1177Fixture(t)
|
||||
m.freeBytesFn = func(string) int64 { return 10 }
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
if err == nil || !strings.Contains(err.Error(), "Nincs elég szabad hely") {
|
||||
t.Fatalf("want the no-room refusal, got %v", err)
|
||||
}
|
||||
if len(p.calls) != 0 || len(rig.poured) != 0 {
|
||||
t.Fatalf("a refusal must touch nothing — calls %v poured %v", p.calls, rig.poured)
|
||||
}
|
||||
}
|
||||
|
||||
// The control: a restore that works pours nothing back, writes only the unit's definition, starts once, and leaves
|
||||
// no copy behind.
|
||||
func TestR1177_ASuccessfulRestoreIsUnchangedAndLeavesNoCopy(t *testing.T) {
|
||||
m, p, rig, imported := r1177Fixture(t)
|
||||
res, err := m.RestoreFromRecoveryUnit("app")
|
||||
if err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
if len(rig.poured) != 0 || len(rig.removed) != 0 {
|
||||
t.Fatalf("a successful restore poured back %v / removed %v", rig.poured, rig.removed)
|
||||
}
|
||||
if len(*imported) != 1 || res.VolumesReplayed != 2 {
|
||||
t.Fatalf("the restore itself must be unchanged: imported %v volumes %d", *imported, res.VolumesReplayed)
|
||||
}
|
||||
if got := strings.Join(p.calls, ","); got != "stop,recreate,startsvc:immich-postgres,start" {
|
||||
t.Fatalf("sequence = %q", got)
|
||||
}
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
||||
t.Fatalf("the copy must be removed after a successful restore, found %v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1195: a controller killed between the volume step and the database replay. What is on disk at the kill — the
|
||||
// marker and the copy — is what the next start sees; Recover must pour the copy back BEFORE it starts the app.
|
||||
// The kill is emulated by saving the marker and the copy at the replay (the moment of the cut), letting the run
|
||||
// finish, then putting the saved disk back and starting a fresh guard over it.
|
||||
// RED-PROOF: with Recover's put-back removed (or persistUndoForCrash not called) this fails on „nothing was poured
|
||||
// back" — the pre-D22 restart left the app on the unit's data.
|
||||
func TestR1195_AKilledLocalRestoreIsPutBackOnStart(t *testing.T) {
|
||||
m, _, rig, _ := r1177Fixture(t)
|
||||
marker := filepath.Join(t.TempDir(), "appstop-state.json")
|
||||
m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
saved := map[string][]byte{}
|
||||
m.importDBDump = func(context.Context, DiscoveredDB, string) error {
|
||||
// the cut: the volumes are the unit's, the replay has begun
|
||||
_ = filepath.Walk(filepath.Dir(rig.undoRoot), func(p string, fi os.FileInfo, _ error) error {
|
||||
if fi != nil && !fi.IsDir() {
|
||||
b, _ := os.ReadFile(p)
|
||||
saved[p] = b
|
||||
}
|
||||
return nil
|
||||
})
|
||||
b, err := os.ReadFile(marker)
|
||||
if err != nil {
|
||||
t.Fatalf("no app-stop marker at the cut: %v", err)
|
||||
}
|
||||
saved[marker] = b
|
||||
return nil
|
||||
}
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatalf("the run after the cut point: %v", err)
|
||||
}
|
||||
if len(saved) < 3 {
|
||||
t.Fatalf("the copy was not on disk at the cut: %v", saved)
|
||||
}
|
||||
for p, b := range saved { // the disk as the kill left it
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o700)
|
||||
if err := os.WriteFile(p, b, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var calls []string
|
||||
starter := starterFunc(func(name string) error { calls = append(calls, "start:"+name); return nil })
|
||||
defs := []string{}
|
||||
cp := NewCrashPutBack(func(name string) error { calls = append(calls, "stop:"+name); return nil },
|
||||
func(name, dir string) error {
|
||||
defs = append(defs, dir)
|
||||
b, _ := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
|
||||
calls = append(calls, "def:"+strings.TrimSpace(string(b)))
|
||||
return nil
|
||||
}, log.New(io.Discard, "", 0))
|
||||
poured := map[string]string{}
|
||||
cp.volReplay = func(_ string, dir string) (int, error) {
|
||||
entries, _ := os.ReadDir(dir)
|
||||
n := 0
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".tar") {
|
||||
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
var removed []string
|
||||
cp.volRemove = func(v string) error { removed = append(removed, v); return nil }
|
||||
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
g.SetStarter(starter)
|
||||
g.SetPutBacker(cp)
|
||||
rec := g.Recover()
|
||||
|
||||
if rec == nil || len(rec.PutBack) != 1 || rec.PutBack[0] != "app" {
|
||||
t.Fatalf("the interrupted restore must be put back, got %+v", rec)
|
||||
}
|
||||
if poured[r1177Vol] != "LIVE bytes of "+r1177Vol {
|
||||
t.Fatalf("nothing was poured back (or not the live bytes): %v", poured)
|
||||
}
|
||||
if len(removed) != 1 || removed[0] != r1177NewVol {
|
||||
t.Fatalf("the unit's extra volume must be removed, removed %v", removed)
|
||||
}
|
||||
if got := strings.Join(calls, ","); got != "stop:app,def:LIVE compose,start:app" {
|
||||
t.Fatalf("order = %q, want stop → live definition → start", got)
|
||||
}
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
|
||||
t.Fatalf("the used copy must be removed, found %v", left)
|
||||
}
|
||||
if _, err := os.Stat(marker); err == nil {
|
||||
if b, _ := os.ReadFile(marker); strings.Contains(string(b), `"active": true`) {
|
||||
t.Fatalf("the marker must be cleared after the put-back and start: %s", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cut BEFORE the copy was whole leaves no undo record: the old restart, nothing poured.
|
||||
func TestR1195_ACutBeforeTheCopyRestartsAsBefore(t *testing.T) {
|
||||
marker := filepath.Join(t.TempDir(), "appstop-state.json")
|
||||
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls []string
|
||||
g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil }))
|
||||
g2.SetPutBacker(NewCrashPutBack(func(string) error { calls = append(calls, "stop"); return nil }, nil, log.New(io.Discard, "", 0)))
|
||||
rec := g2.Recover()
|
||||
if rec == nil || len(rec.PutBack) != 0 || strings.Join(calls, ",") != "start:app" {
|
||||
t.Fatalf("no copy → the plain restart, got %+v calls %v", rec, calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A put-back that fails at start: the app is NOT started and the marker stays for the next start.
|
||||
func TestR1195_AFailedPutBackOnStartLeavesTheAppStopped(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
marker := filepath.Join(dir, "appstop-state.json")
|
||||
undo := filepath.Join(dir, "backups", "restore-undo", "app", "20261021T000000Z")
|
||||
mustWrite(t, filepath.Join(undo, "undo.json"), `{"stack":"app","copied":["app_data"],"definition":false}`)
|
||||
mustWrite(t, filepath.Join(undo, "app_data.tar"), "LIVE")
|
||||
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.SetUndo("app", undo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls []string
|
||||
g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil }))
|
||||
cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0))
|
||||
cp.volReplay = func(string, string) (int, error) { return 0, errors.New("disk gone") }
|
||||
g2.SetPutBacker(cp)
|
||||
rec := g2.Recover()
|
||||
if rec == nil || len(rec.Failed) != 1 || len(calls) != 0 {
|
||||
t.Fatalf("a failed put-back must not start the app: %+v calls %v", rec, calls)
|
||||
}
|
||||
if b, _ := os.ReadFile(marker); !strings.Contains(string(b), `"active": true`) {
|
||||
t.Fatal("the marker must stay so the next start tries again")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(undo, "app_data.tar")); err != nil {
|
||||
t.Fatal("the copy must stay on disk after a failed put-back")
|
||||
}
|
||||
}
|
||||
|
||||
type starterFunc func(string) error
|
||||
|
||||
func (f starterFunc) StartStack(n string) error { return f(n) }
|
||||
|
||||
// R-1195's MEASURED case is the OFF-SITE full restore (rehearsal 2026-10-20 s2): the same cut, on that path — the
|
||||
// marker and the copy on disk at the replay are what the next start pours back.
|
||||
// RED-PROOF: without persistUndoForCrash in ReconstituteFromOffsite the marker holds no Undo → nothing poured back.
|
||||
func TestR1195_AKilledOffsiteRestoreIsPutBackOnStart(t *testing.T) {
|
||||
m, _, rig, _, _, _ := r893PutBackFixture(t)
|
||||
marker := filepath.Join(t.TempDir(), "appstop-state.json")
|
||||
m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
saved := map[string][]byte{}
|
||||
m.importDBDump = func(context.Context, DiscoveredDB, string) error {
|
||||
_ = filepath.Walk(rig.undoRoot, func(p string, fi os.FileInfo, _ error) error {
|
||||
if fi != nil && !fi.IsDir() {
|
||||
b, _ := os.ReadFile(p)
|
||||
saved[p] = b
|
||||
}
|
||||
return nil
|
||||
})
|
||||
b, _ := os.ReadFile(marker)
|
||||
saved[marker] = b
|
||||
return nil
|
||||
}
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
|
||||
t.Fatalf("the run after the cut point: %v", err)
|
||||
}
|
||||
for p, b := range saved {
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o700)
|
||||
_ = os.WriteFile(p, b, 0o600)
|
||||
}
|
||||
poured := map[string]string{}
|
||||
cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0))
|
||||
cp.volReplay = func(_ string, dir string) (int, error) {
|
||||
entries, _ := os.ReadDir(dir)
|
||||
n := 0
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".tar") {
|
||||
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
var removed []string
|
||||
cp.volRemove = func(v string) error { removed = append(removed, v); return nil }
|
||||
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
|
||||
g.SetStarter(starterFunc(func(string) error { return nil }))
|
||||
g.SetPutBacker(cp)
|
||||
rec := g.Recover()
|
||||
if rec == nil || len(rec.PutBack) != 1 || poured[r893LiveVol] != "LIVE bytes of "+r893LiveVol {
|
||||
t.Fatalf("the cut off-site restore must be put back on start: rec %+v poured %v", rec, poured)
|
||||
}
|
||||
if len(removed) != 1 || removed[0] != r893NewVol {
|
||||
t.Fatalf("the snapshot's extra volume must be removed, removed %v", removed)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user