package backup import ( "context" "errors" "io" "log" "os" "path/filepath" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/util" ) // D22 (operator ruling 2026-10-21 20:05): the off-site restore's „put back exactly as it was" (R-893 option A) now // covers the LOCAL unit restore (R-1177) — and a restore cut by a power loss is put back on start (R-1195, // restore_crash_putback.go). These tests pin the CONSEQUENCE: which bytes are poured back, what is removed, which // definition is written last, whether the app runs, what stays on disk. const r1177Vol, r1177NewVol = "app_data", "app_newvol" type r1177Rig struct { unitVolDir string undoRoot string poured map[string]string removed []string unitErr error // the unit's own volume replay putBackErr error defsWrote []string // RecreateStackDefinitionFromUnit's folders, in order liveDir string } // r1177Provider records which definition folder was written, so a test can tell the unit's from the live one. type r1177Provider struct { *fakeRecoveryProvider rig *r1177Rig } func (p *r1177Provider) RecreateStackDefinitionFromUnit(name, dir string, env map[string]string) error { p.rig.defsWrote = append(p.rig.defsWrote, dir) return p.fakeRecoveryProvider.RecreateStackDefinitionFromUnit(name, dir, env) } // r1177Fixture: the r47 unit for "app" (Postgres dump), plus a volume archive in the unit, a live definition on disk, // one live volume the box has (app_data) and one only the unit has (app_newvol). Docker is never reached. func r1177Fixture(t *testing.T) (*Manager, *r1177Provider, *r1177Rig, *[]string) { t.Helper() m, prov, imported := r47UnitFixture(t, immichLikeCompose, true) rig := &r1177Rig{poured: map[string]string{}, liveDir: t.TempDir()} prov.info.StackDir = rig.liveDir mustWrite(t, filepath.Join(rig.liveDir, "docker-compose.yml"), "LIVE compose\n") mustWrite(t, filepath.Join(rig.liveDir, "app.yaml"), "deployed: true\nenv:\n SUBDOMAIN: app\n") p := &r1177Provider{fakeRecoveryProvider: prov, rig: rig} m.stackProvider = p unit := m.primaryUnitDirFor("app") rig.unitVolDir = UnitVolumeDumpDir(unit) mustWrite(t, filepath.Join(rig.unitVolDir, r1177Vol+".tar"), "UNIT bytes") mustWrite(t, filepath.Join(rig.unitVolDir, r1177NewVol+".tar"), "UNIT new") rig.undoRoot = restoreUndoDir(m.namespaceRoot(m.GetAppDrivePath("app")), "app") m.undoVolumeProbeFn = func(vol string) (int64, bool, error) { return 1000, vol == r1177Vol, nil } m.undoVolumeTarFn = func(vol, dir string) error { return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("LIVE bytes of "+vol), 0o600) } m.undoVolumeRemoveFn = func(vol string) error { rig.removed = append(rig.removed, vol); return nil } m.volumeReplayFrom = func(_ string, dir string) (int, error) { if dir == rig.unitVolDir { if rig.unitErr != nil { return 0, rig.unitErr } return 2, nil } if !strings.HasPrefix(dir, rig.undoRoot) { t.Errorf("a volume replay from an unexpected folder %q", dir) return 0, errors.New("unexpected folder") } if rig.putBackErr != nil { return 0, rig.putBackErr } entries, _ := os.ReadDir(dir) n := 0 for _, e := range entries { if strings.HasSuffix(e.Name(), ".tar") { b, _ := os.ReadFile(filepath.Join(dir, e.Name())) rig.poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b) n++ } } return n, nil } return m, p, rig, imported } // assertPutBack is the consequence D22 promises: the live bytes are back, the unit's extra volume is gone, the LIVE // definition was written last, the app runs, the copy is removed, and the household reads „nothing changed". func assertPutBack(t *testing.T, err error, p *r1177Provider, rig *r1177Rig) { t.Helper() if want := util.MsgError("err.backup.restore_failed_put_back", "app").Error(); err == nil || err.Error() != want { t.Fatalf("the household must read the put-back sentence, got: %v", err) } if rig.poured[r1177Vol] != "LIVE bytes of "+r1177Vol { t.Fatalf("the live volume was not poured back byte for byte: %v", rig.poured) } if len(rig.removed) != 1 || rig.removed[0] != r1177NewVol { t.Fatalf("the volume only the unit has must be removed, removed %v", rig.removed) } if n := len(rig.defsWrote); n == 0 || !strings.Contains(rig.defsWrote[n-1], "felhom-livedef-") { t.Fatalf("the LIVE definition must be written back last, wrote %v", rig.defsWrote) } if !p.fullStarted || p.calls[len(p.calls)-1] != "start" { t.Fatalf("the app must run again, calls %v", p.calls) } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Fatalf("the copy must be removed once it is poured back, found %v", left) } } // R-1177's measured case (C3, 9202): a damaged volume archive in the unit. Before D22 the live volume was gone, the // database replay skipped and the app started on nothing (Postgres crash-looped). // RED-PROOF: on the pre-D22 restore_unit.go this fails on the error text („completed with data errors") and on the // live bytes never poured back. func TestR1177_ADamagedVolumePutsTheLiveDataBackAndStartsTheApp(t *testing.T) { m, p, rig, imported := r1177Fixture(t) rig.unitErr = errors.New("failed to restore 1 volume(s): [app_data]") _, err := m.RestoreFromRecoveryUnit("app") assertPutBack(t, err, p, rig) if len(*imported) != 0 { t.Fatalf("the database copy was replayed over a volume that failed (R-638): %v", *imported) } } // A database replay that fails half-way: the live volumes (the database's own among them) go back. func TestR1177_AFailedDatabaseReplayPutsTheLiveDataBack(t *testing.T) { m, p, rig, _ := r1177Fixture(t) m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("ERROR: relation exists") } _, err := m.RestoreFromRecoveryUnit("app") assertPutBack(t, err, p, rig) } // A put-back that fails: the app is HELD and the copy stays on disk — never started on data nobody knows. func TestR1177_AFailedPutBackHoldsTheAppAndKeepsTheCopy(t *testing.T) { m, p, rig, _ := r1177Fixture(t) rig.unitErr = errors.New("tar: short read") rig.putBackErr = errors.New("tar xf failed") _, err := m.RestoreFromRecoveryUnit("app") if want := util.MsgError("err.backup.restore_failed_held_mixed", "app").Error(); err == nil || err.Error() != want { t.Fatalf("a failed put-back must hold, got: %v", err) } if p.fullStarted { t.Fatalf("the app must NOT start on volumes that could not be put back — calls %v", p.calls) } if left := undoCopies(t, rig.undoRoot); len(left) != 1 || filepath.Base(left[0]) != r1177Vol+".tar" { t.Fatalf("the copy must stay on disk for support, found %v", left) } } // Not enough room for the copy: refused BEFORE the stop, nothing touched. func TestR1177_NoRoomForTheCopyRefusesBeforeTheStop(t *testing.T) { m, p, rig, _ := r1177Fixture(t) m.freeBytesFn = func(string) int64 { return 10 } _, err := m.RestoreFromRecoveryUnit("app") if err == nil || !strings.Contains(err.Error(), "Nincs elég szabad hely") { t.Fatalf("want the no-room refusal, got %v", err) } if len(p.calls) != 0 || len(rig.poured) != 0 { t.Fatalf("a refusal must touch nothing — calls %v poured %v", p.calls, rig.poured) } } // The control: a restore that works pours nothing back, writes only the unit's definition, starts once, and leaves // no copy behind. func TestR1177_ASuccessfulRestoreIsUnchangedAndLeavesNoCopy(t *testing.T) { m, p, rig, imported := r1177Fixture(t) res, err := m.RestoreFromRecoveryUnit("app") if err != nil { t.Fatalf("restore: %v", err) } if len(rig.poured) != 0 || len(rig.removed) != 0 { t.Fatalf("a successful restore poured back %v / removed %v", rig.poured, rig.removed) } if len(*imported) != 1 || res.VolumesReplayed != 2 { t.Fatalf("the restore itself must be unchanged: imported %v volumes %d", *imported, res.VolumesReplayed) } if got := strings.Join(p.calls, ","); got != "stop,recreate,startsvc:immich-postgres,start" { t.Fatalf("sequence = %q", got) } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Fatalf("the copy must be removed after a successful restore, found %v", left) } } // R-1195: a controller killed between the volume step and the database replay. What is on disk at the kill — the // marker and the copy — is what the next start sees; Recover must pour the copy back BEFORE it starts the app. // The kill is emulated by saving the marker and the copy at the replay (the moment of the cut), letting the run // finish, then putting the saved disk back and starting a fresh guard over it. // RED-PROOF: with Recover's put-back removed (or persistUndoForCrash not called) this fails on „nothing was poured // back" — the pre-D22 restart left the app on the unit's data. func TestR1195_AKilledLocalRestoreIsPutBackOnStart(t *testing.T) { m, _, rig, _ := r1177Fixture(t) marker := filepath.Join(t.TempDir(), "appstop-state.json") m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0)) saved := map[string][]byte{} m.importDBDump = func(context.Context, DiscoveredDB, string) error { // the cut: the volumes are the unit's, the replay has begun _ = filepath.Walk(filepath.Dir(rig.undoRoot), func(p string, fi os.FileInfo, _ error) error { if fi != nil && !fi.IsDir() { b, _ := os.ReadFile(p) saved[p] = b } return nil }) b, err := os.ReadFile(marker) if err != nil { t.Fatalf("no app-stop marker at the cut: %v", err) } saved[marker] = b return nil } if _, err := m.RestoreFromRecoveryUnit("app"); err != nil { t.Fatalf("the run after the cut point: %v", err) } if len(saved) < 3 { t.Fatalf("the copy was not on disk at the cut: %v", saved) } for p, b := range saved { // the disk as the kill left it _ = os.MkdirAll(filepath.Dir(p), 0o700) if err := os.WriteFile(p, b, 0o600); err != nil { t.Fatal(err) } } var calls []string starter := starterFunc(func(name string) error { calls = append(calls, "start:"+name); return nil }) defs := []string{} cp := NewCrashPutBack(func(name string) error { calls = append(calls, "stop:"+name); return nil }, func(name, dir string) error { defs = append(defs, dir) b, _ := os.ReadFile(filepath.Join(dir, "docker-compose.yml")) calls = append(calls, "def:"+strings.TrimSpace(string(b))) return nil }, log.New(io.Discard, "", 0)) poured := map[string]string{} cp.volReplay = func(_ string, dir string) (int, error) { entries, _ := os.ReadDir(dir) n := 0 for _, e := range entries { if strings.HasSuffix(e.Name(), ".tar") { b, _ := os.ReadFile(filepath.Join(dir, e.Name())) poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b) n++ } } return n, nil } var removed []string cp.volRemove = func(v string) error { removed = append(removed, v); return nil } g := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) g.SetStarter(starter) g.SetPutBacker(cp) rec := g.Recover() if rec == nil || len(rec.PutBack) != 1 || rec.PutBack[0] != "app" { t.Fatalf("the interrupted restore must be put back, got %+v", rec) } if poured[r1177Vol] != "LIVE bytes of "+r1177Vol { t.Fatalf("nothing was poured back (or not the live bytes): %v", poured) } if len(removed) != 1 || removed[0] != r1177NewVol { t.Fatalf("the unit's extra volume must be removed, removed %v", removed) } if got := strings.Join(calls, ","); got != "stop:app,def:LIVE compose,start:app" { t.Fatalf("order = %q, want stop → live definition → start", got) } if left := undoCopies(t, rig.undoRoot); len(left) != 0 { t.Fatalf("the used copy must be removed, found %v", left) } if _, err := os.Stat(marker); err == nil { if b, _ := os.ReadFile(marker); strings.Contains(string(b), `"active": true`) { t.Fatalf("the marker must be cleared after the put-back and start: %s", b) } } } // A cut BEFORE the copy was whole leaves no undo record: the old restart, nothing poured. func TestR1195_ACutBeforeTheCopyRestartsAsBefore(t *testing.T) { marker := filepath.Join(t.TempDir(), "appstop-state.json") g := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil { t.Fatal(err) } var calls []string g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil })) g2.SetPutBacker(NewCrashPutBack(func(string) error { calls = append(calls, "stop"); return nil }, nil, log.New(io.Discard, "", 0))) rec := g2.Recover() if rec == nil || len(rec.PutBack) != 0 || strings.Join(calls, ",") != "start:app" { t.Fatalf("no copy → the plain restart, got %+v calls %v", rec, calls) } } // A put-back that fails at start: the app is NOT started and the marker stays for the next start. func TestR1195_AFailedPutBackOnStartLeavesTheAppStopped(t *testing.T) { dir := t.TempDir() marker := filepath.Join(dir, "appstop-state.json") undo := filepath.Join(dir, "backups", "restore-undo", "app", "20261021T000000Z") mustWrite(t, filepath.Join(undo, "undo.json"), `{"stack":"app","copied":["app_data"],"definition":false}`) mustWrite(t, filepath.Join(undo, "app_data.tar"), "LIVE") g := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil { t.Fatal(err) } if err := g.SetUndo("app", undo); err != nil { t.Fatal(err) } var calls []string g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil })) cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0)) cp.volReplay = func(string, string) (int, error) { return 0, errors.New("disk gone") } g2.SetPutBacker(cp) rec := g2.Recover() if rec == nil || len(rec.Failed) != 1 || len(calls) != 0 { t.Fatalf("a failed put-back must not start the app: %+v calls %v", rec, calls) } if b, _ := os.ReadFile(marker); !strings.Contains(string(b), `"active": true`) { t.Fatal("the marker must stay so the next start tries again") } if _, err := os.Stat(filepath.Join(undo, "app_data.tar")); err != nil { t.Fatal("the copy must stay on disk after a failed put-back") } } type starterFunc func(string) error func (f starterFunc) StartStack(n string) error { return f(n) } // R-1195's MEASURED case is the OFF-SITE full restore (rehearsal 2026-10-20 s2): the same cut, on that path — the // marker and the copy on disk at the replay are what the next start pours back. // RED-PROOF: without persistUndoForCrash in ReconstituteFromOffsite the marker holds no Undo → nothing poured back. func TestR1195_AKilledOffsiteRestoreIsPutBackOnStart(t *testing.T) { m, _, rig, _, _, _ := r893PutBackFixture(t) marker := filepath.Join(t.TempDir(), "appstop-state.json") m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0)) saved := map[string][]byte{} m.importDBDump = func(context.Context, DiscoveredDB, string) error { _ = filepath.Walk(rig.undoRoot, func(p string, fi os.FileInfo, _ error) error { if fi != nil && !fi.IsDir() { b, _ := os.ReadFile(p) saved[p] = b } return nil }) b, _ := os.ReadFile(marker) saved[marker] = b return nil } if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil { t.Fatalf("the run after the cut point: %v", err) } for p, b := range saved { _ = os.MkdirAll(filepath.Dir(p), 0o700) _ = os.WriteFile(p, b, 0o600) } poured := map[string]string{} cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0)) cp.volReplay = func(_ string, dir string) (int, error) { entries, _ := os.ReadDir(dir) n := 0 for _, e := range entries { if strings.HasSuffix(e.Name(), ".tar") { b, _ := os.ReadFile(filepath.Join(dir, e.Name())) poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b) n++ } } return n, nil } var removed []string cp.volRemove = func(v string) error { removed = append(removed, v); return nil } g := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) g.SetStarter(starterFunc(func(string) error { return nil })) g.SetPutBacker(cp) rec := g.Recover() if rec == nil || len(rec.PutBack) != 1 || poured[r893LiveVol] != "LIVE bytes of "+r893LiveVol { t.Fatalf("the cut off-site restore must be put back on start: rec %+v poured %v", rec, poured) } if len(removed) != 1 || removed[0] != r893NewVol { t.Fatalf("the snapshot's extra volume must be removed, removed %v", removed) } }