359 lines
15 KiB
Go
359 lines
15 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// R-1256 (review round 2, HUNT-1): four restore edges — H4, H5, H6 and H8, each pinned below.
|
|
//
|
|
// H4: an off-site restore writes the snapshot's definition before any data moves. The writer
|
|
// (stackAdapter.RecreateStackDefinitionFromUnit) writes the compose FIRST and the app's settings after, so a
|
|
// failure can come back with the snapshot's compose already in the stack dir. The failure branch used to start
|
|
// the app at once — on the snapshot's (older) compose, over the live data. Now the live definition is written
|
|
// back first; only then is the app started. When the write-back fails too, the app is held, never started.
|
|
|
|
// r1256DefFailProvider fails the FIRST definition write (the snapshot's), after recording it — as the real
|
|
// writer does when its compose write succeeded and its settings write did not.
|
|
type r1256DefFailProvider struct {
|
|
*r893Provider
|
|
failWrites int // how many writes fail, from the first
|
|
failFrom int // when set, every write from this one (1-based) on fails
|
|
}
|
|
|
|
func (p *r1256DefFailProvider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error {
|
|
if err := p.r893Provider.RecreateStackDefinitionFromUnit(name, composeDir, env); err != nil {
|
|
return err
|
|
}
|
|
if len(p.defs) <= p.failWrites || (p.failFrom > 0 && len(p.defs) >= p.failFrom) {
|
|
return errors.New("app.yaml write failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func r1256DefFailFixture(t *testing.T, failWrites int) (*Manager, *r1256DefFailProvider, *int) {
|
|
t.Helper()
|
|
m, vp, _, notified, _ := r893FixtureA(t, true, 0)
|
|
p := &r1256DefFailProvider{r893Provider: vp, failWrites: failWrites}
|
|
m.SetStackProvider(p)
|
|
return m, p, notified
|
|
}
|
|
|
|
// RED-PROOF: remove the live write-back from the definition-write failure branch (restartStack alone) → one
|
|
// definition write (the snapshot's), then the start → FAILS.
|
|
func TestR1256_H4_AFailedDefinitionWriteStartsTheAppAtItsLiveDefinition(t *testing.T) {
|
|
m, p, notified := r1256DefFailFixture(t, 1)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed definition write must be surfaced")
|
|
}
|
|
if len(p.defs) != 2 || !strings.Contains(strings.Join(p.defs[1], " "), "postgres:18-alpine") {
|
|
t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", p.defs)
|
|
}
|
|
if !p.fullStarted {
|
|
t.Fatalf("the app must run again at its live definition — calls %v", p.calls)
|
|
}
|
|
// The write-back comes BEFORE the start.
|
|
lastRecreate, lastStart := -1, -1
|
|
for i, c := range p.calls {
|
|
switch c {
|
|
case "recreate":
|
|
lastRecreate = i
|
|
case "start":
|
|
lastStart = i
|
|
}
|
|
}
|
|
if lastRecreate > lastStart {
|
|
t.Fatalf("the app was started before the live definition was written back — calls %v", p.calls)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); held {
|
|
t.Fatal("the live definition is back: the app must not be held")
|
|
}
|
|
if *notified != 0 {
|
|
t.Errorf("hold notices %d, want 0", *notified)
|
|
}
|
|
}
|
|
|
|
// The write-back fails too: the stack dir may still hold the snapshot's compose, so the app is NOT started; it is
|
|
// held (restore_mixed: the definition moved and could not be put back) and the operator is told once.
|
|
// RED-PROOF: start the app when the write-back fails → fullStarted → FAILS.
|
|
func TestR1256_H4_AFailedWriteBackHoldsTheApp(t *testing.T) {
|
|
m, p, notified := r1256DefFailFixture(t, 2)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed definition write must be surfaced")
|
|
}
|
|
if p.fullStarted {
|
|
t.Fatalf("the app was STARTED although its live definition could not be written back — calls %v", p.calls)
|
|
}
|
|
held, _ := m.RestoreHoldFor("immich")
|
|
if !held {
|
|
t.Fatal("no hold: every start path would start the app on the snapshot's definition")
|
|
}
|
|
if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed {
|
|
t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed)
|
|
}
|
|
if *notified != 1 {
|
|
t.Errorf("the operator must be told once, got %d", *notified)
|
|
}
|
|
}
|
|
|
|
// ---- H5: a put-back whose stop fails must not go on; `volume rm -f` errors count; a failed put-back keeps its copy.
|
|
|
|
// r1256StopFailProvider fails the Nth StopStack (1-based).
|
|
type r1256StopFailProvider struct {
|
|
*r893Provider
|
|
failAt, n int
|
|
}
|
|
|
|
func (p *r1256StopFailProvider) StopStack(name string) error {
|
|
p.n++
|
|
err := p.r893Provider.StopStack(name)
|
|
if p.n == p.failAt {
|
|
return errors.New("compose down: timeout")
|
|
}
|
|
return err
|
|
}
|
|
|
|
// The off-site put-back: the replay fails after the snapshot's volume replaced the live one; the stop before the
|
|
// put-back fails. Pouring the copy back under a running app could mix it with what the app writes, so nothing is
|
|
// poured, the copy is KEPT, and the app is held.
|
|
// RED-PROOF: restore the „(continuing)" WARN in putBackAfterFailedRestore → the copy is poured and the app started
|
|
// → FAILS.
|
|
func TestR1256_H5_APutBackWhoseStopFailsDoesNotPourAndKeepsTheCopy(t *testing.T) {
|
|
m, vp, rig, _, notified, putBack := r893PutBackFixture(t)
|
|
p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
|
|
m.SetStackProvider(p)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed restore must be surfaced")
|
|
}
|
|
if p.n < 2 {
|
|
t.Fatalf("the put-back's stop never ran (stops %d) — the fixture did not reach the put-back", p.n)
|
|
}
|
|
if len(rig.poured) != 0 || len(rig.removed) != 0 {
|
|
t.Fatalf("the copy was poured (%v) / volumes removed (%v) after a FAILED stop", rig.poured, rig.removed)
|
|
}
|
|
if vp.fullStarted {
|
|
t.Fatalf("the app was started after a failed put-back — calls %v", vp.calls)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); !held {
|
|
t.Fatal("a put-back that could not run must hold the app (D20)")
|
|
}
|
|
if left := undoCopies(t, rig.undoRoot); len(left) == 0 {
|
|
t.Fatal("the copy must be KEPT after a failed put-back (D20)")
|
|
}
|
|
if *notified != 1 || *putBack != 0 {
|
|
t.Errorf("hold notices %d (want 1), put-back notices %d (want 0)", *notified, *putBack)
|
|
}
|
|
}
|
|
|
|
// The live definition cannot be written back after the volumes were poured: the app is held and the copy is KEPT
|
|
// (D20: a put-back that fails keeps its copy for support).
|
|
// RED-PROOF: drop `c.keep = true` from the definition write-back failure branch → the copy is removed → FAILS.
|
|
func TestR1256_H5_AFailedDefinitionWriteBackKeepsTheCopy(t *testing.T) {
|
|
m, vp, rig, _, _, _ := r893PutBackFixture(t)
|
|
// A version change, so a live definition exists to be written back; the second write (the write-back) fails.
|
|
if err := os.WriteFile(vp.composePath, []byte(vtCompose("18")), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
vp.livePins = ParseComposeImages(vp.composePath)
|
|
p := &r1256DefFailProvider{r893Provider: vp, failWrites: 0}
|
|
p.failFrom = 2
|
|
m.SetStackProvider(p)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if err == nil {
|
|
t.Fatal("a failed restore must be surfaced")
|
|
}
|
|
if len(p.defs) < 2 {
|
|
t.Fatalf("the write-back never ran: writes %v", p.defs)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); !held {
|
|
t.Fatal("a failed write-back must hold the app")
|
|
}
|
|
if left := undoCopies(t, rig.undoRoot); len(left) == 0 {
|
|
t.Fatal("the copy must be KEPT after a failed put-back (D20)")
|
|
}
|
|
}
|
|
|
|
// The crash put-back (a restore cut by a power loss): the stop before pouring fails → an error (the guard then
|
|
// holds the app and keeps the copy, R-1251 H2), nothing poured.
|
|
// RED-PROOF: restore the „(continuing)" WARN in PutBackInterruptedRestore → nil error, the copy poured → FAILS.
|
|
func TestR1256_H5_ACrashPutBackWhoseStopFailsRefuses(t *testing.T) {
|
|
dir := t.TempDir()
|
|
undo := filepath.Join(dir, "backups", "restore-undo", "app", "20261028T000000Z")
|
|
mustWrite(t, filepath.Join(undo, "undo.json"), `{"stack":"app","copied":["app_data"],"definition":false}`)
|
|
mustWrite(t, filepath.Join(undo, "app_data.tar"), "LIVE")
|
|
poured := 0
|
|
cp := NewCrashPutBack(func(string) error { return errors.New("compose down: timeout") }, nil, log.New(io.Discard, "", 0))
|
|
cp.volReplay = func(string, string) (int, error) { poured++; return 1, nil }
|
|
if err := cp.PutBackInterruptedRestore("app", undo); err == nil {
|
|
t.Fatal("a put-back whose stop failed must return an error")
|
|
}
|
|
if poured != 0 {
|
|
t.Fatal("the copy was poured back under an app that could not be stopped")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(undo, "app_data.tar")); err != nil {
|
|
t.Fatal("the copy must stay on disk")
|
|
}
|
|
}
|
|
|
|
// restoreDockerVolumesFrom: a `docker volume rm -f` that fails (the volume is still in use) used to be ignored; the
|
|
// `volume create` that follows is a no-op on an existing volume, so the tar was unpacked ON TOP of the old content —
|
|
// a merge. Now that volume counts as failed and is not touched.
|
|
// RED-PROOF: ignore the rm error again → the create and the tar run for the volume → FAILS.
|
|
func TestR1256_H5_AVolumeThatCannotBeRemovedIsNotMergedInto(t *testing.T) {
|
|
bin := t.TempDir()
|
|
calls := filepath.Join(bin, "calls.log")
|
|
stub := "#!/bin/sh\necho \"$@\" >> " + calls + "\n" +
|
|
"case \"$*\" in 'compose version --short') echo v2.29.7;; 'volume rm -f app_busy') echo 'volume is in use' >&2; exit 1;; esac\nexit 0\n"
|
|
if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(stub), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
dump := t.TempDir()
|
|
for _, n := range []string{"app_busy.tar", "app_free.tar"} {
|
|
mustWrite(t, filepath.Join(dump, n), "tar")
|
|
}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0)}
|
|
n, err := m.restoreDockerVolumesFrom("app", dump)
|
|
if err == nil || n != 1 {
|
|
t.Fatalf("restored %d, err %v — want 1 and an error naming app_busy", n, err)
|
|
}
|
|
if !strings.Contains(err.Error(), "app_busy") {
|
|
t.Errorf("the error must name the volume: %v", err)
|
|
}
|
|
b, _ := os.ReadFile(calls)
|
|
for _, l := range strings.Split(string(b), "\n") {
|
|
if (strings.HasPrefix(l, "volume create") || strings.HasPrefix(l, "run ")) && strings.Contains(l, "app_busy") {
|
|
t.Fatalf("the volume that could not be removed was written into: %q", l)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- H6: a second-drive mirror must not start between the page's refusal and the restore's own record.
|
|
|
|
// The page asked (nothing ran), then a mirror began, then the restore began: the begin itself refuses, records
|
|
// nothing, and the mirror — asked under the same lock — refuses once a restore is recorded.
|
|
// RED-PROOF: make TryBeginRestoreOp record the op without asking (the old BeginRestoreOp) → the restore begins beside
|
|
// the mirror → FAILS.
|
|
func TestR1256_H6_ARestoreCannotBeginBesideAMirrorThatStartedAfterTheCheck(t *testing.T) {
|
|
tier2ResetForTest()
|
|
defer tier2ResetForTest()
|
|
m, _ := newOffboxManager(t)
|
|
if busy, _ := m.RestoreBusy(); busy != "" {
|
|
t.Fatalf("fixture: nothing runs, got %q", busy)
|
|
}
|
|
// the window: the page's check passed; now a mirror begins
|
|
if !m.beginTier2Run("immich") {
|
|
t.Fatal("fixture: the mirror must begin while no restore is recorded")
|
|
}
|
|
if busy, app := m.TryBeginRestoreOp("restore", "immich"); busy != RestoreBusyMirror || app != "immich" {
|
|
t.Fatalf("the restore began beside a running mirror: busy=%q app=%q", busy, app)
|
|
}
|
|
if m.RestoreStatus().Running {
|
|
t.Fatal("a refused begin left a restore op recorded")
|
|
}
|
|
tier2CopyEnd("immich")
|
|
if busy, _ := m.TryBeginRestoreOp("restore", "immich"); busy != "" {
|
|
t.Fatalf("with the mirror done the restore must begin, got %q", busy)
|
|
}
|
|
if m.beginTier2Run("immich") {
|
|
t.Fatal("a mirror began while the restore of the same app is recorded")
|
|
}
|
|
// a second restore is refused while the first runs; the backup run refuses too
|
|
if busy, _ := m.TryBeginRestoreOp("restore", "other"); busy != RestoreBusyRestore {
|
|
t.Fatalf("a second restore began beside the first: %q", busy)
|
|
}
|
|
m.EndRestoreOp(true, "ok")
|
|
if err := m.AcquireRunningForTest(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if busy, _ := m.TryBeginRestoreOp("restore", "immich"); busy != RestoreBusyBackup {
|
|
t.Fatalf("a restore began beside a backup run: %q", busy)
|
|
}
|
|
m.ReleaseRunningForTest()
|
|
}
|
|
|
|
// ---- H8: an unused restore copy is cleaned up; a copy a hold or a pending put-back needs is never touched.
|
|
|
|
func r1256Copy(t *testing.T, m *Manager, app string, age time.Duration, now time.Time) string {
|
|
t.Helper()
|
|
dir := filepath.Join(restoreUndoDir(m.namespaceRoot(m.systemDataPath), app), now.Add(-age).UTC().Format(restoreCopyStampLayout))
|
|
mustWrite(t, filepath.Join(dir, app+"_data.tar"), "LIVE")
|
|
return dir
|
|
}
|
|
|
|
func r1256Exists(p string) bool { _, err := os.Stat(p); return err == nil }
|
|
|
|
// RED-PROOF: drop the hold check (or the marker check) from SweepUnusedRestoreCopies → the held / named copy is removed
|
|
// → FAILS; drop the age check → the young copy is removed → FAILS.
|
|
func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) {
|
|
m, sett := newOffboxManager(t)
|
|
now := time.Now()
|
|
old := 40 * 24 * time.Hour
|
|
unused := r1256Copy(t, m, "unused", old, now)
|
|
young := r1256Copy(t, m, "young", 24*time.Hour, now)
|
|
held := r1256Copy(t, m, "held", old, now)
|
|
waiting := r1256Copy(t, m, "waiting", old, now)
|
|
named := r1256Copy(t, m, "named", old, now)
|
|
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "held", At: now.Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "waiting", At: now.Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// a restore cut by a power loss: the marker (no longer active after a failed recovery's End) still names the copy
|
|
if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.appStop.SetUndo("named", named); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got := m.SweepUnusedRestoreCopies(now)
|
|
if len(got) != 1 || got[0] != unused || r1256Exists(unused) {
|
|
t.Fatalf("removed %v — want exactly the unused old copy %s", got, unused)
|
|
}
|
|
for _, keep := range []string{young, held, waiting, named} {
|
|
if !r1256Exists(keep) {
|
|
t.Errorf("a needed copy was removed: %s", keep)
|
|
}
|
|
}
|
|
if r1256Exists(filepath.Dir(unused)) {
|
|
t.Error("the emptied per-app folder must go too")
|
|
}
|
|
}
|
|
|
|
// While a restore runs nothing is removed; a marker that cannot be read stops the sweep.
|
|
// RED-PROOF: drop the RestoreBusy check → the old copy goes during a restore → FAILS.
|
|
func TestR1256_H8_NothingIsRemovedDuringARestoreOrWithAnUnreadableMarker(t *testing.T) {
|
|
m, _ := newOffboxManager(t)
|
|
now := time.Now()
|
|
c := r1256Copy(t, m, "app", 40*24*time.Hour, now)
|
|
m.BeginRestoreOp("restore", "other")
|
|
if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) {
|
|
t.Fatalf("removed %v during a restore", got)
|
|
}
|
|
m.EndRestoreOp(true, "ok")
|
|
mustWrite(t, m.appStop.path, "{not json")
|
|
if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) {
|
|
t.Fatalf("removed %v with an unreadable marker", got)
|
|
}
|
|
if err := os.Remove(m.appStop.path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := m.SweepUnusedRestoreCopies(now); len(got) != 1 {
|
|
t.Fatalf("control: with nothing in the way the old copy goes, removed %v", got)
|
|
}
|
|
}
|