package backup import ( "context" "errors" "io" "log" "os" "path/filepath" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // R-1256 (review round 2, HUNT-1): four restore edges — H4, H5, H6 and H8, each pinned below. // // H4: an off-site restore writes the snapshot's definition before any data moves. The writer // (stackAdapter.RecreateStackDefinitionFromUnit) writes the compose FIRST and the app's settings after, so a // failure can come back with the snapshot's compose already in the stack dir. The failure branch used to start // the app at once — on the snapshot's (older) compose, over the live data. Now the live definition is written // back first; only then is the app started. When the write-back fails too, the app is held, never started. // r1256DefFailProvider fails the FIRST definition write (the snapshot's), after recording it — as the real // writer does when its compose write succeeded and its settings write did not. type r1256DefFailProvider struct { *r893Provider failWrites int // how many writes fail, from the first failFrom int // when set, every write from this one (1-based) on fails } func (p *r1256DefFailProvider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error { if err := p.r893Provider.RecreateStackDefinitionFromUnit(name, composeDir, env); err != nil { return err } if len(p.defs) <= p.failWrites || (p.failFrom > 0 && len(p.defs) >= p.failFrom) { return errors.New("app.yaml write failed") } return nil } func r1256DefFailFixture(t *testing.T, failWrites int) (*Manager, *r1256DefFailProvider, *int) { t.Helper() m, vp, _, notified, _ := r893FixtureA(t, true, 0) p := &r1256DefFailProvider{r893Provider: vp, failWrites: failWrites} m.SetStackProvider(p) return m, p, notified } // RED-PROOF: remove the live write-back from the definition-write failure branch (restartStack alone) → one // definition write (the snapshot's), then the start → FAILS. func TestR1256_H4_AFailedDefinitionWriteStartsTheAppAtItsLiveDefinition(t *testing.T) { m, p, notified := r1256DefFailFixture(t, 1) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed definition write must be surfaced") } if len(p.defs) != 2 || !strings.Contains(strings.Join(p.defs[1], " "), "postgres:18-alpine") { t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", p.defs) } if !p.fullStarted { t.Fatalf("the app must run again at its live definition — calls %v", p.calls) } // The write-back comes BEFORE the start. lastRecreate, lastStart := -1, -1 for i, c := range p.calls { switch c { case "recreate": lastRecreate = i case "start": lastStart = i } } if lastRecreate > lastStart { t.Fatalf("the app was started before the live definition was written back — calls %v", p.calls) } if held, _ := m.RestoreHoldFor("immich"); held { t.Fatal("the live definition is back: the app must not be held") } if *notified != 0 { t.Errorf("hold notices %d, want 0", *notified) } } // The write-back fails too: the stack dir may still hold the snapshot's compose, so the app is NOT started; it is // held (restore_mixed: the definition moved and could not be put back) and the operator is told once. // RED-PROOF: start the app when the write-back fails → fullStarted → FAILS. func TestR1256_H4_AFailedWriteBackHoldsTheApp(t *testing.T) { m, p, notified := r1256DefFailFixture(t, 2) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed definition write must be surfaced") } if p.fullStarted { t.Fatalf("the app was STARTED although its live definition could not be written back — calls %v", p.calls) } held, _ := m.RestoreHoldFor("immich") if !held { t.Fatal("no hold: every start path would start the app on the snapshot's definition") } if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed { t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed) } if *notified != 1 { t.Errorf("the operator must be told once, got %d", *notified) } } // ---- H5: a put-back whose stop fails must not go on; `volume rm -f` errors count; a failed put-back keeps its copy. // r1256StopFailProvider fails the Nth StopStack (1-based). type r1256StopFailProvider struct { *r893Provider failAt, n int } func (p *r1256StopFailProvider) StopStack(name string) error { p.n++ err := p.r893Provider.StopStack(name) if p.n == p.failAt { return errors.New("compose down: timeout") } return err } // The off-site put-back: the replay fails after the snapshot's volume replaced the live one; the stop before the // put-back fails. Pouring the copy back under a running app could mix it with what the app writes, so nothing is // poured, the copy is KEPT, and the app is held. // RED-PROOF: restore the „(continuing)" WARN in putBackAfterFailedRestore → the copy is poured and the app started // → FAILS. func TestR1256_H5_APutBackWhoseStopFailsDoesNotPourAndKeepsTheCopy(t *testing.T) { m, vp, rig, _, notified, putBack := r893PutBackFixture(t) p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's m.SetStackProvider(p) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed restore must be surfaced") } if p.n < 2 { t.Fatalf("the put-back's stop never ran (stops %d) — the fixture did not reach the put-back", p.n) } if len(rig.poured) != 0 || len(rig.removed) != 0 { t.Fatalf("the copy was poured (%v) / volumes removed (%v) after a FAILED stop", rig.poured, rig.removed) } if vp.fullStarted { t.Fatalf("the app was started after a failed put-back — calls %v", vp.calls) } if held, _ := m.RestoreHoldFor("immich"); !held { t.Fatal("a put-back that could not run must hold the app (D20)") } if left := undoCopies(t, rig.undoRoot); len(left) == 0 { t.Fatal("the copy must be KEPT after a failed put-back (D20)") } if *notified != 1 || *putBack != 0 { t.Errorf("hold notices %d (want 1), put-back notices %d (want 0)", *notified, *putBack) } } // The live definition cannot be written back after the volumes were poured: the app is held and the copy is KEPT // (D20: a put-back that fails keeps its copy for support). // RED-PROOF: drop `c.keep = true` from the definition write-back failure branch → the copy is removed → FAILS. func TestR1256_H5_AFailedDefinitionWriteBackKeepsTheCopy(t *testing.T) { m, vp, rig, _, _, _ := r893PutBackFixture(t) // A version change, so a live definition exists to be written back; the second write (the write-back) fails. if err := os.WriteFile(vp.composePath, []byte(vtCompose("18")), 0o644); err != nil { t.Fatal(err) } vp.livePins = ParseComposeImages(vp.composePath) p := &r1256DefFailProvider{r893Provider: vp, failWrites: 0} p.failFrom = 2 m.SetStackProvider(p) _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) if err == nil { t.Fatal("a failed restore must be surfaced") } if len(p.defs) < 2 { t.Fatalf("the write-back never ran: writes %v", p.defs) } if held, _ := m.RestoreHoldFor("immich"); !held { t.Fatal("a failed write-back must hold the app") } if left := undoCopies(t, rig.undoRoot); len(left) == 0 { t.Fatal("the copy must be KEPT after a failed put-back (D20)") } } // The crash put-back (a restore cut by a power loss): the stop before pouring fails → an error (the guard then // holds the app and keeps the copy, R-1251 H2), nothing poured. // RED-PROOF: restore the „(continuing)" WARN in PutBackInterruptedRestore → nil error, the copy poured → FAILS. func TestR1256_H5_ACrashPutBackWhoseStopFailsRefuses(t *testing.T) { dir := t.TempDir() undo := filepath.Join(dir, "backups", "restore-undo", "app", "20261028T000000Z") mustWrite(t, filepath.Join(undo, "undo.json"), `{"stack":"app","copied":["app_data"],"definition":false}`) mustWrite(t, filepath.Join(undo, "app_data.tar"), "LIVE") poured := 0 cp := NewCrashPutBack(func(string) error { return errors.New("compose down: timeout") }, nil, log.New(io.Discard, "", 0)) cp.volReplay = func(string, string) (int, error) { poured++; return 1, nil } if err := cp.PutBackInterruptedRestore("app", undo); err == nil { t.Fatal("a put-back whose stop failed must return an error") } if poured != 0 { t.Fatal("the copy was poured back under an app that could not be stopped") } if _, err := os.Stat(filepath.Join(undo, "app_data.tar")); err != nil { t.Fatal("the copy must stay on disk") } } // restoreDockerVolumesFrom: a `docker volume rm -f` that fails (the volume is still in use) used to be ignored; the // `volume create` that follows is a no-op on an existing volume, so the tar was unpacked ON TOP of the old content — // a merge. Now that volume counts as failed and is not touched. // RED-PROOF: ignore the rm error again → the create and the tar run for the volume → FAILS. func TestR1256_H5_AVolumeThatCannotBeRemovedIsNotMergedInto(t *testing.T) { bin := t.TempDir() calls := filepath.Join(bin, "calls.log") stub := "#!/bin/sh\necho \"$@\" >> " + calls + "\n" + "case \"$*\" in 'compose version --short') echo v2.29.7;; 'volume rm -f app_busy') echo 'volume is in use' >&2; exit 1;; esac\nexit 0\n" if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(stub), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) dump := t.TempDir() for _, n := range []string{"app_busy.tar", "app_free.tar"} { mustWrite(t, filepath.Join(dump, n), "tar") } m := &Manager{logger: log.New(io.Discard, "", 0)} n, err := m.restoreDockerVolumesFrom("app", dump) if err == nil || n != 1 { t.Fatalf("restored %d, err %v — want 1 and an error naming app_busy", n, err) } if !strings.Contains(err.Error(), "app_busy") { t.Errorf("the error must name the volume: %v", err) } b, _ := os.ReadFile(calls) for _, l := range strings.Split(string(b), "\n") { if (strings.HasPrefix(l, "volume create") || strings.HasPrefix(l, "run ")) && strings.Contains(l, "app_busy") { t.Fatalf("the volume that could not be removed was written into: %q", l) } } } // ---- H6: a second-drive mirror must not start between the page's refusal and the restore's own record. // The page asked (nothing ran), then a mirror began, then the restore began: the begin itself refuses, records // nothing, and the mirror — asked under the same lock — refuses once a restore is recorded. // RED-PROOF: make TryBeginRestoreOp record the op without asking (the old BeginRestoreOp) → the restore begins beside // the mirror → FAILS. func TestR1256_H6_ARestoreCannotBeginBesideAMirrorThatStartedAfterTheCheck(t *testing.T) { tier2ResetForTest() defer tier2ResetForTest() m, _ := newOffboxManager(t) if busy, _ := m.RestoreBusy(); busy != "" { t.Fatalf("fixture: nothing runs, got %q", busy) } // the window: the page's check passed; now a mirror begins if !m.beginTier2Run("immich") { t.Fatal("fixture: the mirror must begin while no restore is recorded") } if busy, app := m.TryBeginRestoreOp("restore", "immich"); busy != RestoreBusyMirror || app != "immich" { t.Fatalf("the restore began beside a running mirror: busy=%q app=%q", busy, app) } if m.RestoreStatus().Running { t.Fatal("a refused begin left a restore op recorded") } tier2CopyEnd("immich") if busy, _ := m.TryBeginRestoreOp("restore", "immich"); busy != "" { t.Fatalf("with the mirror done the restore must begin, got %q", busy) } if m.beginTier2Run("immich") { t.Fatal("a mirror began while the restore of the same app is recorded") } // a second restore is refused while the first runs; the backup run refuses too if busy, _ := m.TryBeginRestoreOp("restore", "other"); busy != RestoreBusyRestore { t.Fatalf("a second restore began beside the first: %q", busy) } m.EndRestoreOp(true, "ok") if err := m.AcquireRunningForTest(); err != nil { t.Fatal(err) } if busy, _ := m.TryBeginRestoreOp("restore", "immich"); busy != RestoreBusyBackup { t.Fatalf("a restore began beside a backup run: %q", busy) } m.ReleaseRunningForTest() } // ---- H8: an unused restore copy is cleaned up; a copy a hold or a pending put-back needs is never touched. func r1256Copy(t *testing.T, m *Manager, app string, age time.Duration, now time.Time) string { t.Helper() dir := filepath.Join(restoreUndoDir(m.namespaceRoot(m.systemDataPath), app), now.Add(-age).UTC().Format(restoreCopyStampLayout)) mustWrite(t, filepath.Join(dir, app+"_data.tar"), "LIVE") return dir } func r1256Exists(p string) bool { _, err := os.Stat(p); return err == nil } // RED-PROOF: drop the hold check (or the marker check) from SweepUnusedRestoreCopies → the held / named copy is removed // → FAILS; drop the age check → the young copy is removed → FAILS. func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) { m, sett := newOffboxManager(t) now := time.Now() old := 20 * 24 * time.Hour unused := r1256Copy(t, m, "unused", old, now) young := r1256Copy(t, m, "young", 24*time.Hour, now) held := r1256Copy(t, m, "held", old, now) waiting := r1256Copy(t, m, "waiting", old, now) named := r1256Copy(t, m, "named", old, now) if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "held", At: now.Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}); err != nil { t.Fatal(err) } if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "waiting", At: now.Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending}); err != nil { t.Fatal(err) } // a restore cut by a power loss: the marker (no longer active after a failed recovery's End) still names the copy if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil { t.Fatal(err) } if err := m.appStop.SetUndo("named", named); err != nil { t.Fatal(err) } got := m.SweepUnusedRestoreCopies(now) if len(got) != 1 || got[0] != unused || r1256Exists(unused) { t.Fatalf("removed %v — want exactly the unused old copy %s", got, unused) } for _, keep := range []string{young, held, waiting, named} { if !r1256Exists(keep) { t.Errorf("a needed copy was removed: %s", keep) } } if r1256Exists(filepath.Dir(unused)) { t.Error("the emptied per-app folder must go too") } } // While a restore runs nothing is removed; a marker that cannot be read stops the sweep. // RED-PROOF: drop the RestoreBusy check → the old copy goes during a restore → FAILS. func TestR1256_H8_NothingIsRemovedDuringARestoreOrWithAnUnreadableMarker(t *testing.T) { m, _ := newOffboxManager(t) now := time.Now() c := r1256Copy(t, m, "app", 20*24*time.Hour, now) m.BeginRestoreOp("restore", "other") if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) { t.Fatalf("removed %v during a restore", got) } m.EndRestoreOp(true, "ok") mustWrite(t, m.appStop.path, "{not json") if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) { t.Fatalf("removed %v with an unreadable marker", got) } if err := os.Remove(m.appStop.path); err != nil { t.Fatal(err) } if got := m.SweepUnusedRestoreCopies(now); len(got) != 1 { t.Fatalf("control: with nothing in the way the old copy goes, removed %v", got) } }