package backup import ( "errors" "io" "log" "os" "path/filepath" "strings" "sync" "testing" "time" ) // Fix A (brief 2026-10-26; R-1270 follow-up) — the undo copy is released only after a STABLE start. // COMPANION RED-PROOF (REPORT.md): on `e8d2151` (the copy dropped at the restore's return / at the watcher's first // running sample) TestFixA_AFlappingAppKeepsTheCopy and TestFixA_AHealthcheckStillStartingKeepsTheCopy fail. // sampledProvider is the D22 rig's provider with a scripted container sample per call. type sampledProvider struct { *r1177Provider mu sync.Mutex n int running func(n int) bool sample func(n int) []ContainerSample } func (p *sampledProvider) RefreshAndIsRunning(string) bool { p.mu.Lock() defer p.mu.Unlock() p.n++ return p.running(p.n) } func (p *sampledProvider) ContainerSamples(string) ([]ContainerSample, error) { p.mu.Lock() defer p.mu.Unlock() return p.sample(p.n), nil } func fixARig(t *testing.T, running func(int) bool, sample func(int) []ContainerSample) (*Manager, *r1177Rig, *[]string) { t.Helper() m, p, rig, _ := r1177Fixture(t) sp := &sampledProvider{r1177Provider: p, running: running, sample: sample} m.stackProvider = sp var alarms []string var mu sync.Mutex m.SetRestoreNotStartedNotify(func(stack, kind string, _ error) { mu.Lock(); alarms = append(alarms, kind); mu.Unlock() }) return m, rig, &alarms } func withStable(t *testing.T, window, watch time.Duration) { t.Helper() w0, p0, r0 := stableStartWindow, stableStartPoll, stableReleaseWatch stableStartWindow, stableStartPoll, stableReleaseWatch = window, 2*time.Millisecond, watch t.Cleanup(func() { stableStartWindow, stableStartPoll, stableReleaseWatch = w0, p0, r0 }) } func healthy(id string, restarts int) ContainerSample { return ContainerSample{ID: id, Running: true, RestartCount: restarts, Health: "healthy"} } // running → restarting → running: the copy stays, and the operator hears once that the start was not stable. func TestFixA_AFlappingAppKeepsTheCopy(t *testing.T) { withStable(t, 60*time.Millisecond, 400*time.Millisecond) m, rig, alarms := fixARig(t, func(n int) bool { return n%3 != 0 }, // every third sample: restarting func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }) // and the count grows lateStartWatch = 400 * time.Millisecond defer func() { lateStartWatch = 30 * time.Minute }() m.BeginRestoreOp("restore", "app") _, err := m.RestoreFromRecoveryUnit("app") if !errors.Is(err, ErrRestoreNotStarted) { t.Fatalf("a running → restarting → running app is „not started yet\" on the page too, got %v", err) } m.EndRestoreOpNotStarted(err.Error()) time.Sleep(600 * time.Millisecond) if left := undoCopies(t, rig.undoRoot); len(left) != 1 { t.Fatalf("a flapping app must keep its way back, found %v", left) } if len(*alarms) != 1 { t.Fatalf("the operator hears once, got %v", *alarms) } if st := m.RestoreStatus(); st.Last.OK { t.Fatal("the late-start watch must never turn a flapping app into a success") } } // A stable stack: the copy is still there right after the restore, and gone after the window. func TestFixA_AStableAppReleasesTheCopyAfterTheWindow(t *testing.T) { withStable(t, 150*time.Millisecond, 3*time.Second) m, rig, alarms := fixARig(t, func(int) bool { return true }, func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0), {ID: "db", Running: true}} }) // The production order: the handler begins the record, the restore runs (and starts the watch), the handler ends it. m.BeginRestoreOp("restore", "app") if _, err := m.RestoreFromRecoveryUnit("app"); err != nil { t.Fatal(err) } m.EndRestoreOp(true, "ok") if left := undoCopies(t, rig.undoRoot); len(left) != 1 { t.Fatalf("the copy is NOT released at the restore's return any more, found %v", left) } if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 { t.Fatalf("a stable app releases the copy after the window, found %v", left) } if len(*alarms) != 0 { t.Fatalf("no alarm for a stable app: %v", *alarms) } } // A container whose healthcheck still says `starting` is not started. func TestFixA_AHealthcheckStillStartingKeepsTheCopy(t *testing.T) { withStable(t, 30*time.Millisecond, 300*time.Millisecond) m, rig, _ := fixARig(t, func(int) bool { return true }, func(int) []ContainerSample { return []ContainerSample{{ID: "c1", Running: true, Health: "starting"}} }) if _, err := m.RestoreFromRecoveryUnit("app"); !errors.Is(err, ErrRestoreNotStarted) { t.Fatalf("a healthcheck still starting is not started, got %v", err) } time.Sleep(500 * time.Millisecond) if left := undoCopies(t, rig.undoRoot); len(left) != 1 { t.Fatalf("a healthcheck still starting must keep the copy, found %v", left) } } // A new restore of the app during the watch ends it untouched (unchanged rule). func TestFixA_ANewRestoreDuringTheWatchLeavesTheCopy(t *testing.T) { withStable(t, 400*time.Millisecond, 2*time.Second) m, rig, alarms := fixARig(t, func(int) bool { return true }, func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} }) if _, err := m.RestoreFromRecoveryUnit("app"); err != nil { t.Fatal(err) } m.BeginRestoreOp("restore", "app") time.Sleep(700 * time.Millisecond) if left := undoCopies(t, rig.undoRoot); len(left) != 1 { t.Fatalf("a new restore owns the app: the watch ends and the copy stays, found %v", left) } if len(*alarms) != 0 { t.Fatalf("an aborted watch raises no alarm: %v", *alarms) } } // The late-start watcher (after a not-started verdict) uses the same rule: a flapping app never becomes a success. func TestFixA_TheLateStartWatcherNeedsAStableStart(t *testing.T) { withStable(t, 60*time.Millisecond, 2*time.Second) lateStartWatch = 400 * time.Millisecond defer func() { lateStartWatch = 30 * time.Minute }() m := &Manager{logger: log.New(io.Discard, "", 0)} sp := &sampledProvider{r1177Provider: &r1177Provider{fakeRecoveryProvider: &fakeRecoveryProvider{}}, running: func(n int) bool { return n%3 != 0 }, sample: func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }} m.stackProvider = sp m.BeginRestoreOp("restore", "app") m.EndRestoreOpNotStarted("not yet") m.watchLateStart("app", "", time.Now(), lateStartWatch) if st := m.RestoreStatus(); st.Last.OK { t.Fatal("a flapping app must never turn the record into a success") } } // ShortenRestoreTimingForTests must never reach production: only test files may call it. func TestFixA_TheTestTimingSwitchIsNeverCalledByProduction(t *testing.T) { root := filepath.Join("..", "..") _ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error { if err != nil || fi.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || strings.HasSuffix(p, "restore_verdict.go") { return nil } if b, _ := os.ReadFile(p); strings.Contains(string(b), "ShortenRestoreTimingForTests") { t.Errorf("%s calls the test-only timing switch", p) } return nil }) }