D49 = A follow-ups: (A) the undo copy is released only after a stable start — 3 min of running, healthy, no restart, one rule for the first verdict and the late-start watch; a watch ends only for a restore begun after it; (B, R-1279) the .fab import downloads first with the restores' D48 helper, before it removes anything. Red-proved. Commit only.

controller/internal/backup/stable_start_test.go

@@ -0,0 +1,147 @@
+package backup
+
+import (
+	"io"
+	"log"
+	"sync"
+	"testing"
+	"time"
+)
+
+// Fix A (brief 2026-10-26; R-1270 follow-up) — the undo copy is released only after a STABLE start.
+// COMPANION RED-PROOF (REPORT.md): on `e8d2151` (the copy dropped at the restore's return / at the watcher's first
+// running sample) TestFixA_AFlappingAppKeepsTheCopy and TestFixA_AHealthcheckStillStartingKeepsTheCopy fail.
+
+// sampledProvider is the D22 rig's provider with a scripted container sample per call.
+type sampledProvider struct {
+	*r1177Provider
+	mu      sync.Mutex
+	n       int
+	running func(n int) bool
+	sample  func(n int) []ContainerSample
+}
+
+func (p *sampledProvider) RefreshAndIsRunning(string) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.n++
+	return p.running(p.n)
+}
+
+func (p *sampledProvider) ContainerSamples(string) ([]ContainerSample, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.sample(p.n), nil
+}
+
+func fixARig(t *testing.T, running func(int) bool, sample func(int) []ContainerSample) (*Manager, *r1177Rig, *[]string) {
+	t.Helper()
+	m, p, rig, _ := r1177Fixture(t)
+	sp := &sampledProvider{r1177Provider: p, running: running, sample: sample}
+	m.stackProvider = sp
+	var alarms []string
+	var mu sync.Mutex
+	m.SetRestoreNotStartedNotify(func(stack, kind string, _ error) { mu.Lock(); alarms = append(alarms, kind); mu.Unlock() })
+	return m, rig, &alarms
+}
+
+func withStable(t *testing.T, window, watch time.Duration) {
+	t.Helper()
+	w0, p0, r0 := stableStartWindow, stableStartPoll, stableReleaseWatch
+	stableStartWindow, stableStartPoll, stableReleaseWatch = window, 2*time.Millisecond, watch
+	t.Cleanup(func() { stableStartWindow, stableStartPoll, stableReleaseWatch = w0, p0, r0 })
+}
+
+func healthy(id string, restarts int) ContainerSample {
+	return ContainerSample{ID: id, Running: true, RestartCount: restarts, Health: "healthy"}
+}
+
+// running → restarting → running: the copy stays, and the operator hears once that the start was not stable.
+func TestFixA_AFlappingAppKeepsTheCopy(t *testing.T) {
+	withStable(t, 60*time.Millisecond, 400*time.Millisecond)
+	m, rig, alarms := fixARig(t,
+		func(n int) bool { return n%3 != 0 },                                           // every third sample: restarting
+		func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }) // and the count grows
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatalf("the restore itself passes its 90 s health wait (a running sample): %v", err)
+	}
+	time.Sleep(600 * time.Millisecond)
+	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
+		t.Fatalf("a flapping app must keep its way back, found %v", left)
+	}
+	if len(*alarms) != 1 {
+		t.Fatalf("the operator hears once that the start was not stable, got %v", *alarms)
+	}
+}
+
+// A stable stack: the copy is still there right after the restore, and gone after the window.
+func TestFixA_AStableAppReleasesTheCopyAfterTheWindow(t *testing.T) {
+	withStable(t, 150*time.Millisecond, 3*time.Second)
+	m, rig, alarms := fixARig(t, func(int) bool { return true },
+		func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0), {ID: "db", Running: true}} })
+	// The production order: the handler begins the record, the restore runs (and starts the watch), the handler ends it.
+	m.BeginRestoreOp("restore", "app")
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	m.EndRestoreOp(true, "ok")
+	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
+		t.Fatalf("the copy is NOT released at the restore's return any more, found %v", left)
+	}
+	if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 {
+		t.Fatalf("a stable app releases the copy after the window, found %v", left)
+	}
+	if len(*alarms) != 0 {
+		t.Fatalf("no alarm for a stable app: %v", *alarms)
+	}
+}
+
+// A container whose healthcheck still says `starting` is not started.
+func TestFixA_AHealthcheckStillStartingKeepsTheCopy(t *testing.T) {
+	withStable(t, 30*time.Millisecond, 300*time.Millisecond)
+	m, rig, _ := fixARig(t, func(int) bool { return true },
+		func(int) []ContainerSample { return []ContainerSample{{ID: "c1", Running: true, Health: "starting"}} })
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(500 * time.Millisecond)
+	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
+		t.Fatalf("a healthcheck still starting must keep the copy, found %v", left)
+	}
+}
+
+// A new restore of the app during the watch ends it untouched (unchanged rule).
+func TestFixA_ANewRestoreDuringTheWatchLeavesTheCopy(t *testing.T) {
+	withStable(t, 400*time.Millisecond, 2*time.Second)
+	m, rig, alarms := fixARig(t, func(int) bool { return true },
+		func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	m.BeginRestoreOp("restore", "app")
+	time.Sleep(700 * time.Millisecond)
+	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
+		t.Fatalf("a new restore owns the app: the watch ends and the copy stays, found %v", left)
+	}
+	if len(*alarms) != 0 {
+		t.Fatalf("an aborted watch raises no alarm: %v", *alarms)
+	}
+}
+
+// The late-start watcher (after a not-started verdict) uses the same rule: a flapping app never becomes a success.
+func TestFixA_TheLateStartWatcherNeedsAStableStart(t *testing.T) {
+	withStable(t, 60*time.Millisecond, 2*time.Second)
+	lateStartWatch = 400 * time.Millisecond
+	defer func() { lateStartWatch = 30 * time.Minute }()
+	m := &Manager{logger: log.New(io.Discard, "", 0)}
+	sp := &sampledProvider{r1177Provider: &r1177Provider{fakeRecoveryProvider: &fakeRecoveryProvider{}},
+		running: func(n int) bool { return n%3 != 0 },
+		sample:  func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }}
+	m.stackProvider = sp
+	m.BeginRestoreOp("restore", "app")
+	m.EndRestoreOpNotStarted("not yet")
+	m.watchLateStart("app", "", time.Now(), lateStartWatch)
+	if st := m.RestoreStatus(); st.Last.OK {
+		t.Fatal("a flapping app must never turn the record into a success")
+	}
+}