D49 = A follow-ups: (A) the undo copy is released only after a stable start — 3 min of running, healthy, no restart, one rule for the first verdict and the late-start watch; a watch ends only for a restore begun after it; (B, R-1279) the .fab import downloads first with the restores' D48 helper, before it removes anything. Red-proved. Commit only.

2026-10-25 12:45:33 +01:00
parent e8d2151a77
commit 7720439440
16 changed files with 549 additions and 27 deletions
@@ -2,6 +2,7 @@ package backup

 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -55,7 +56,7 @@ func (m *Manager) restoreNotStarted(stack, kind string, healthErr error, undo *l
 	if m.restoreNotStartedNotify != nil {
 		m.restoreNotStartedNotify(stack, kind, healthErr)
 	}
-	go m.watchLateStart(stack, copyDir, time.Now())
+	go m.watchLateStart(stack, copyDir, time.Now(), lateStartWatch)
 	return util.MsgErrorf(ErrRestoreNotStarted, "err.backup.restore_not_started", stack)
 }

@@ -66,28 +67,166 @@ func orNone(s string) string {
 	return s
 }

-// watchLateStart waits for the app after a not-started verdict. When it runs: the record says so and the kept copy is
-// removed (the normal rule after a successful restore). A new restore of the app ends the watch untouched.
-func (m *Manager) watchLateStart(stack, copyDir string, since time.Time) {
-	deadline := since.Add(lateStartWatch)
-	for time.Now().Before(deadline) {
-		time.Sleep(lateStartPoll)
-		if m.restoreRunningFor(stack) {
-			return // a new restore of this app owns it now
-		}
-		if m.stackProvider == nil || !m.stackProvider.RefreshAndIsRunning(stack) {
-			continue
-		}
-		if copyDir != "" {
-			if err := os.RemoveAll(copyDir); err != nil {
-				m.logger.Printf("[WARN] [backup] %s: started late; the kept restore copy %s could not be removed: %v", stack, copyDir, err)
-			} else {
-				_ = os.Remove(filepath.Dir(copyDir))
-			}
-		}
-		m.markLateStart(stack, m.note("note.restore.late_started", stack))
-		m.logger.Printf("[INFO] [backup] %s: the app started %s after its restore — the restore is a success now; the restore copy is released (R-1270)", stack, time.Since(since).Round(time.Second))
+// watchLateStart waits for the app after a not-started verdict. When it has run STABLY (stableStart): the record says
+// so and the kept copy is removed (the normal rule after a successful restore). A new restore of the app ends the watch
+// untouched.
+func (m *Manager) watchLateStart(stack, copyDir string, since time.Time, watch time.Duration) {
+	switch m.waitStableStart(stack, since, since.Add(watch)) {
+	case stableAborted:
+		return // a new restore of this app owns it now
+	case stableNot:
+		m.logger.Printf("[WARN] [backup] %s: still not running stably %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir))
 		return
 	}
-	m.logger.Printf("[WARN] [backup] %s: still not running %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir))
+	removeCopyDir(m, stack, copyDir)
+	m.markLateStart(stack, m.note("note.restore.late_started", stack))
+	m.logger.Printf("[INFO] [backup] %s: the app started %s after its restore and ran stably for %s — the restore is a success now; the restore copy is released (R-1270)", stack, time.Since(since).Round(time.Second), stableStartWindow)
+}
+
+func removeCopyDir(m *Manager, stack, copyDir string) {
+	if copyDir == "" {
+		return
+	}
+	if err := os.RemoveAll(copyDir); err != nil {
+		m.logger.Printf("[WARN] [backup] %s: the restore copy %s could not be removed: %v", stack, copyDir, err)
+		return
+	}
+	_ = os.Remove(filepath.Dir(copyDir))
+}
+
+// ── R-1270 follow-up (brief 2026-10-26, fix A): the undo copy goes only after a STABLE start ─────────────────────────
+//
+// MEASURED (code at `e8d2151`): the watcher deleted the undo copy on the FIRST sample in which the stack read
+// „running", and a restore whose health wait passed (one running sample, waitForHealthy) dropped it at once. An app
+// that crashes and restarts reads „running" for a few seconds between crashes, so its way back could go while it was
+// still broken.
+//
+// THE RULE (one helper for both places). „Started" means: for stableStartWindow, every sample (each stableStartPoll)
+// has the stack running, every container with a healthcheck `healthy`, the same containers, and no container's
+// RestartCount grown. The first verdict may still say success at once (the page; the 90 s health wait is unchanged) —
+// the copy is RELEASED only after the window (releaseAfterStableStart). The verdict and the release are therefore two
+// moments; `07` §6.3 says so. The window: 3 minutes — Docker's restart back-off grows to one minute, so a container
+// that crashes after up to a minute of running restarts at least once inside it, and RestartCount catches every restart
+// whatever its timing; it costs the household nothing (the app runs meanwhile), only the copy's disk for 3 minutes more.
+
+var (
+	stableStartWindow  = 3 * time.Minute
+	stableStartPoll    = 10 * time.Second
+	stableReleaseWatch = 30 * time.Minute
+)
+
+// ContainerSample is one container of a stack as a stable-start sample sees it.
+type ContainerSample struct {
+	ID           string
+	Running      bool
+	RestartCount int
+	Health       string // "" when the container has no healthcheck
+}
+
+// containerSampler is the optional provider half that reads the stack's containers (restart counts, health). A provider
+// without it is judged on RefreshAndIsRunning alone.
+type containerSampler interface {
+	ContainerSamples(name string) ([]ContainerSample, error)
+}
+
+type stableResult int
+
+const (
+	stableYes stableResult = iota
+	stableNot
+	stableAborted
+)
+
+// stableSample is one look: ok, and the fingerprint (container id → restart count) a stable window must keep.
+func (m *Manager) stableSample(stack string) (bool, map[string]int) {
+	if m.stackProvider == nil || !m.stackProvider.RefreshAndIsRunning(stack) {
+		return false, nil
+	}
+	cs, ok := m.stackProvider.(containerSampler)
+	if !ok {
+		return true, map[string]int{}
+	}
+	samples, err := cs.ContainerSamples(stack)
+	if err != nil || len(samples) == 0 {
+		return false, nil
+	}
+	fp := make(map[string]int, len(samples))
+	for _, c := range samples {
+		if !c.Running || (c.Health != "" && c.Health != "healthy") {
+			return false, nil
+		}
+		fp[c.ID] = c.RestartCount
+	}
+	return true, fp
+}
+
+func sameRestartFingerprint(a, b map[string]int) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for k, v := range a {
+		if bv, ok := b[k]; !ok || bv != v {
+			return false
+		}
+	}
+	return true
+}
+
+// waitStableStart waits until deadline for a stable window. stableAborted when a NEW restore of the app begins — one
+// that began after this watch did. (The watch starts inside its own restore, before the handler ends that restore's
+// record, so „a restore of the app is running" alone would end every watch at once.)
+func (m *Manager) waitStableStart(stack string, watchFrom, deadline time.Time) stableResult {
+	var since time.Time
+	var fp0 map[string]int
+	window, poll := stableStartWindow, stableStartPoll // read once: the rule this watch began under
+	for time.Now().Before(deadline) {
+		if m.restoreBegunAfter(stack, watchFrom) {
+			return stableAborted
+		}
+		ok, fp := m.stableSample(stack)
+		switch {
+		case !ok:
+			since, fp0 = time.Time{}, nil
+		case since.IsZero() || !sameRestartFingerprint(fp0, fp):
+			since, fp0 = time.Now(), fp
+		case time.Since(since) >= window:
+			return stableYes
+		}
+		time.Sleep(poll)
+	}
+	return stableNot
+}
+
+// releaseAfterStableStart is the success path's release: the copy stays until the app ran stably for the window, then
+// it is removed. If the app does not run stably within stableReleaseWatch, the copy stays for support and the operator
+// is told once (the page already said the restore worked — the app's own status shows what it does now).
+func (m *Manager) releaseAfterStableStart(stack, kind string, undo *liveVolumeCopy) {
+	if undo == nil || undo.dir == "" {
+		return
+	}
+	undo.deferred = true
+	dir := undo.dir
+	from := time.Now() // taken HERE, before the goroutine: a restore begun after this moment is a new one
+	until := from.Add(stableReleaseWatch)
+	go func() {
+		switch m.waitStableStart(stack, from, until) {
+		case stableYes:
+			removeCopyDir(m, stack, dir)
+			m.logger.Printf("[INFO] [backup] %s: ran stably for %s after its %s — the restore copy is released (R-1270)", stack, stableStartWindow, kind)
+		case stableAborted:
+			m.logger.Printf("[INFO] [backup] %s: a new restore began before the %s's start was stable — its restore copy stays for support (%s)", stack, kind, dir)
+		default:
+			m.logger.Printf("[ERROR] [backup] %s: did not run stably within %s after its %s — the restore copy stays for support (%s) (R-1270)", stack, stableReleaseWatch, kind, dir)
+			if m.restoreNotStartedNotify != nil {
+				m.restoreNotStartedNotify(stack, kind+" (not stable after its start)", fmt.Errorf("no stable %s within %s", stableStartWindow, stableReleaseWatch))
+			}
+		}
+	}()
+}
+
+// restoreBegunAfter reports whether a restore of stack is running that began after t.
+func (m *Manager) restoreBegunAfter(stack string, t time.Time) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.restoreRunningForLocked(stack) && m.opStartedAt.After(t)
 }