Fix A, the page too: the restore's first verdict needs a stable 45 s after the 90 s health wait on every path, so an app that runs, crashes and restarts reads „not started yet”; the copy still waits the full 3 minutes. Red-proved. Commit only.
gates / gates (push) Successful in 1m30s

This commit is contained in:
2026-10-25 13:06:06 +01:00
parent 7720439440
commit 2149a13378
10 changed files with 88 additions and 18 deletions
@@ -1,8 +1,12 @@
package backup
import (
"errors"
"io"
"log"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -62,15 +66,23 @@ func TestFixA_AFlappingAppKeepsTheCopy(t *testing.T) {
m, rig, alarms := fixARig(t,
func(n int) bool { return n%3 != 0 }, // every third sample: restarting
func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }) // and the count grows
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
t.Fatalf("the restore itself passes its 90 s health wait (a running sample): %v", err)
lateStartWatch = 400 * time.Millisecond
defer func() { lateStartWatch = 30 * time.Minute }()
m.BeginRestoreOp("restore", "app")
_, err := m.RestoreFromRecoveryUnit("app")
if !errors.Is(err, ErrRestoreNotStarted) {
t.Fatalf("a running → restarting → running app is „not started yet\" on the page too, got %v", err)
}
m.EndRestoreOpNotStarted(err.Error())
time.Sleep(600 * time.Millisecond)
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
t.Fatalf("a flapping app must keep its way back, found %v", left)
}
if len(*alarms) != 1 {
t.Fatalf("the operator hears once that the start was not stable, got %v", *alarms)
t.Fatalf("the operator hears once, got %v", *alarms)
}
if st := m.RestoreStatus(); st.Last.OK {
t.Fatal("the late-start watch must never turn a flapping app into a success")
}
}
@@ -101,8 +113,8 @@ func TestFixA_AHealthcheckStillStartingKeepsTheCopy(t *testing.T) {
withStable(t, 30*time.Millisecond, 300*time.Millisecond)
m, rig, _ := fixARig(t, func(int) bool { return true },
func(int) []ContainerSample { return []ContainerSample{{ID: "c1", Running: true, Health: "starting"}} })
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
t.Fatal(err)
if _, err := m.RestoreFromRecoveryUnit("app"); !errors.Is(err, ErrRestoreNotStarted) {
t.Fatalf("a healthcheck still starting is not started, got %v", err)
}
time.Sleep(500 * time.Millisecond)
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
@@ -145,3 +157,17 @@ func TestFixA_TheLateStartWatcherNeedsAStableStart(t *testing.T) {
t.Fatal("a flapping app must never turn the record into a success")
}
}
// ShortenRestoreTimingForTests must never reach production: only test files may call it.
func TestFixA_TheTestTimingSwitchIsNeverCalledByProduction(t *testing.T) {
root := filepath.Join("..", "..")
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || strings.HasSuffix(p, "restore_verdict.go") {
return nil
}
if b, _ := os.ReadFile(p); strings.Contains(string(b), "ShortenRestoreTimingForTests") {
t.Errorf("%s calls the test-only timing switch", p)
}
return nil
})
}