Fix A, the page too: the restore's first verdict needs a stable 45 s after the 90 s health wait on every path, so an app that runs, crashes and restarts reads „not started yet”; the copy still waits the full 3 minutes. Red-proved. Commit only.
gates / gates (push) Successful in 1m30s
gates / gates (push) Successful in 1m30s
This commit is contained in:
@@ -1,8 +1,12 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -62,15 +66,23 @@ func TestFixA_AFlappingAppKeepsTheCopy(t *testing.T) {
|
||||
m, rig, alarms := fixARig(t,
|
||||
func(n int) bool { return n%3 != 0 }, // every third sample: restarting
|
||||
func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }) // and the count grows
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatalf("the restore itself passes its 90 s health wait (a running sample): %v", err)
|
||||
lateStartWatch = 400 * time.Millisecond
|
||||
defer func() { lateStartWatch = 30 * time.Minute }()
|
||||
m.BeginRestoreOp("restore", "app")
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
if !errors.Is(err, ErrRestoreNotStarted) {
|
||||
t.Fatalf("a running → restarting → running app is „not started yet\" on the page too, got %v", err)
|
||||
}
|
||||
m.EndRestoreOpNotStarted(err.Error())
|
||||
time.Sleep(600 * time.Millisecond)
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
||||
t.Fatalf("a flapping app must keep its way back, found %v", left)
|
||||
}
|
||||
if len(*alarms) != 1 {
|
||||
t.Fatalf("the operator hears once that the start was not stable, got %v", *alarms)
|
||||
t.Fatalf("the operator hears once, got %v", *alarms)
|
||||
}
|
||||
if st := m.RestoreStatus(); st.Last.OK {
|
||||
t.Fatal("the late-start watch must never turn a flapping app into a success")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,8 +113,8 @@ func TestFixA_AHealthcheckStillStartingKeepsTheCopy(t *testing.T) {
|
||||
withStable(t, 30*time.Millisecond, 300*time.Millisecond)
|
||||
m, rig, _ := fixARig(t, func(int) bool { return true },
|
||||
func(int) []ContainerSample { return []ContainerSample{{ID: "c1", Running: true, Health: "starting"}} })
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatal(err)
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); !errors.Is(err, ErrRestoreNotStarted) {
|
||||
t.Fatalf("a healthcheck still starting is not started, got %v", err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
||||
@@ -145,3 +157,17 @@ func TestFixA_TheLateStartWatcherNeedsAStableStart(t *testing.T) {
|
||||
t.Fatal("a flapping app must never turn the record into a success")
|
||||
}
|
||||
}
|
||||
|
||||
// ShortenRestoreTimingForTests must never reach production: only test files may call it.
|
||||
func TestFixA_TheTestTimingSwitchIsNeverCalledByProduction(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
||||
if err != nil || fi.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || strings.HasSuffix(p, "restore_verdict.go") {
|
||||
return nil
|
||||
}
|
||||
if b, _ := os.ReadFile(p); strings.Contains(string(b), "ShortenRestoreTimingForTests") {
|
||||
t.Errorf("%s calls the test-only timing switch", p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user