174 lines
6.9 KiB
Go
174 lines
6.9 KiB
Go
package backup
|
|
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Fix A (brief 2026-10-26; R-1270 follow-up) — the undo copy is released only after a STABLE start.
|
|
// COMPANION RED-PROOF (REPORT.md): on `e8d2151` (the copy dropped at the restore's return / at the watcher's first
|
|
// running sample) TestFixA_AFlappingAppKeepsTheCopy and TestFixA_AHealthcheckStillStartingKeepsTheCopy fail.
|
|
|
|
// sampledProvider is the D22 rig's provider with a scripted container sample per call.
|
|
type sampledProvider struct {
|
|
*r1177Provider
|
|
mu sync.Mutex
|
|
n int
|
|
running func(n int) bool
|
|
sample func(n int) []ContainerSample
|
|
}
|
|
|
|
func (p *sampledProvider) RefreshAndIsRunning(string) bool {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.n++
|
|
return p.running(p.n)
|
|
}
|
|
|
|
func (p *sampledProvider) ContainerSamples(string) ([]ContainerSample, error) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.sample(p.n), nil
|
|
}
|
|
|
|
func fixARig(t *testing.T, running func(int) bool, sample func(int) []ContainerSample) (*Manager, *r1177Rig, *[]string) {
|
|
t.Helper()
|
|
m, p, rig, _ := r1177Fixture(t)
|
|
sp := &sampledProvider{r1177Provider: p, running: running, sample: sample}
|
|
m.stackProvider = sp
|
|
var alarms []string
|
|
var mu sync.Mutex
|
|
m.SetRestoreNotStartedNotify(func(stack, kind string, _ error) { mu.Lock(); alarms = append(alarms, kind); mu.Unlock() })
|
|
return m, rig, &alarms
|
|
}
|
|
|
|
func withStable(t *testing.T, window, watch time.Duration) {
|
|
t.Helper()
|
|
w0, p0, r0 := stableStartWindow, stableStartPoll, stableReleaseWatch
|
|
stableStartWindow, stableStartPoll, stableReleaseWatch = window, 2*time.Millisecond, watch
|
|
t.Cleanup(func() { stableStartWindow, stableStartPoll, stableReleaseWatch = w0, p0, r0 })
|
|
}
|
|
|
|
func healthy(id string, restarts int) ContainerSample {
|
|
return ContainerSample{ID: id, Running: true, RestartCount: restarts, Health: "healthy"}
|
|
}
|
|
|
|
// running → restarting → running: the copy stays, and the operator hears once that the start was not stable.
|
|
func TestFixA_AFlappingAppKeepsTheCopy(t *testing.T) {
|
|
withStable(t, 60*time.Millisecond, 400*time.Millisecond)
|
|
m, rig, alarms := fixARig(t,
|
|
func(n int) bool { return n%3 != 0 }, // every third sample: restarting
|
|
func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }) // and the count grows
|
|
lateStartWatch = 400 * time.Millisecond
|
|
defer func() { lateStartWatch = 30 * time.Minute }()
|
|
m.BeginRestoreOp("restore", "app")
|
|
_, err := m.RestoreFromRecoveryUnit("app")
|
|
if !errors.Is(err, ErrRestoreNotStarted) {
|
|
t.Fatalf("a running → restarting → running app is „not started yet\" on the page too, got %v", err)
|
|
}
|
|
m.EndRestoreOpNotStarted(err.Error())
|
|
time.Sleep(600 * time.Millisecond)
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
|
t.Fatalf("a flapping app must keep its way back, found %v", left)
|
|
}
|
|
if len(*alarms) != 1 {
|
|
t.Fatalf("the operator hears once, got %v", *alarms)
|
|
}
|
|
if st := m.RestoreStatus(); st.Last.OK {
|
|
t.Fatal("the late-start watch must never turn a flapping app into a success")
|
|
}
|
|
}
|
|
|
|
// A stable stack: the copy is still there right after the restore, and gone after the window.
|
|
func TestFixA_AStableAppReleasesTheCopyAfterTheWindow(t *testing.T) {
|
|
withStable(t, 150*time.Millisecond, 3*time.Second)
|
|
m, rig, alarms := fixARig(t, func(int) bool { return true },
|
|
func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0), {ID: "db", Running: true}} })
|
|
// The production order: the handler begins the record, the restore runs (and starts the watch), the handler ends it.
|
|
m.BeginRestoreOp("restore", "app")
|
|
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.EndRestoreOp(true, "ok")
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
|
t.Fatalf("the copy is NOT released at the restore's return any more, found %v", left)
|
|
}
|
|
if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 {
|
|
t.Fatalf("a stable app releases the copy after the window, found %v", left)
|
|
}
|
|
if len(*alarms) != 0 {
|
|
t.Fatalf("no alarm for a stable app: %v", *alarms)
|
|
}
|
|
}
|
|
|
|
// A container whose healthcheck still says `starting` is not started.
|
|
func TestFixA_AHealthcheckStillStartingKeepsTheCopy(t *testing.T) {
|
|
withStable(t, 30*time.Millisecond, 300*time.Millisecond)
|
|
m, rig, _ := fixARig(t, func(int) bool { return true },
|
|
func(int) []ContainerSample { return []ContainerSample{{ID: "c1", Running: true, Health: "starting"}} })
|
|
if _, err := m.RestoreFromRecoveryUnit("app"); !errors.Is(err, ErrRestoreNotStarted) {
|
|
t.Fatalf("a healthcheck still starting is not started, got %v", err)
|
|
}
|
|
time.Sleep(500 * time.Millisecond)
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
|
t.Fatalf("a healthcheck still starting must keep the copy, found %v", left)
|
|
}
|
|
}
|
|
|
|
// A new restore of the app during the watch ends it untouched (unchanged rule).
|
|
func TestFixA_ANewRestoreDuringTheWatchLeavesTheCopy(t *testing.T) {
|
|
withStable(t, 400*time.Millisecond, 2*time.Second)
|
|
m, rig, alarms := fixARig(t, func(int) bool { return true },
|
|
func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
|
|
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.BeginRestoreOp("restore", "app")
|
|
time.Sleep(700 * time.Millisecond)
|
|
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
|
t.Fatalf("a new restore owns the app: the watch ends and the copy stays, found %v", left)
|
|
}
|
|
if len(*alarms) != 0 {
|
|
t.Fatalf("an aborted watch raises no alarm: %v", *alarms)
|
|
}
|
|
}
|
|
|
|
// The late-start watcher (after a not-started verdict) uses the same rule: a flapping app never becomes a success.
|
|
func TestFixA_TheLateStartWatcherNeedsAStableStart(t *testing.T) {
|
|
withStable(t, 60*time.Millisecond, 2*time.Second)
|
|
lateStartWatch = 400 * time.Millisecond
|
|
defer func() { lateStartWatch = 30 * time.Minute }()
|
|
m := &Manager{logger: log.New(io.Discard, "", 0)}
|
|
sp := &sampledProvider{r1177Provider: &r1177Provider{fakeRecoveryProvider: &fakeRecoveryProvider{}},
|
|
running: func(n int) bool { return n%3 != 0 },
|
|
sample: func(n int) []ContainerSample { return []ContainerSample{healthy("c1", n/3)} }}
|
|
m.stackProvider = sp
|
|
m.BeginRestoreOp("restore", "app")
|
|
m.EndRestoreOpNotStarted("not yet")
|
|
m.watchLateStart("app", "", time.Now(), lateStartWatch)
|
|
if st := m.RestoreStatus(); st.Last.OK {
|
|
t.Fatal("a flapping app must never turn the record into a success")
|
|
}
|
|
}
|
|
|
|
// ShortenRestoreTimingForTests must never reach production: only test files may call it.
|
|
func TestFixA_TheTestTimingSwitchIsNeverCalledByProduction(t *testing.T) {
|
|
root := filepath.Join("..", "..")
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil || fi.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || strings.HasSuffix(p, "restore_verdict.go") {
|
|
return nil
|
|
}
|
|
if b, _ := os.ReadFile(p); strings.Contains(string(b), "ShortenRestoreTimingForTests") {
|
|
t.Errorf("%s calls the test-only timing switch", p)
|
|
}
|
|
return nil
|
|
})
|
|
}
|