Fix A, the page too: the restore's first verdict needs a stable 45 s after the 90 s health wait on every path, so an app that runs, crashes and restarts reads „not started yet”; the copy still waits the full 3 minutes. Red-proved. Commit only.
gates / gates (push) Successful in 1m30s
gates / gates (push) Successful in 1m30s
This commit is contained in:
1 parent
7720439440
commit
2149a13378
10 files changed
+88
-18
No files matched your search
@@ -110,9 +110,13 @@ func removeCopyDir(m *Manager, stack, copyDir string) {
|
||||
// whatever its timing; it costs the household nothing (the app runs meanwhile), only the copy's disk for 3 minutes more.
|
||||
|
||||
var (
|
||||
stableStartWindow = 3 * time.Minute
|
||||
stableStartPoll = 10 * time.Second
|
||||
stableReleaseWatch = 30 * time.Minute
|
||||
// verdictStableWindow is the first verdict's own stable window (after the 90 s health wait found the app running):
|
||||
// 45 s — longer than the 20 s crash cycle of the measured case, so an app that runs, crashes and restarts is
|
||||
// „not started yet" on the page too, and short enough that a good restore is reported within about two minutes.
|
||||
verdictStableWindow = 45 * time.Second
|
||||
stableStartWindow = 3 * time.Minute
|
||||
stableStartPoll = 10 * time.Second
|
||||
stableReleaseWatch = 30 * time.Minute
|
||||
)
|
||||
|
||||
// ContainerSample is one container of a stack as a stable-start sample sees it.
|
||||
@@ -176,9 +180,14 @@ func sameRestartFingerprint(a, b map[string]int) bool {
|
||||
// that began after this watch did. (The watch starts inside its own restore, before the handler ends that restore's
|
||||
// record, so „a restore of the app is running" alone would end every watch at once.)
|
||||
func (m *Manager) waitStableStart(stack string, watchFrom, deadline time.Time) stableResult {
|
||||
return m.waitStableFor(stack, watchFrom, deadline, stableStartWindow)
|
||||
}
|
||||
|
||||
// waitStableFor is waitStableStart with an explicit window (the first verdict's is shorter, verdictStableWindow).
|
||||
func (m *Manager) waitStableFor(stack string, watchFrom, deadline time.Time, window time.Duration) stableResult {
|
||||
var since time.Time
|
||||
var fp0 map[string]int
|
||||
window, poll := stableStartWindow, stableStartPoll // read once: the rule this watch began under
|
||||
poll := stableStartPoll // read once: the rule this watch began under
|
||||
for time.Now().Before(deadline) {
|
||||
if m.restoreBegunAfter(stack, watchFrom) {
|
||||
return stableAborted
|
||||
@@ -230,3 +239,27 @@ func (m *Manager) restoreBegunAfter(stack string, t time.Time) bool {
|
||||
defer m.mu.Unlock()
|
||||
return m.restoreRunningForLocked(stack) && m.opStartedAt.After(t)
|
||||
}
|
||||
|
||||
// restoredAppStarted is the restore's first verdict (every path): the 90 s health wait (one running sample, as before)
|
||||
// AND then a stable verdictStableWindow within another health timeout — so an app that crashes and restarts is not
|
||||
// reported as started. nil = started (the page says so; the copy is still released only after stableStartWindow).
|
||||
func (m *Manager) restoredAppStarted(stack string) error {
|
||||
if err := m.waitForHealthy(stack, healthTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
from := time.Now()
|
||||
switch m.waitStableFor(stack, from, from.Add(healthTimeout+verdictStableWindow), verdictStableWindow) {
|
||||
case stableYes:
|
||||
return nil
|
||||
case stableAborted:
|
||||
return fmt.Errorf("a new restore of %s began", stack)
|
||||
}
|
||||
return fmt.Errorf("stack %s did not run stably for %s (it restarted or turned unhealthy) after restore", stack, verdictStableWindow)
|
||||
}
|
||||
|
||||
// ShortenRestoreTimingForTests (TESTS ONLY — other packages' TestMain) makes the stable-start rule take milliseconds,
|
||||
// as this package's own TestMain does; a web or api test that runs a real restore then does not wait 45 s per call.
|
||||
// Never called by production code (pinned by TestR488_HealthWaitDefaultsAreProduction reading the defaults).
|
||||
func ShortenRestoreTimingForTests() {
|
||||
verdictStableWindow, stableStartWindow, stableStartPoll = 10*time.Millisecond, 30*time.Millisecond, 2*time.Millisecond
|
||||
}
|
||||
Reference in new issue
Block a user