admin
3c6b49b31c
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
103 lines
4.0 KiB
Go
103 lines
4.0 KiB
Go
package stacks
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
var uT0 = time.Date(2026, 9, 24, 9, 22, 0, 0, time.UTC)
|
|
|
|
// gokapi's measured shape: Docker's back-off caps a steady crash loop at ~1 restart a minute (539 → 546
|
|
// in 7 min on 9202). Sampled every 30 s, as the dead-app loop does.
|
|
func gokapiSamples(minutes int) []unhealthySample {
|
|
var ss []unhealthySample
|
|
for i := 0; i <= minutes*2; i++ {
|
|
ss = append(ss, unhealthySample{at: uT0.Add(time.Duration(i) * 30 * time.Second), restarts: 539 + int64(i/2), kills: 0})
|
|
}
|
|
return ss
|
|
}
|
|
|
|
// TestR667_TheMeasuredCrashLoopTrips — the shape the old detector never saw.
|
|
//
|
|
// COMPANION RED-PROOF (REPORT): make judgeUnhealthy compare only the LAST TWO samples (the "clock that
|
|
// resets at every look" shape of RestartingSince) — this test then fails at "gokapi's loop did not trip".
|
|
func TestR667_TheMeasuredCrashLoopTrips(t *testing.T) {
|
|
if k, _, _ := judgeUnhealthy(gokapiSamples(5)); k != "" {
|
|
t.Fatalf("5 minutes of 1/min restarts tripped already (%s) — the threshold is 6 in 10 min", k)
|
|
}
|
|
k, n, w := judgeUnhealthy(gokapiSamples(7))
|
|
if k != UnhealthyCrashLoop || n < CrashLoopRestarts || w != CrashLoopWindow {
|
|
t.Fatalf("gokapi's loop did not trip: kind=%q n=%d", k, n)
|
|
}
|
|
}
|
|
|
|
func TestR667_SlowStartsAndResetsDoNotTrip(t *testing.T) {
|
|
// the worst healthy container measured on the demo boxes: one restart at first start
|
|
one := []unhealthySample{{at: uT0, restarts: 0}, {at: uT0.Add(30 * time.Second), restarts: 1}, {at: uT0.Add(9 * time.Minute), restarts: 1}}
|
|
if k, _, _ := judgeUnhealthy(one); k != "" {
|
|
t.Fatalf("a single first-start restart tripped: %s", k)
|
|
}
|
|
// compose recreated the container: RestartCount 40 → 0 → 3 is a reset, not -37 or +43
|
|
reset := []unhealthySample{{at: uT0, restarts: 40}, {at: uT0.Add(time.Minute), restarts: 0}, {at: uT0.Add(2 * time.Minute), restarts: 3}}
|
|
if k, _, _ := judgeUnhealthy(reset); k != "" {
|
|
t.Fatalf("a recreate read as a crash loop: %s", k)
|
|
}
|
|
// restarts spread over more than the window: 6 restarts in 30 min is not a loop
|
|
var spread []unhealthySample
|
|
for i := 0; i <= 6; i++ {
|
|
spread = append(spread, unhealthySample{at: uT0.Add(time.Duration(i) * 5 * time.Minute), restarts: int64(i)})
|
|
}
|
|
if k, _, _ := judgeUnhealthy(spread); k != "" {
|
|
t.Fatalf("6 restarts in 30 min tripped: %s", k)
|
|
}
|
|
}
|
|
|
|
func TestR667_OOMStormTrips(t *testing.T) {
|
|
ss := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(10 * time.Minute), kills: 12}, {at: uT0.Add(25 * time.Minute), kills: 23}}
|
|
if k, n, _ := judgeUnhealthy(ss); k != UnhealthyOOMStorm || n != 20 {
|
|
t.Fatalf("kind=%q n=%d, want an OOM storm of 20", k, n)
|
|
}
|
|
calm := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(25 * time.Minute), kills: 10}}
|
|
if k, _, _ := judgeUnhealthy(calm); k != "" {
|
|
t.Fatalf("7 kills tripped: %s", k)
|
|
}
|
|
}
|
|
|
|
// ObserveUnhealthy through its docker seam: RestartCount summed per app, one verdict per episode, and an
|
|
// app that is updating is never judged.
|
|
func TestR667_ObserveSumsPerAppAndSkipsUpdating(t *testing.T) {
|
|
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
|
|
"gokapi": {Name: "gokapi", Deployed: true, Containers: []ContainerInfo{{Name: "gokapi"}}},
|
|
"romm": {Name: "romm", Deployed: true, Updating: true, Containers: []ContainerInfo{{Name: "romm"}}},
|
|
}}
|
|
count := int64(539)
|
|
m.execFn = func(name string, args ...string) (string, error) {
|
|
a := strings.Join(args, " ")
|
|
out := ""
|
|
if strings.Contains(a, "gokapi") {
|
|
out += fmt.Sprintf("/gokapi|%d\n", count)
|
|
}
|
|
if strings.Contains(a, "romm") {
|
|
out += "/romm|999\n"
|
|
}
|
|
return out, nil
|
|
}
|
|
var got []UnhealthyVerdict
|
|
for i := 0; i <= 16 && len(got) == 0; i++ {
|
|
got = m.ObserveUnhealthy(uT0.Add(time.Duration(i)*30*time.Second), nil)
|
|
if i%2 == 1 {
|
|
count++
|
|
}
|
|
}
|
|
if len(got) != 1 || got[0].Stack != "gokapi" || got[0].Kind != UnhealthyCrashLoop {
|
|
t.Fatalf("verdicts = %+v, want exactly gokapi's crash loop", got)
|
|
}
|
|
if again := m.ObserveUnhealthy(uT0.Add(9*time.Minute), nil); len(again) != 0 {
|
|
t.Fatalf("the same episode tripped twice: %+v", again)
|
|
}
|
|
}
|