package notify import ( "io" "log" "testing" "time" ) // R-636 — a repeating OOM problem gets ONE louder alarm; one hiccup stays one quiet warning. func stormRecorder(t *testing.T) (*Notifier, *time.Time, map[string]int) { t.Helper() n := New("", "", "c1", nil, log.New(io.Discard, "", 0), false) got := map[string]int{} n.pushFn = func(eventType, severity, msg, msgCustomer string, details interface{}) { got[eventType+"/"+severity]++ } clock := time.Date(2026, 9, 22, 9, 0, 0, 0, time.UTC) n.nowFn = func() time.Time { return clock } return n, &clock, got } // COMPANION RED-PROOF (REPORT.md): make oomStormLocked always return false — the 20-kill case sends no // storm and this fails. func TestR636_TwentyKillsInThirtyMinutesIsOneStorm(t *testing.T) { for _, c := range []struct { kills int64 storm int }{{19, 0}, {20, 1}, {200, 1}} { n, clock, got := stormRecorder(t) start := clock.Add(-time.Hour).Format(time.RFC3339Nano) // started long before: no zero baseline for i := int64(0); i <= 40; i++ { // 20 minutes of 30 s scans, counter 5 → 5+kills n.NotifyAppOOM("romm", "romm", start, 5+c.kills*i/40, "1280M", "1279M") *clock = clock.Add(30 * time.Second) } if got["app_oom_storm/error"] != c.storm { t.Errorf("kills=%d in 20 min: storms=%d, want %d", c.kills, got["app_oom_storm/error"], c.storm) } if c.kills == 200 { // RomM's shape: it keeps going for six hours — still ONE storm for the run for i := int64(1); i <= 720; i++ { n.NotifyAppOOM("romm", "romm", start, 205+i*12, "1280M", "1279M") *clock = clock.Add(30 * time.Second) } if got["app_oom_storm/error"] != 1 { t.Errorf("six hours of storm must stay ONE storm per container run, got %d", got["app_oom_storm/error"]) } } if got["app_oom/warning"] != 1 { t.Errorf("kills=%d: the quiet app_oom must still go exactly once per run, got %d", c.kills, got["app_oom/warning"]) } } } // One kill, and then the flag just stays set for hours (the sticky OOMKilled): never a storm. func TestR636_OneHiccupNeverStorms(t *testing.T) { n, clock, got := stormRecorder(t) start := clock.Format(time.RFC3339Nano) for i := 0; i < 6*120; i++ { // six hours of 30 s scans, counter stuck at 1 n.NotifyAppOOM("paperless-ngx", "paperless-webserver", start, 1, "768M", "700M") *clock = clock.Add(30 * time.Second) } if got["app_oom_storm/error"] != 0 || got["app_oom/warning"] != 1 { t.Fatalf("one hiccup: want 1 warning and 0 storms, got %v", got) } } // Slow and steady — 20 kills spread over three hours is not a storm. func TestR636_SlowKillsAreNotAStorm(t *testing.T) { n, clock, got := stormRecorder(t) start := clock.Add(-time.Hour).Format(time.RFC3339Nano) for i := int64(0); i <= 360; i++ { // 3 h, counter +20 in total n.NotifyAppOOM("x", "x", start, i*20/360, "", "") *clock = clock.Add(30 * time.Second) } if got["app_oom_storm/error"] != 0 { t.Fatalf("20 kills over 3 h must not storm, got %v", got) } } // A run first seen already storming (the controller restarted mid-storm) fires at once: its counter // started at zero at its own start, inside the window. An unreadable counter never escalates. func TestR636_AlreadyStormingAtFirstSightAndUnreadable(t *testing.T) { n, clock, got := stormRecorder(t) n.NotifyAppOOM("romm", "romm", clock.Add(-10*time.Minute).Format(time.RFC3339Nano), 150, "1280M", "1279M") if got["app_oom_storm/error"] != 1 { t.Fatalf("150 kills 10 min after the start must storm at first sight, got %v", got) } n2, _, got2 := stormRecorder(t) for i := 0; i < 100; i++ { n2.NotifyAppOOM("romm", "romm", "", -1, "", "") } if got2["app_oom_storm/error"] != 0 { t.Fatalf("an unreadable counter must never escalate, got %v", got2) } }