felhom-controller/controller/internal/notify/r636_oom_storm_test.go

package notify

import (
	"io"
	"log"
	"testing"
	"time"
)

// R-636 — a repeating OOM problem gets ONE louder alarm; one hiccup stays one quiet warning.
func stormRecorder(t *testing.T) (*Notifier, *time.Time, map[string]int) {
	t.Helper()
	n := New("", "", "c1", nil, log.New(io.Discard, "", 0), false)
	got := map[string]int{}
	n.pushFn = func(eventType, severity, msg, msgCustomer string, details interface{}) {
		got[eventType+"/"+severity]++
	}
	clock := time.Date(2026, 9, 22, 9, 0, 0, 0, time.UTC)
	n.nowFn = func() time.Time { return clock }
	return n, &clock, got
}

// COMPANION RED-PROOF (REPORT.md): make oomStormLocked always return false — the 20-kill case sends no
// storm and this fails.
func TestR636_TwentyKillsInThirtyMinutesIsOneStorm(t *testing.T) {
	for _, c := range []struct {
		kills int64
		storm int
	}{{19, 0}, {20, 1}, {200, 1}} {
		n, clock, got := stormRecorder(t)
		start := clock.Add(-time.Hour).Format(time.RFC3339Nano) // started long before: no zero baseline
		for i := int64(0); i <= 40; i++ {                       // 20 minutes of 30 s scans, counter 5 → 5+kills
			n.NotifyAppOOM("romm", "romm", start, 5+c.kills*i/40, "1280M", "1279M")
			*clock = clock.Add(30 * time.Second)
		}
		if got["app_oom_storm/error"] != c.storm {
			t.Errorf("kills=%d in 20 min: storms=%d, want %d", c.kills, got["app_oom_storm/error"], c.storm)
		}
		if c.kills == 200 { // RomM's shape: it keeps going for six hours — still ONE storm for the run
			for i := int64(1); i <= 720; i++ {
				n.NotifyAppOOM("romm", "romm", start, 205+i*12, "1280M", "1279M")
				*clock = clock.Add(30 * time.Second)
			}
			if got["app_oom_storm/error"] != 1 {
				t.Errorf("six hours of storm must stay ONE storm per container run, got %d", got["app_oom_storm/error"])
			}
		}
		if got["app_oom/warning"] != 1 {
			t.Errorf("kills=%d: the quiet app_oom must still go exactly once per run, got %d", c.kills, got["app_oom/warning"])
		}
	}
}

// One kill, and then the flag just stays set for hours (the sticky OOMKilled): never a storm.
func TestR636_OneHiccupNeverStorms(t *testing.T) {
	n, clock, got := stormRecorder(t)
	start := clock.Format(time.RFC3339Nano)
	for i := 0; i < 6*120; i++ { // six hours of 30 s scans, counter stuck at 1
		n.NotifyAppOOM("paperless-ngx", "paperless-webserver", start, 1, "768M", "700M")
		*clock = clock.Add(30 * time.Second)
	}
	if got["app_oom_storm/error"] != 0 || got["app_oom/warning"] != 1 {
		t.Fatalf("one hiccup: want 1 warning and 0 storms, got %v", got)
	}
}

// Slow and steady — 20 kills spread over three hours is not a storm.
func TestR636_SlowKillsAreNotAStorm(t *testing.T) {
	n, clock, got := stormRecorder(t)
	start := clock.Add(-time.Hour).Format(time.RFC3339Nano)
	for i := int64(0); i <= 360; i++ { // 3 h, counter +20 in total
		n.NotifyAppOOM("x", "x", start, i*20/360, "", "")
		*clock = clock.Add(30 * time.Second)
	}
	if got["app_oom_storm/error"] != 0 {
		t.Fatalf("20 kills over 3 h must not storm, got %v", got)
	}
}

// A run first seen already storming (the controller restarted mid-storm) fires at once: its counter
// started at zero at its own start, inside the window. An unreadable counter never escalates.
func TestR636_AlreadyStormingAtFirstSightAndUnreadable(t *testing.T) {
	n, clock, got := stormRecorder(t)
	n.NotifyAppOOM("romm", "romm", clock.Add(-10*time.Minute).Format(time.RFC3339Nano), 150, "1280M", "1279M")
	if got["app_oom_storm/error"] != 1 {
		t.Fatalf("150 kills 10 min after the start must storm at first sight, got %v", got)
	}
	n2, _, got2 := stormRecorder(t)
	for i := 0; i < 100; i++ {
		n2.NotifyAppOOM("romm", "romm", "", -1, "", "")
	}
	if got2["app_oom_storm/error"] != 0 {
		t.Fatalf("an unreadable counter must never escalate, got %v", got2)
	}
}