controller v0.265.0: R-634 cause fixed, held apps say so, OOM storm alarm, R-647 leftovers

R-634: a whole-box backup no longer stops/restarts a DEPLOYING app (the
measured cause of containers running under 'not deployed'); StopStack
and StartStack refuse a deploying stack for every caller.
R-625: held badge 'Stopped - restore needed', no Update button.
R-636: kernel oom_kill counter; 20+ in 30 min -> one app_oom_storm.
R-647: held error per reader, copy_holds key, two log wordings.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/notify/notifier.go

@@ -46,6 +46,11 @@ type Notifier struct {
 
 	// oomSeen (R-514) remembers container runs already reported as OOM-killed.
 	oomSeen map[string]bool
+	// oomTrack (R-636) remembers, per container run, the kill counter over the last 30 minutes and
+	// whether the ONE storm alarm of that run has gone.
+	oomTrack map[string]*oomRun
+	// nowFn is a test clock (nil → time.Now).
+	nowFn func() time.Time
 	// appDown tracks which deployed apps are currently in the DOWN state so app_start_failed fires
 	// ONCE per running→down transition, not every health cycle (fix-3 anti-spam). In-memory: a
 	// controller restart re-notifies once (acceptable — better than missing). The hub owns the real
@@ -359,7 +364,7 @@ func (n *Notifier) pushEventBoth(eventType, severity, message, messageCustomer s
 // Detects both degradation (ok→warn, ok→fail, warn→fail) and recovery (fail→ok, warn→ok, fail→warn).
 func (n *Notifier) NotifyHealthChange(status string, issues, warnings []string) {
 	if !n.enabled {
-		n.dropped("health_change", status)
+		n.dropped("health_change", healthSeverity(status)) // R-647: the severity, not the health status
 		return
 	}
 
@@ -714,21 +719,108 @@ func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
 // limit (Docker State.OOMKilled). Fires ONCE per container run (keyed by StartedAt), so a container
 // that stays OOM-marked does not repeat. "warning" — the hub vocabulary (R-329). Operator-only
 // hub-side (hub >= v0.114.0 registers it): the household sees the dashboard tag.
-func (n *Notifier) NotifyAppOOM(stack, container, startedAt string) {
+func (n *Notifier) NotifyAppOOM(stack, container, startedAt string, kills int64, memLimit, peak string) {
 	key := container + "|" + startedAt
 	n.mu.Lock()
 	if n.oomSeen == nil {
 		n.oomSeen = map[string]bool{}
 	}
-	if n.oomSeen[key] {
-		n.mu.Unlock()
-		return
-	}
+	first := !n.oomSeen[key]
 	n.oomSeen[key] = true
+	storm, inWindow := n.oomStormLocked(key, startedAt, kills)
 	n.mu.Unlock()
-	n.emit("app_oom", "warning",
-		fmt.Sprintf("Alkalmazás memóriája elfogyott: %s (%s) — egy folyamatát a memóriakorlát leállította", stack, container),
-		AppDetails{StackName: stack, DisplayName: container})
+	if first {
+		n.emit("app_oom", "warning",
+			fmt.Sprintf("Alkalmazás memóriája elfogyott: %s (%s) — egy folyamatát a memóriakorlát leállította", stack, container),
+			AppDetails{StackName: stack, DisplayName: container})
+	}
+	if storm {
+		n.logger.Printf("[ERROR] [notify] %s: container %s OOM STORM — %d kills in %d min (limit %s, peak %s)",
+			stack, container, inWindow, oomStormWindowMin, memLimit, peak)
+		n.emit("app_oom_storm", "error",
+			fmt.Sprintf("Alkalmazás memóriája ismételten elfogy: %s (%s) — %d leállítás %d percen belül (korlát %s, csúcs %s)",
+				stack, container, inWindow, oomStormWindowMin, memLimit, peak),
+			OOMStormDetails{App: stack, StackName: stack, Container: container, Kills: inWindow,
+				WindowMin: oomStormWindowMin, MemLimit: memLimit, Peak: peak})
+	}
+}
+
+// R-636 (v0.265.0) — ONE louder alarm per container run when OOM kills keep coming.
+//
+// The once-per-run app_oom stays exactly as it was (it is what stops a crash loop from mailing 4,530
+// times, R-629). Beside it: when the kernel's own kill counter for the SAME run rises by
+// oomStormKills or more within oomStormWindowMin minutes, app_oom_storm goes once, at `error`,
+// operator-only. RomM's real rate on 2026-09-22 was 4,530 kills in 6 h ≈ 375 per 30 min; one hiccup
+// is 1–3. Twenty in thirty minutes sits far from both. Pinned by TestR636_*.
+const (
+	oomStormKills     = 20
+	oomStormWindowMin = 30
+)
+
+type oomSample struct {
+	at    time.Time
+	kills int64
+}
+
+type oomRun struct {
+	samples   []oomSample
+	stormSent bool
+}
+
+// OOMStormDetails is the app_oom_storm payload. stack_name lets the hub's per-app cooldown tell two
+// storming apps apart.
+type OOMStormDetails struct {
+	App       string `json:"app"`
+	StackName string `json:"stack_name"`
+	Container string `json:"container"`
+	Kills     int64  `json:"kills"`
+	WindowMin int    `json:"window_min"`
+	MemLimit  string `json:"mem_limit,omitempty"`
+	Peak      string `json:"peak,omitempty"`
+}
+
+func (n *Notifier) now() time.Time {
+	if n.nowFn != nil {
+		return n.nowFn()
+	}
+	return time.Now()
+}
+
+// oomStormLocked records one reading and says whether the storm alarm fires now. Caller holds n.mu.
+// A run seen for the first time within the window of its own start gets a zero reading AT its start:
+// the counter began at zero there, so a run that is already storming when the controller first looks
+// is not made to wait another thirty minutes.
+func (n *Notifier) oomStormLocked(key, startedAt string, kills int64) (bool, int64) {
+	if kills < 0 {
+		return false, 0 // unreadable: never escalate on a guess
+	}
+	if n.oomTrack == nil {
+		n.oomTrack = map[string]*oomRun{}
+	}
+	now := n.now()
+	window := time.Duration(oomStormWindowMin) * time.Minute
+	r := n.oomTrack[key]
+	if r == nil {
+		r = &oomRun{}
+		if t, err := time.Parse(time.RFC3339Nano, startedAt); err == nil && now.Sub(t) <= window {
+			r.samples = append(r.samples, oomSample{at: t, kills: 0})
+		}
+		n.oomTrack[key] = r
+	}
+	r.samples = append(r.samples, oomSample{at: now, kills: kills})
+	keep := r.samples[:0]
+	for _, s := range r.samples {
+		if now.Sub(s.at) <= window {
+			keep = append(keep, s)
+		}
+	}
+	r.samples = keep
+	inWindow := kills - r.samples[0].kills
+	if r.stormSent || inWindow < oomStormKills {
+		return false, inWindow
+	}
+	r.stormSent = true
+	return true, inWindow
 }
 
 // DiskHealthDetails is the event-detail payload for disk_health_degraded.
@@ -1211,3 +1303,15 @@ func (n *Notifier) NotifyAppUpdateHeld(d AppUpdateDetails, sentence func(lang st
 	}
 	n.pushEventBoth("app_update_held", "error", hu, household, d)
 }
+
+// healthSeverity is the event severity a health status would be sent at (R-647): the disabled path
+// names what it drops, and "warn" is a health STATUS, not a severity the hub knows.
+func healthSeverity(status string) string {
+	switch status {
+	case "fail":
+		return "error"
+	case "warn":
+		return "warning"
+	}
+	return "info"
+}

controller/internal/notify/r636_oom_storm_test.go

@@ -0,0 +1,95 @@
+package notify
+
+import (
+	"io"
+	"log"
+	"testing"
+	"time"
+)
+
+// R-636 — a repeating OOM problem gets ONE louder alarm; one hiccup stays one quiet warning.
+func stormRecorder(t *testing.T) (*Notifier, *time.Time, map[string]int) {
+	t.Helper()
+	n := New("", "", "c1", nil, log.New(io.Discard, "", 0), false)
+	got := map[string]int{}
+	n.pushFn = func(eventType, severity, msg, msgCustomer string, details interface{}) {
+		got[eventType+"/"+severity]++
+	}
+	clock := time.Date(2026, 9, 22, 9, 0, 0, 0, time.UTC)
+	n.nowFn = func() time.Time { return clock }
+	return n, &clock, got
+}
+
+// COMPANION RED-PROOF (REPORT.md): make oomStormLocked always return false — the 20-kill case sends no
+// storm and this fails.
+func TestR636_TwentyKillsInThirtyMinutesIsOneStorm(t *testing.T) {
+	for _, c := range []struct {
+		kills int64
+		storm int
+	}{{19, 0}, {20, 1}, {200, 1}} {
+		n, clock, got := stormRecorder(t)
+		start := clock.Add(-time.Hour).Format(time.RFC3339Nano) // started long before: no zero baseline
+		for i := int64(0); i <= 40; i++ { // 20 minutes of 30 s scans, counter 5 → 5+kills
+			n.NotifyAppOOM("romm", "romm", start, 5+c.kills*i/40, "1280M", "1279M")
+			*clock = clock.Add(30 * time.Second)
+		}
+		if got["app_oom_storm/error"] != c.storm {
+			t.Errorf("kills=%d in 20 min: storms=%d, want %d", c.kills, got["app_oom_storm/error"], c.storm)
+		}
+		if c.kills == 200 { // RomM's shape: it keeps going for six hours — still ONE storm for the run
+			for i := int64(1); i <= 720; i++ {
+				n.NotifyAppOOM("romm", "romm", start, 205+i*12, "1280M", "1279M")
+				*clock = clock.Add(30 * time.Second)
+			}
+			if got["app_oom_storm/error"] != 1 {
+				t.Errorf("six hours of storm must stay ONE storm per container run, got %d", got["app_oom_storm/error"])
+			}
+		}
+		if got["app_oom/warning"] != 1 {
+			t.Errorf("kills=%d: the quiet app_oom must still go exactly once per run, got %d", c.kills, got["app_oom/warning"])
+		}
+	}
+}
+
+// One kill, and then the flag just stays set for hours (the sticky OOMKilled): never a storm.
+func TestR636_OneHiccupNeverStorms(t *testing.T) {
+	n, clock, got := stormRecorder(t)
+	start := clock.Format(time.RFC3339Nano)
+	for i := 0; i < 6*120; i++ { // six hours of 30 s scans, counter stuck at 1
+		n.NotifyAppOOM("paperless-ngx", "paperless-webserver", start, 1, "768M", "700M")
+		*clock = clock.Add(30 * time.Second)
+	}
+	if got["app_oom_storm/error"] != 0 || got["app_oom/warning"] != 1 {
+		t.Fatalf("one hiccup: want 1 warning and 0 storms, got %v", got)
+	}
+}
+
+// Slow and steady — 20 kills spread over three hours is not a storm.
+func TestR636_SlowKillsAreNotAStorm(t *testing.T) {
+	n, clock, got := stormRecorder(t)
+	start := clock.Add(-time.Hour).Format(time.RFC3339Nano)
+	for i := int64(0); i <= 360; i++ { // 3 h, counter +20 in total
+		n.NotifyAppOOM("x", "x", start, i*20/360, "", "")
+		*clock = clock.Add(30 * time.Second)
+	}
+	if got["app_oom_storm/error"] != 0 {
+		t.Fatalf("20 kills over 3 h must not storm, got %v", got)
+	}
+}
+
+// A run first seen already storming (the controller restarted mid-storm) fires at once: its counter
+// started at zero at its own start, inside the window. An unreadable counter never escalates.
+func TestR636_AlreadyStormingAtFirstSightAndUnreadable(t *testing.T) {
+	n, clock, got := stormRecorder(t)
+	n.NotifyAppOOM("romm", "romm", clock.Add(-10*time.Minute).Format(time.RFC3339Nano), 150, "1280M", "1279M")
+	if got["app_oom_storm/error"] != 1 {
+		t.Fatalf("150 kills 10 min after the start must storm at first sight, got %v", got)
+	}
+	n2, _, got2 := stormRecorder(t)
+	for i := 0; i < 100; i++ {
+		n2.NotifyAppOOM("romm", "romm", "", -1, "", "")
+	}
+	if got2["app_oom_storm/error"] != 0 {
+		t.Fatalf("an unreadable counter must never escalate, got %v", got2)
+	}
+}

controller/internal/notify/update_events_test.go

@@ -102,3 +102,15 @@ func TestR620_DisabledNotifierSaysWhatItDrops(t *testing.T) {
 		}
 	}
 }
+
+// R-647 (3) — the disabled path names the SEVERITY a health change would be sent at, never the health
+// status. COMPANION RED-PROOF (REPORT.md): pass `status` again — the line reads "severity warn".
+func TestR647_DisabledHealthChangeNamesTheSeverity(t *testing.T) {
+	var buf bytes.Buffer
+	n := New("", "", "c1", nil, log.New(&buf, "", 0), false)
+	buf.Reset()
+	n.NotifyHealthChange("warn", nil, nil)
+	if !strings.Contains(buf.String(), "DROPPED event health_change (severity warning)") {
+		t.Fatalf("want the severity 'warning' named, got:\n%s", buf.String())
+	}
+}