controller v0.265.0: R-634 cause fixed, held apps say so, OOM storm alarm, R-647 leftovers

R-634: a whole-box backup no longer stops/restarts a DEPLOYING app (the
measured cause of containers running under 'not deployed'); StopStack
and StartStack refuse a deploying stack for every caller.
R-625: held badge 'Stopped - restore needed', no Update button.
R-636: kernel oom_kill counter; 20+ in 30 min -> one app_oom_storm.
R-647: held error per reader, copy_holds key, two log wordings.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/notify/notifier.go

@@ -46,6 +46,11 @@ type Notifier struct {
 
 	// oomSeen (R-514) remembers container runs already reported as OOM-killed.
 	oomSeen map[string]bool
+	// oomTrack (R-636) remembers, per container run, the kill counter over the last 30 minutes and
+	// whether the ONE storm alarm of that run has gone.
+	oomTrack map[string]*oomRun
+	// nowFn is a test clock (nil → time.Now).
+	nowFn func() time.Time
 	// appDown tracks which deployed apps are currently in the DOWN state so app_start_failed fires
 	// ONCE per running→down transition, not every health cycle (fix-3 anti-spam). In-memory: a
 	// controller restart re-notifies once (acceptable — better than missing). The hub owns the real
@@ -359,7 +364,7 @@ func (n *Notifier) pushEventBoth(eventType, severity, message, messageCustomer s
 // Detects both degradation (ok→warn, ok→fail, warn→fail) and recovery (fail→ok, warn→ok, fail→warn).
 func (n *Notifier) NotifyHealthChange(status string, issues, warnings []string) {
 	if !n.enabled {
-		n.dropped("health_change", status)
+		n.dropped("health_change", healthSeverity(status)) // R-647: the severity, not the health status
 		return
 	}
 
@@ -714,21 +719,108 @@ func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
 // limit (Docker State.OOMKilled). Fires ONCE per container run (keyed by StartedAt), so a container
 // that stays OOM-marked does not repeat. "warning" — the hub vocabulary (R-329). Operator-only
 // hub-side (hub >= v0.114.0 registers it): the household sees the dashboard tag.
-func (n *Notifier) NotifyAppOOM(stack, container, startedAt string) {
+func (n *Notifier) NotifyAppOOM(stack, container, startedAt string, kills int64, memLimit, peak string) {
 	key := container + "|" + startedAt
 	n.mu.Lock()
 	if n.oomSeen == nil {
 		n.oomSeen = map[string]bool{}
 	}
-	if n.oomSeen[key] {
-		n.mu.Unlock()
-		return
-	}
+	first := !n.oomSeen[key]
 	n.oomSeen[key] = true
+	storm, inWindow := n.oomStormLocked(key, startedAt, kills)
 	n.mu.Unlock()
-	n.emit("app_oom", "warning",
-		fmt.Sprintf("Alkalmazás memóriája elfogyott: %s (%s) — egy folyamatát a memóriakorlát leállította", stack, container),
-		AppDetails{StackName: stack, DisplayName: container})
+	if first {
+		n.emit("app_oom", "warning",
+			fmt.Sprintf("Alkalmazás memóriája elfogyott: %s (%s) — egy folyamatát a memóriakorlát leállította", stack, container),
+			AppDetails{StackName: stack, DisplayName: container})
+	}
+	if storm {
+		n.logger.Printf("[ERROR] [notify] %s: container %s OOM STORM — %d kills in %d min (limit %s, peak %s)",
+			stack, container, inWindow, oomStormWindowMin, memLimit, peak)
+		n.emit("app_oom_storm", "error",
+			fmt.Sprintf("Alkalmazás memóriája ismételten elfogy: %s (%s) — %d leállítás %d percen belül (korlát %s, csúcs %s)",
+				stack, container, inWindow, oomStormWindowMin, memLimit, peak),
+			OOMStormDetails{App: stack, StackName: stack, Container: container, Kills: inWindow,
+				WindowMin: oomStormWindowMin, MemLimit: memLimit, Peak: peak})
+	}
+}
+
+// R-636 (v0.265.0) — ONE louder alarm per container run when OOM kills keep coming.
+//
+// The once-per-run app_oom stays exactly as it was (it is what stops a crash loop from mailing 4,530
+// times, R-629). Beside it: when the kernel's own kill counter for the SAME run rises by
+// oomStormKills or more within oomStormWindowMin minutes, app_oom_storm goes once, at `error`,
+// operator-only. RomM's real rate on 2026-09-22 was 4,530 kills in 6 h ≈ 375 per 30 min; one hiccup
+// is 1–3. Twenty in thirty minutes sits far from both. Pinned by TestR636_*.
+const (
+	oomStormKills     = 20
+	oomStormWindowMin = 30
+)
+
+type oomSample struct {
+	at    time.Time
+	kills int64
+}
+
+type oomRun struct {
+	samples   []oomSample
+	stormSent bool
+}
+
+// OOMStormDetails is the app_oom_storm payload. stack_name lets the hub's per-app cooldown tell two
+// storming apps apart.
+type OOMStormDetails struct {
+	App       string `json:"app"`
+	StackName string `json:"stack_name"`
+	Container string `json:"container"`
+	Kills     int64  `json:"kills"`
+	WindowMin int    `json:"window_min"`
+	MemLimit  string `json:"mem_limit,omitempty"`
+	Peak      string `json:"peak,omitempty"`
+}
+
+func (n *Notifier) now() time.Time {
+	if n.nowFn != nil {
+		return n.nowFn()
+	}
+	return time.Now()
+}
+
+// oomStormLocked records one reading and says whether the storm alarm fires now. Caller holds n.mu.
+// A run seen for the first time within the window of its own start gets a zero reading AT its start:
+// the counter began at zero there, so a run that is already storming when the controller first looks
+// is not made to wait another thirty minutes.
+func (n *Notifier) oomStormLocked(key, startedAt string, kills int64) (bool, int64) {
+	if kills < 0 {
+		return false, 0 // unreadable: never escalate on a guess
+	}
+	if n.oomTrack == nil {
+		n.oomTrack = map[string]*oomRun{}
+	}
+	now := n.now()
+	window := time.Duration(oomStormWindowMin) * time.Minute
+	r := n.oomTrack[key]
+	if r == nil {
+		r = &oomRun{}
+		if t, err := time.Parse(time.RFC3339Nano, startedAt); err == nil && now.Sub(t) <= window {
+			r.samples = append(r.samples, oomSample{at: t, kills: 0})
+		}
+		n.oomTrack[key] = r
+	}
+	r.samples = append(r.samples, oomSample{at: now, kills: kills})
+	keep := r.samples[:0]
+	for _, s := range r.samples {
+		if now.Sub(s.at) <= window {
+			keep = append(keep, s)
+		}
+	}
+	r.samples = keep
+	inWindow := kills - r.samples[0].kills
+	if r.stormSent || inWindow < oomStormKills {
+		return false, inWindow
+	}
+	r.stormSent = true
+	return true, inWindow
 }
 
 // DiskHealthDetails is the event-detail payload for disk_health_degraded.
@@ -1211,3 +1303,15 @@ func (n *Notifier) NotifyAppUpdateHeld(d AppUpdateDetails, sentence func(lang st
 	}
 	n.pushEventBoth("app_update_held", "error", hu, household, d)
 }
+
+// healthSeverity is the event severity a health status would be sent at (R-647): the disabled path
+// names what it drops, and "warn" is a health STATUS, not a severity the hub knows.
+func healthSeverity(status string) string {
+	switch status {
+	case "fail":
+		return "error"
+	case "warn":
+		return "warning"
+	}
+	return "info"
+}