diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 98af6183..f42f45a1 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,17 @@ +## v0.121.0 — `app_oom_storm`: a repeating OOM problem reaches the operator louder (2026-09-23, R-636) + +Controller v0.265.0 sends `app_oom_storm` (severity `error`) once per container run when the kernel's +OOM-kill counter for the same run rises by 20+ within 30 minutes. `app_oom` (warning, once per run) is +unchanged. + +- **Allow-listed AND operator-only, in the same commit** (`allowedEventTypes` + `operatorOnlyEvents`): + the details carry raw container names and memory figures, and a type allow-listed but not + operator-only would reach a household as raw English (the v0.78.0 defect). Pinned together by + `TestAppOOMStormIsAllowlistedAndOperatorOnly`. +- **Per-app operator cooldown:** joined R-389's `perAppCooldownEvents` (the fence test widened on purpose, + with its reason) — two apps storming on one night are two alarms. +- Red-proofs: three (REPORT). + ## v0.120.0 — the household is TOLD when an update is undone or held (2026-09-23, `09` §3 decision 15) Controller v0.264.0 sends two new events when a guarded app update does not go through. This hub diff --git a/hub/internal/api/chaosnight_events_test.go b/hub/internal/api/chaosnight_events_test.go index 448cf145..03c91cf6 100644 --- a/hub/internal/api/chaosnight_events_test.go +++ b/hub/internal/api/chaosnight_events_test.go @@ -67,3 +67,16 @@ func TestAppUpdateOutcomesReachTheHousehold(t *testing.T) { } } } + +// R-636 (v0.121.0) — app_oom_storm is allow-listed AND operator-only, pinned together (allow-listed +// but not operator-only is the v0.78.0 defect: raw English to a household). +// RED-PROOF (REPORT.md): drop it from operatorOnlyEvents → "must be operator-only". +func TestAppOOMStormIsAllowlistedAndOperatorOnly(t *testing.T) { + et := "app_oom_storm" + if !allowedEventTypes[et] { + t.Fatalf("%s must be in allowedEventTypes — the controller's push would 400", et) + } + if !notify.IsOperatorOnly(et) { + t.Fatalf("%s must be operator-only — container names and memory figures are not a household's business", et) + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index a6441d38..e2e97de4 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -2140,6 +2140,9 @@ var allowedEventTypes = map[string]bool{ "pbsdr_adopted": true, "backup_tier_skipped": true, "app_oom": true, + // R-636 (hub v0.121.0, controller v0.265.0): the same container run OOM-killed 20+ times in 30 min. + // Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit. + "app_oom_storm": true, // Special "test": true, } diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index b3e737c5..1fca20e7 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -392,6 +392,9 @@ var perAppCooldownEvents = map[string]bool{ // v0.120.0: an update outcome is per app with no digest — two apps on one night are two alarms. "app_update_undone": true, "app_update_held": true, + // v0.121.0 (R-636): a storm is one app's event with no digest behind it — two apps storming on one + // night are two alarms. (The controller already sends it at most once per container run.) + "app_oom_storm": true, } // perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The @@ -652,6 +655,10 @@ var operatorOnlyEvents = map[string]bool{ // names — the household's side is the dashboard tag. Registered in the same commit. "backup_tier_skipped": true, "app_oom": true, + // R-636 (v0.121.0, controller v0.265.0). The LOUDER sibling of app_oom: the same container run + // OOM-killed 20+ times in 30 minutes. Same audience, same reason — raw container names and memory + // figures; the household's side is the dashboard. Registered in the same commit that mints it. + "app_oom_storm": true, } // IsOperatorOnly reports whether an event type is barred from customer dispatch. Exported so the diff --git a/hub/internal/notify/r389_cooldown_grain_test.go b/hub/internal/notify/r389_cooldown_grain_test.go index 7886c403..ba55eb0d 100644 --- a/hub/internal/notify/r389_cooldown_grain_test.go +++ b/hub/internal/notify/r389_cooldown_grain_test.go @@ -68,7 +68,8 @@ func TestR389_StackSuffixIsAllowListedAndFailSoft(t *testing.T) { // an update outcome is one app's event, with no digest behind it — two apps undone on one night are // two alarms. The backup family stays coarse, as the fence says. func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) { - want := []string{"app_start_failed", "app_update_held", "app_update_undone"} + // v0.121.0 widened it again, on purpose (R-636): an OOM storm is one app's event with no digest. + want := []string{"app_oom_storm", "app_start_failed", "app_update_held", "app_update_undone"} ok := len(perAppCooldownEvents) == len(want) for _, w := range want { ok = ok && perAppCooldownEvents[w] @@ -78,7 +79,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) { for k := range perAppCooldownEvents { got = append(got, k) } - t.Fatalf("perAppCooldownEvents = %v, want exactly [app_start_failed app_update_held app_update_undone]. Adding a member is the "+ + t.Fatalf("perAppCooldownEvents = %v, want exactly [app_oom_storm app_start_failed app_update_held app_update_undone]. Adding a member is the "+ "fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+ "disk sends one digest, not one mail per app. Read the fence before widening this.", got) }