diff --git a/documentation/architecture/09-update-architecture.md b/documentation/architecture/09-update-architecture.md index a549510c..943a040f 100644 --- a/documentation/architecture/09-update-architecture.md +++ b/documentation/architecture/09-update-architecture.md @@ -329,6 +329,18 @@ builds them (`audits/update-rulings-2026-09-23/`). operator-only). The operator's English was used with one word dropped („please" — the house rule, `i18n_missing_gate.py`); the Hungarian verbatim, its formal register recorded against R-516. +### 2026-09-24 (afternoon) — three more operator rulings + +26. **R-661, option A: one action brings a file app back WHOLE from the second drive** — the unit (settings + + database) and the drive files together. The second drive then counts as a whole copy in decision 25's + truth table. R-538's guard stays: this is a new action beside it, not its removal. +27. **R-666, option B: while a held app's page says support is informed, Remove offers only „remove the app, + keep my data".** The household keeps control; the data stays. The no-whole-copy sentence moves to the + informal voice, like every other screen. +28. **An app in a crash loop, or in an out-of-memory storm, is STOPPED by the box, and the household and the + operator are told. A press on Start gives it one more try.** Operator's words: *"if it is in a crashloop, + or consuming resources, then yes, definitely stopped at least."* + **RomM follow-ups, operator-agreed the same day:** the test bench watches memory after an update (`upgrade-test.py`, 2026-09-23); a version move checks the memory limit (gate or checklist — §6.4); R-636's louder repeated alarm. diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 4641ffa1..ac05a48c 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,18 @@ +## v0.123.0 — `app_stopped_unhealthy`: the household is told when the box stops a crashing app (2026-09-24, `09` §3 decision 28) + +Controller v0.269.0 stops an app in a crash loop (≥ 6 restarts in 10 min) or an out-of-memory storm (≥ 20 +kernel OOM kills in 30 min) and sends `app_stopped_unhealthy` (warning), details `app`, `stack_name`, `kind`, +`count`, `window_min`, `trip` (2 = a repeat within 24 h, the household sentence then says support is told). + +- **A household event:** `allowedEventTypes`; `mail.event.app_stopped_unhealthy` in hu AND en (app named); + `appNamedMailEvents`; `defaultSeedEvents`; the one-time ADD-ONLY `seed_app_stopped_unhealthy_v1` for existing + households (the v0.120.0 shape — the box pushes its list on every notifications-page save, and seeds the same + type add-only on its side). +- **Per-app cooldown on both legs** (`perAppCooldownEvents`, `perAppCustomerCooldownEvents`; the R-389 fence test + widened on purpose). +- Pinned by `TestAppStoppedUnhealthyIsAHouseholdEvent` (red-proof: dropped from `allowedEventTypes` → fails). + Mail goldens captured for the new type (hu, en). + ## v0.122.0 — `app_hold_no_whole_copy`: a stranded held app reaches support (2026-09-24, R-659) Controller v0.268.0 sends `app_hold_no_whole_copy` (severity `critical`) when an update AND its diff --git a/hub/internal/api/chaosnight_events_test.go b/hub/internal/api/chaosnight_events_test.go index 46166327..1d582235 100644 --- a/hub/internal/api/chaosnight_events_test.go +++ b/hub/internal/api/chaosnight_events_test.go @@ -92,3 +92,22 @@ func TestAppHoldNoWholeCopyIsAllowlistedAndOperatorOnly(t *testing.T) { t.Fatalf("%s must be operator-only — the household is told by app_update_held; this carries support's detail", et) } } + +// Decision 28 (hub v0.123.0) — app_stopped_unhealthy is a HOUSEHOLD event: allow-listed, NOT operator-only, +// and it has a Hungarian AND English mail entry (never the raw-English fallback). +// RED-PROOF (REPORT.md): drop it from allowedEventTypes → "the controller's push would 400". +func TestAppStoppedUnhealthyIsAHouseholdEvent(t *testing.T) { + et := "app_stopped_unhealthy" + if !allowedEventTypes[et] { + t.Fatalf("%s must be in allowedEventTypes — the controller's push would 400", et) + } + if notify.IsOperatorOnly(et) { + t.Fatalf("%s must reach the household", et) + } + for _, lang := range []string{"hu", "en"} { + subject, _ := notify.FormatCustomerEmail(lang, "c1", et, "warning", "RAW-ENGLISH-SENTINEL", "", `{"stack_name":"gokapi"}`) + if strings.Contains(subject, "RAW-ENGLISH-SENTINEL") || !strings.Contains(subject, "gokapi:") { + t.Fatalf("%s/%s subject must be the mail entry naming the app, got %q", et, lang, subject) + } + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 4f68839a..f9874fbe 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -2019,6 +2019,9 @@ var allowedEventTypes = map[string]bool{ // operator; both carry a Hungarian AND English `mail.event.*` entry, never the raw-English fallback. "app_update_undone": true, "app_update_held": true, + // v0.123.0 (decision 28, controller v0.269.0): the box stopped an app that kept crashing / ran out + // of memory. Household + operator; hu AND en `mail.event.*`; per-app cooldown on both legs. + "app_stopped_unhealthy": true, // Controller-pushed events "controller_started": true, diff --git a/hub/internal/claim/engine.go b/hub/internal/claim/engine.go index 113700ff..211e6c2c 100644 --- a/hub/internal/claim/engine.go +++ b/hub/internal/claim/engine.go @@ -238,6 +238,9 @@ var defaultSeedEvents = []string{ // add-only store.SeedEventTypesOnce. "app_update_undone", "app_update_held", + // v0.123.0 (`09` §3 decision 28): the box stopped an app in a crash loop / OOM storm. Existing + // households get it by the one-time, add-only store.SeedEventTypesOnce. + "app_stopped_unhealthy", } // MarkClaimed records a controller-reported successful claim and sends the one-time confirmation diff --git a/hub/internal/i18n/locales/en.json b/hub/internal/i18n/locales/en.json index 63512f8c..dd195cc2 100644 --- a/hub/internal/i18n/locales/en.json +++ b/hub/internal/i18n/locales/en.json @@ -81,5 +81,6 @@ "mail.test.subject": "[Felhom] Test notification", "mail.test.body": "Dear Customer,\n\nThis is a test notification from the Felhom monitoring system.\nNotifications are working correctly.\n\nBest regards,\nFelhom.eu monitoring", "mail.event.app_update_undone": "%s: the update did not work; the app runs on its previous version", - "mail.event.app_update_held": "%s: the app is stopped and needs a restore" + "mail.event.app_update_held": "%s: the app is stopped and needs a restore", + "mail.event.app_stopped_unhealthy": "%s: the app was stopped because it kept crashing" } diff --git a/hub/internal/i18n/locales/hu.json b/hub/internal/i18n/locales/hu.json index bcc1337f..17cfdae0 100644 --- a/hub/internal/i18n/locales/hu.json +++ b/hub/internal/i18n/locales/hu.json @@ -81,5 +81,6 @@ "mail.test.subject": "[Felhom] Teszt értesítés", "mail.test.body": "Kedves Ügyfél!\n\nEz egy teszt értesítés a Felhom monitoring rendszerből.\nAz értesítések megfelelően működnek.\n\nÜdvözlettel,\nFelhom.eu monitoring", "mail.event.app_update_undone": "%s: a frissítés nem sikerült, az alkalmazás a korábbi változattal fut", - "mail.event.app_update_held": "%s: az alkalmazás leállítva, visszaállítás szükséges" + "mail.event.app_update_held": "%s: az alkalmazás leállítva, visszaállítás szükséges", + "mail.event.app_stopped_unhealthy": "%s: az alkalmazást leállítottuk, mert újra és újra összeomlott" } diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 46d1c7ad..3795433f 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -398,6 +398,8 @@ var perAppCooldownEvents = map[string]bool{ // v0.122.0 (R-659): a held app with no whole copy on its box is one app's event with no digest — // two apps stranded on one night are two alarms, and the second must not be swallowed. "app_hold_no_whole_copy": true, + // v0.123.0 (decision 28): one app's stop, no digest behind it. + "app_stopped_unhealthy": true, } // perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The @@ -407,6 +409,8 @@ var perAppCooldownEvents = map[string]bool{ var perAppCustomerCooldownEvents = map[string]bool{ "app_update_undone": true, "app_update_held": true, + // v0.123.0 (decision 28): two apps stopped on one night are two mails. + "app_stopped_unhealthy": true, } // cooldownStackSuffix returns ":"+stack_name when the event's details carry a non-empty `stack_name` diff --git a/hub/internal/notify/r389_cooldown_grain_test.go b/hub/internal/notify/r389_cooldown_grain_test.go index 27ff26f5..289b2bd7 100644 --- a/hub/internal/notify/r389_cooldown_grain_test.go +++ b/hub/internal/notify/r389_cooldown_grain_test.go @@ -70,7 +70,8 @@ func TestR389_StackSuffixIsAllowListedAndFailSoft(t *testing.T) { func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) { // v0.121.0 widened it again, on purpose (R-636): an OOM storm is one app's event with no digest. // v0.122.0 once more (R-659): a stranded held app is one app's event with no digest. - want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_start_failed", "app_update_held", "app_update_undone"} + // v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest. + want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"} ok := len(perAppCooldownEvents) == len(want) for _, w := range want { ok = ok && perAppCooldownEvents[w] @@ -80,7 +81,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) { for k := range perAppCooldownEvents { got = append(got, k) } - t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_start_failed app_update_held app_update_undone]. Adding a member is the "+ + t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+ "fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+ "disk sends one digest, not one mail per app. Read the fence before widening this.", got) } diff --git a/hub/internal/notify/templates.go b/hub/internal/notify/templates.go index f46ed192..03c2fce0 100644 --- a/hub/internal/notify/templates.go +++ b/hub/internal/notify/templates.go @@ -383,6 +383,8 @@ func trimRepeatedUsage(reason, targetPath string) string { var appNamedMailEvents = map[string]bool{ "app_update_undone": true, "app_update_held": true, + // v0.123.0 (`09` §3 decision 28): the box stopped an app that kept crashing / ran out of memory. + "app_stopped_unhealthy": true, } // stackNameOf reads `stack_name` from an event's details, "" when absent or unreadable. diff --git a/hub/internal/notify/testdata/mail_goldens/en/customer_event_app_stopped_unhealthy.txt b/hub/internal/notify/testdata/mail_goldens/en/customer_event_app_stopped_unhealthy.txt new file mode 100644 index 00000000..a39090b4 --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/en/customer_event_app_stopped_unhealthy.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Warning: ?: the app was stopped because it kept crashing +--- +Dear Customer, + +Your Felhom system sent the following notification: + +?: the app was stopped because it kept crashing + +Details: +- Server: demo-fixture +- Time: 2026-01-15 10:30 +- Level: Warning +- Type: app_stopped_unhealthy + +If you have any questions, contact your operator. + +Best regards, +Felhom.eu monitoring diff --git a/hub/internal/notify/testdata/mail_goldens/hu/customer_event_app_stopped_unhealthy.txt b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_app_stopped_unhealthy.txt new file mode 100644 index 00000000..2be9f00d --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_app_stopped_unhealthy.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Figyelmeztetés: ?: az alkalmazást leállítottuk, mert újra és újra összeomlott +--- +Kedves Ügyfél! + +A Felhom rendszered a következő értesítést küldte: + +?: az alkalmazást leállítottuk, mert újra és újra összeomlott + +Részletek: +- Szerver: demo-fixture +- Időpont: 2026-01-15 10:30 +- Szint: Figyelmeztetés +- Típus: app_stopped_unhealthy + +Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel. + +Üdvözlettel, +Felhom.eu monitoring diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index b213c03b..e1def2c0 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -801,6 +801,14 @@ func (s *Store) migrate() error { s.logger.Printf("[INFO] [store] app-update event types added to %d household(s)' notification prefs (one-time, add-only): %v", len(changed), changed) } + // v0.123.0 (`09` §3 decision 28): every existing household hears about an app the box stopped. + // ONE-TIME and ADD-ONLY — see SeedEventTypesOnce. + if changed, err := s.SeedEventTypesOnce("seed_app_stopped_unhealthy_v1", []string{"app_stopped_unhealthy"}); err != nil { + return fmt.Errorf("unhealthy-stop event seed: %w", err) + } else if changed != nil && s.logger != nil { + s.logger.Printf("[INFO] [store] app_stopped_unhealthy added to %d household(s)' notification prefs (one-time, add-only): %v", len(changed), changed) + } + return nil }