package notify import ( "strings" "testing" ) // R-389 — the operator cooldown named the event TYPE and not the APP, so only the first broken app // per hour was ever mailed. // // THE DEFECT. `processOperator` keys the 1-hour cooldown on // `customerID:eventType[:tier][:run_id]`. None of those name an app. Measured live on `demo-hp` // 2026-08-23: `bookstack` alarmed at 09:27:51 and was `sent`; `privatebin` alarmed four minutes // later and was logged `suppressed — operator cooldown 1h, key=demo-hp:app_start_failed`. Three apps // dying together produce one mail. // // THE LAYER. These sit at the key builder and at `processOperator`. The key is where the collapse // happens, and the stored notification rows are where it is visible — asserting only that the suffix // function returns a string would repeat the "mechanism pinned, consequence unpinned" mistake. // // RED-PROOF (observed, see REPORT.md): drop `cooldownStackSuffix` from the key expression in // `processOperator` and TestR389_TwoAppsInsideTheHourBothReachTheOperator fails with // `2 apps down inside the hour produced 1 operator mail(s), want 2`. // --- the suffix itself --------------------------------------------------------------------------- func TestR389_StackSuffixIsAllowListedAndFailSoft(t *testing.T) { const appDetails = `{"stack_name":"bookstack","display_name":"BookStack"}` cases := []struct { name string eventType string details string want string }{ {"the allow-listed type gets the app", "app_start_failed", appDetails, ":bookstack"}, // THE FENCE. crossdrive_failed is severity `error`, reaches the operator leg, and carries // stack_name through CrossDriveDetails — a payload-shape rule would have split it per app and // silently undone R-97a/R-182. {"crossdrive_failed is NOT split per app", "crossdrive_failed", `{"stack_name":"bookstack","method":"rsync"}`, ""}, {"app_deployed is not in the register", "app_deployed", appDetails, ""}, {"app_removed is not in the register", "app_removed", appDetails, ""}, {"backup_failed is not in the register", "backup_failed", appDetails, ""}, // Fail-soft: a degraded payload must fall back to today's key, never panic, never drop. {"empty details", "app_start_failed", "", ""}, {"no stack_name key", "app_start_failed", `{"display_name":"BookStack"}`, ""}, {"empty stack_name", "app_start_failed", `{"stack_name":""}`, ""}, {"malformed JSON that still contains the token", "app_start_failed", `{"stack_name":`, ""}, {"null details", "app_start_failed", "null", ""}, {"array instead of object", "app_start_failed", `["stack_name"]`, ""}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := cooldownStackSuffix(tc.eventType, tc.details); got != tc.want { t.Fatalf("cooldownStackSuffix(%q, %q) = %q, want %q", tc.eventType, tc.details, got, tc.want) } }) } } // The register must stay narrow. A new entry is a deliberate act and should fail this until // someone changes it on purpose, having read the fence. // // v0.120.0 widened it ON PURPOSE, by the brief "the undo reaches the fleet" (`09` §3 decision 15): // an update outcome is one app's event, with no digest behind it — two apps undone on one night are // two alarms. The backup family stays coarse, as the fence says. func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) { // v0.121.0 widened it again, on purpose (R-636): an OOM storm is one app's event with no digest. // v0.122.0 once more (R-659): a stranded held app is one app's event with no digest. // v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest. want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"} ok := len(perAppCooldownEvents) == len(want) for _, w := range want { ok = ok && perAppCooldownEvents[w] } if !ok { var got []string for k := range perAppCooldownEvents { got = append(got, k) } t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+ "fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+ "disk sends one digest, not one mail per app. Read the fence before widening this.", got) } } // --- Scenario C: the absence claim, WITH its positive control ------------------------------------ // // "No other event type's key changed" is an absence claim. The control below proves this test can // SEE a key change first — otherwise a broken key builder would make every row look unchanged and // the test would pass forever. func TestR389_NoOtherEventTypeKeyChanges(t *testing.T) { // The v0.107.0 key expression, modelled inline. This is the BEFORE value, and modelling it here // rather than reading it from git is deliberate: the comparison must survive the file moving. oldKey := func(customerID, eventType, details string) string { return customerID + ":" + eventType + cooldownTierSuffix(details) + cooldownRunSuffix(details) } newKey := func(customerID, eventType, details string) string { return customerID + ":" + eventType + cooldownTierSuffix(details) + cooldownRunSuffix(details) + cooldownStackSuffix(eventType, details) } // POSITIVE CONTROL FIRST: the pair must be able to differ at all. if oldKey("c1", "app_start_failed", `{"stack_name":"bookstack"}`) == newKey("c1", "app_start_failed", `{"stack_name":"bookstack"}`) { t.Fatal("the control failed: old and new key agree even for the allow-listed type, so this " + "test cannot see a key change and its 'unchanged' verdicts below would be worthless") } // Every other type — including the ones that carry stack_name — must be byte-identical. for _, tc := range []struct{ eventType, details string }{ {"crossdrive_failed", `{"stack_name":"bookstack","method":"rsync"}`}, {"crossdrive_completed", `{"stack_name":"docmost"}`}, {"app_deployed", `{"stack_name":"bookstack","display_name":"BookStack"}`}, {"app_removed", `{"stack_name":"bookstack"}`}, {"backup_failed", `{"error":"boom"}`}, {"backup_run_failures", `{"run_id":"r-42"}`}, {"whole_guest_backup_failed", `{"tier":"felhom-pbs"}`}, {"db_dump_failed", `{"stack_name":"docmost"}`}, {"disk_warning", `{"path":"/mnt/data"}`}, {"expected_backup_missed", `{"tier":"offsite","run_id":"r-9"}`}, {"storage_disconnected", `{}`}, {"health_critical", ""}, } { o := oldKey("demo-hp", tc.eventType, tc.details) n := newKey("demo-hp", tc.eventType, tc.details) if o != n { t.Errorf("%s: cooldown key CHANGED\n v0.107.0: %s\n v0.108.0: %s\n"+ "Only app_start_failed may move. Splitting a backup-family key per app undoes R-97a "+ "and R-182 — twenty mails where one digest belongs.", tc.eventType, o, n) } } } // --- the CONSEQUENCE, asserted from the stored notification rows -------------------------------- // appEvent builds the details payload the controller actually sends for app_start_failed. func appEvent(stack string) string { return `{"stack_name":"` + stack + `","display_name":"` + strings.ToUpper(stack[:1]) + stack[1:] + `"}` } // operatorMails returns the operator-channel rows for a customer, newest first. func operatorMails(t *testing.T, d *Dispatcher, customerID string) (sent, suppressed int, rows []string) { t.Helper() entries, err := d.store.GetRecentNotifications(customerID, 50) if err != nil { t.Fatal(err) } for _, e := range entries { if e.Channel != "operator" || e.EventType != "app_start_failed" { continue } rows = append(rows, e.Status+" | "+e.Message+" | "+e.ErrorMessage) switch e.Status { case "sent": sent++ case "suppressed": suppressed++ } } return sent, suppressed, rows } // SCENARIO A — two apps go down inside the hour. Both must reach the operator. // // This is R-389's whole point, and it asserts the STORED rows, not a return value. func TestR389_TwoAppsInsideTheHourBothReachTheOperator(t *testing.T) { st := opOnlyStore(t) rec := &sentTo{} d := opOnlyDispatcher(t, st, rec) // POSITIVE CONTROL: the operator leg must be able to deliver at all in this configuration, // before any count below means anything. (Yesterday's live run could not prove the customer leg // because no address was set — do not repeat that shape.) d.ProcessEvent("c1", "app_start_failed", "warning", "Telepített alkalmazás nem fut: BookStack", appEvent("bookstack"), "controller") if sent, _, rows := operatorMails(t, d, "c1"); sent != 1 { t.Fatalf("control failed: the FIRST app produced %d sent operator row(s), want 1 — the "+ "operator leg is not delivering here, so the counts below would be meaningless. rows=%v", sent, rows) } // A different app, same hour, same event type. d.ProcessEvent("c1", "app_start_failed", "warning", "Telepített alkalmazás nem fut: PrivateBin", appEvent("privatebin"), "controller") sent, suppressed, rows := operatorMails(t, d, "c1") if sent != 2 { t.Fatalf("2 apps down inside the hour produced %d operator mail(s), want 2 "+ "(suppressed=%d). This is R-389: the second app's alarm took the first app's cooldown "+ "slot.\nrows: %v", sent, suppressed, rows) } if suppressed != 0 { t.Errorf("a DIFFERENT app was suppressed: %v", rows) } // And the addresses actually attempted — two distinct deliveries, not one row written twice. opCount := 0 for _, to := range rec.to { if to == "operator@felhom.eu" { opCount++ } } if opCount != 2 { t.Errorf("the dispatcher attempted %d operator send(s), want 2 — a stored row without a send "+ "attempt would be a record of something that did not happen", opCount) } } // SCENARIO B — the SAME app twice inside the hour. The hour is unchanged: one mail, one suppression. func TestR389_SameAppTwiceInsideTheHourIsStillSuppressed(t *testing.T) { st := opOnlyStore(t) rec := &sentTo{} d := opOnlyDispatcher(t, st, rec) for i := 0; i < 2; i++ { d.ProcessEvent("c1", "app_start_failed", "warning", "Telepített alkalmazás nem fut: BookStack", appEvent("bookstack"), "controller") } sent, suppressed, rows := operatorMails(t, d, "c1") if sent != 1 || suppressed != 1 { t.Fatalf("the same app twice gave sent=%d suppressed=%d, want 1 and 1 — R-389 changes the "+ "GRAIN, not the hour, and re-alarming the same app is the flood the cooldown exists to "+ "stop.\nrows: %v", sent, suppressed, rows) } // The suppression must still name the key, which is what made R-389 findable at all. if !strings.Contains(rows[0]+rows[1], "bookstack") { t.Errorf("the suppression row does not name the app in its key — that visibility is R-182's "+ "contribution and is how this defect was found: %v", rows) } } // SCENARIO D — details missing or malformed. The key degrades to today's and the mail STILL GOES. func TestR389_DegradedDetailsStillDeliver(t *testing.T) { for _, details := range []string{"", "null", `{}`, `{"stack_name":""}`, `{"stack_name":`} { st := opOnlyStore(t) rec := &sentTo{} d := opOnlyDispatcher(t, st, rec) d.ProcessEvent("c1", "app_start_failed", "warning", "Telepített alkalmazás nem fut", details, "controller") sent, _, rows := operatorMails(t, d, "c1") if sent != 1 { t.Errorf("details %q: %d operator mail(s), want 1 — a degraded payload must fall back to "+ "the old key and still deliver. Losing an alarm is worse than mis-routing one. rows=%v", details, sent, rows) } } } // A backup-family event carrying stack_name must still collapse — the coarse grain is the design. func TestR389_CrossdriveStillCollapsesPerHour(t *testing.T) { st := opOnlyStore(t) rec := &sentTo{} d := opOnlyDispatcher(t, st, rec) for _, stack := range []string{"bookstack", "docmost", "privatebin"} { d.ProcessEvent("c1", "crossdrive_failed", "error", "Másodlagos mentés sikertelen: "+stack, `{"stack_name":"`+stack+`","method":"rsync"}`, "controller") } entries, err := d.store.GetRecentNotifications("c1", 50) if err != nil { t.Fatal(err) } sent := 0 for _, e := range entries { if e.Channel == "operator" && e.EventType == "crossdrive_failed" && e.Status == "sent" { sent++ } } if sent != 1 { t.Fatalf("three crossdrive_failed events produced %d operator mail(s), want 1 — this family's "+ "cooldown is coarse ON PURPOSE (R-97a, R-182), and it carries stack_name, so a global "+ "suffix would have split it into three", sent) } }