admin
331193b898
internal/quiesce had no route to the hub at all: three failed whole-guest backups on 2026-07-27 produced zero events. Hub half of the fix. whole_guest_backup_failed / _recovered are allowlisted with NO customerMessages entry. Deliberately not backup_failed/backup_completed — those have customer Hungarian templates AND sit in demo-felhom's live enabled_events, so reusing them would email the customer that their backup failed while it is still retrying behind the R-88 breaker. The recovery joins recoveredPairedDownTypes because it is severity info and severityNotifies drops info — otherwise the operator hears it break and never hears it heal. Its customer leg is pairing-gated and can never fire. Operator cooldown gains a per-tier dimension from the event details, so one tier cannot mask another for an hour. Narrow: empty suffix unless a tier is sent, so no existing event type changes.
81 lines
3.4 KiB
Go
81 lines
3.4 KiB
Go
package notify
|
|
|
|
import "testing"
|
|
|
|
// R-97a — the operator cooldown must not let one backup tier mask another.
|
|
//
|
|
// The cooldown was keyed `customerID + ":" + eventType`, which is right for every event that
|
|
// describes ONE thing. A whole-guest backup failure describes ONE TIER, and a box has two: with the
|
|
// old key, `felhom-pbs` failing at 09:00 swallowed `local` failing at 09:20 for the rest of the hour.
|
|
// The operator would hear about the offsite tier and never about the local one — the exact masking
|
|
// the per-tier signal exists to prevent.
|
|
|
|
func TestCooldownTierSuffix_SeparatesTiers(t *testing.T) {
|
|
pbs := cooldownTierSuffix(`{"tier":"felhom-pbs","error":"connection refused"}`)
|
|
local := cooldownTierSuffix(`{"tier":"local","error":"no space left"}`)
|
|
if pbs == local {
|
|
t.Fatalf("two tiers must produce DIFFERENT cooldown suffixes, both gave %q — one tier would mask the other", pbs)
|
|
}
|
|
if pbs != ":felhom-pbs" || local != ":local" {
|
|
t.Fatalf("suffix should be the tier: got %q and %q", pbs, local)
|
|
}
|
|
}
|
|
|
|
// NARROWNESS IS THE POINT: every event type that does not send a tier must keep its old key exactly,
|
|
// or this becomes a notification flood instead of a fix. Widening the key for everything would turn
|
|
// one hourly app_start_failed into one per app.
|
|
func TestCooldownTierSuffix_EmptyForEverythingElse(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
details string
|
|
}{
|
|
{"no details at all", ""},
|
|
{"details without a tier", `{"error":"boom","drive_count":3}`},
|
|
{"empty tier value", `{"tier":""}`},
|
|
{"malformed json", `{{{not json`},
|
|
{"tier mentioned in a STRING, not as a key", `{"error":"the tier: felhom-pbs is down"}`},
|
|
{"null details", `null`},
|
|
}
|
|
for _, c := range cases {
|
|
if got := cooldownTierSuffix(c.details); got != "" {
|
|
t.Errorf("%s: suffix must be EMPTY so the existing cooldown is unchanged, got %q", c.name, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The recovery must reach the operator despite being severity "info".
|
|
//
|
|
// severityNotifies drops "info", so routing whole_guest_backup_recovered normally would STORE the
|
|
// event and never mail it — the operator would be told the tier broke and never told it healed.
|
|
// Membership in recoveredPairedDownTypes puts it on the recovery branch, which runs BEFORE the
|
|
// severity gate.
|
|
func TestWholeGuestRecovery_IsOnTheRecoveryBranch(t *testing.T) {
|
|
if severityNotifies("info") {
|
|
t.Fatal("premise changed: info now notifies, so the pairing entry may be unnecessary — re-check")
|
|
}
|
|
paired, ok := recoveredPairedDownTypes["whole_guest_backup_recovered"]
|
|
if !ok {
|
|
t.Fatal("whole_guest_backup_recovered must be on the recovery branch, or its 'info' severity makes it silent")
|
|
}
|
|
found := false
|
|
for _, p := range paired {
|
|
if p == "whole_guest_backup_failed" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("the recovery must pair with whole_guest_backup_failed; got %v", paired)
|
|
}
|
|
}
|
|
|
|
// The customer must NOT hear either edge — a customer can take no action on a failed whole-guest
|
|
// backup, and being told it failed while it is retrying is alarming without being actionable.
|
|
func TestWholeGuestBackup_IsNeverCustomerRouted(t *testing.T) {
|
|
for _, et := range []string{"whole_guest_backup_failed", "whole_guest_backup_recovered"} {
|
|
if msg, ok := customerMessages[et]; ok {
|
|
t.Fatalf("%s must have NO customerMessages entry (operator-tier only); found %q — "+
|
|
"adding one silently starts emailing customers about a backup they cannot act on", et, msg)
|
|
}
|
|
}
|
|
}
|