hub v0.78.0 — R-97a: whole-guest backup events, operator-only

internal/quiesce had no route to the hub at all: three failed whole-guest backups
on 2026-07-27 produced zero events. Hub half of the fix.

whole_guest_backup_failed / _recovered are allowlisted with NO customerMessages
entry. Deliberately not backup_failed/backup_completed — those have customer
Hungarian templates AND sit in demo-felhom's live enabled_events, so reusing them
would email the customer that their backup failed while it is still retrying
behind the R-88 breaker.

The recovery joins recoveredPairedDownTypes because it is severity info and
severityNotifies drops info — otherwise the operator hears it break and never
hears it heal. Its customer leg is pairing-gated and can never fire.

Operator cooldown gains a per-tier dimension from the event details, so one tier
cannot mask another for an hour. Narrow: empty suffix unless a tier is sent, so
no existing event type changes.

hub/internal/notify/cooldown_tier_test.go

@@ -0,0 +1,80 @@
+package notify
+
+import "testing"
+
+// R-97a — the operator cooldown must not let one backup tier mask another.
+//
+// The cooldown was keyed `customerID + ":" + eventType`, which is right for every event that
+// describes ONE thing. A whole-guest backup failure describes ONE TIER, and a box has two: with the
+// old key, `felhom-pbs` failing at 09:00 swallowed `local` failing at 09:20 for the rest of the hour.
+// The operator would hear about the offsite tier and never about the local one — the exact masking
+// the per-tier signal exists to prevent.
+
+func TestCooldownTierSuffix_SeparatesTiers(t *testing.T) {
+	pbs := cooldownTierSuffix(`{"tier":"felhom-pbs","error":"connection refused"}`)
+	local := cooldownTierSuffix(`{"tier":"local","error":"no space left"}`)
+	if pbs == local {
+		t.Fatalf("two tiers must produce DIFFERENT cooldown suffixes, both gave %q — one tier would mask the other", pbs)
+	}
+	if pbs != ":felhom-pbs" || local != ":local" {
+		t.Fatalf("suffix should be the tier: got %q and %q", pbs, local)
+	}
+}
+
+// NARROWNESS IS THE POINT: every event type that does not send a tier must keep its old key exactly,
+// or this becomes a notification flood instead of a fix. Widening the key for everything would turn
+// one hourly app_start_failed into one per app.
+func TestCooldownTierSuffix_EmptyForEverythingElse(t *testing.T) {
+	cases := []struct {
+		name    string
+		details string
+	}{
+		{"no details at all", ""},
+		{"details without a tier", `{"error":"boom","drive_count":3}`},
+		{"empty tier value", `{"tier":""}`},
+		{"malformed json", `{{{not json`},
+		{"tier mentioned in a STRING, not as a key", `{"error":"the tier: felhom-pbs is down"}`},
+		{"null details", `null`},
+	}
+	for _, c := range cases {
+		if got := cooldownTierSuffix(c.details); got != "" {
+			t.Errorf("%s: suffix must be EMPTY so the existing cooldown is unchanged, got %q", c.name, got)
+		}
+	}
+}
+
+// The recovery must reach the operator despite being severity "info".
+//
+// severityNotifies drops "info", so routing whole_guest_backup_recovered normally would STORE the
+// event and never mail it — the operator would be told the tier broke and never told it healed.
+// Membership in recoveredPairedDownTypes puts it on the recovery branch, which runs BEFORE the
+// severity gate.
+func TestWholeGuestRecovery_IsOnTheRecoveryBranch(t *testing.T) {
+	if severityNotifies("info") {
+		t.Fatal("premise changed: info now notifies, so the pairing entry may be unnecessary — re-check")
+	}
+	paired, ok := recoveredPairedDownTypes["whole_guest_backup_recovered"]
+	if !ok {
+		t.Fatal("whole_guest_backup_recovered must be on the recovery branch, or its 'info' severity makes it silent")
+	}
+	found := false
+	for _, p := range paired {
+		if p == "whole_guest_backup_failed" {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("the recovery must pair with whole_guest_backup_failed; got %v", paired)
+	}
+}
+
+// The customer must NOT hear either edge — a customer can take no action on a failed whole-guest
+// backup, and being told it failed while it is retrying is alarming without being actionable.
+func TestWholeGuestBackup_IsNeverCustomerRouted(t *testing.T) {
+	for _, et := range []string{"whole_guest_backup_failed", "whole_guest_backup_recovered"} {
+		if msg, ok := customerMessages[et]; ok {
+			t.Fatalf("%s must have NO customerMessages entry (operator-tier only); found %q — "+
+				"adding one silently starts emailing customers about a backup they cannot act on", et, msg)
+		}
+	}
+}

hub/internal/notify/dispatcher.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -70,6 +71,16 @@ func priorityHeaders(severity string) map[string]string {
 var recoveredPairedDownTypes = map[string][]string{
 	"node_recovered": {"node_stale", "node_down"},
 	"host_recovered": {"host_stale", "host_down"},
+	// R-97a. This branch runs BEFORE the severity gate, which is exactly why the recovery belongs
+	// here: `whole_guest_backup_recovered` is severity "info", and severityNotifies drops "info", so
+	// routing it normally would store the event and never mail it — the operator would be told the
+	// tier broke and never told it healed, which is the half of Scenario B that matters.
+	//
+	// The customer leg needs no special handling: it is PAIRING-gated on a customer-channel "sent"
+	// row for the down type, and `whole_guest_backup_failed` has no customerMessages entry and is in
+	// nobody's enabled_events — so no such row can exist, and the customer correctly hears neither
+	// edge. Operator hears both.
+	"whole_guest_backup_recovered": {"whole_guest_backup_failed"},
 }
 
 // severityNotifies reports whether a severity triggers email notifications. warning / error / critical
@@ -225,12 +236,36 @@ func (d *Dispatcher) processRecovery(customerID, eventType, severity, message, d
 	d.store.LogNotification(customerID, eventType, severity, message, "sent", "", "customer")
 }
 
+// cooldownTierSuffix returns ":"+tier when the event's details carry a non-empty `tier`, else "".
+//
+// R-97a. The operator cooldown was keyed `customerID + ":" + eventType` alone, which is correct for
+// every event that describes ONE thing — but a whole-guest backup failure describes ONE TIER, and a
+// box has two. `felhom-pbs` failing at 09:00 would swallow `local` failing at 09:20 for the whole
+// hour, so the operator would be told about the offsite tier and never about the local one. That is
+// precisely the masking the per-tier signal exists to prevent.
+//
+// NARROW ON PURPOSE: the suffix is empty unless the producer opts in by sending a `tier`, so no
+// existing event type's cooldown behaviour changes. Widening the key for everything would, e.g.,
+// turn one hourly `app_start_failed` into one per app — a flood, not a fix.
+func cooldownTierSuffix(detailsJSON string) string {
+	if detailsJSON == "" || !strings.Contains(detailsJSON, "\"tier\"") {
+		return ""
+	}
+	var d struct {
+		Tier string `json:"tier"`
+	}
+	if err := json.Unmarshal([]byte(detailsJSON), &d); err != nil || d.Tier == "" {
+		return ""
+	}
+	return ":" + d.Tier
+}
+
 func (d *Dispatcher) processOperator(customerID, eventType, severity, message, detailsJSON, source string) {
 	if !d.operatorOn || d.operatorEmail == "" {
 		return
 	}
 
-	cooldownKey := customerID + ":" + eventType
+	cooldownKey := customerID + ":" + eventType + cooldownTierSuffix(detailsJSON)
 	d.mu.Lock()
 	if last, ok := d.opCooldowns[cooldownKey]; ok && time.Since(last) < 1*time.Hour {
 		d.mu.Unlock()