hub v0.140.0: a failed operator mail is retried (1, 5, 15 min); a Docker set needs a passing memory-kill check on every ring-0 box (decision 157)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/notify/dispatcher.go

@@ -36,8 +36,19 @@ type Dispatcher struct {
 	// Resend custom headers — used for the high-priority nudge on error/critical mails (v0.71.0,
 	// audit F14-light).
 	sendEmailFn func(to, subject, textBody string, headers map[string]string) error
+
+	// afterFn schedules a delayed call (time.AfterFunc; a seam so tests run the retries at once). Used by
+	// retryOperatorEmail (the lost alarm of 2026-10-05).
+	afterFn func(time.Duration, func())
 }
 
+// operatorRetryDelays are the waits before each further try of an operator mail whose send FAILED (the lost alarm of 2026-10-05). Measured
+// 2026-10-05: demo-hp's whole_guest_backup_failed (error) hit a 10 s Resend client timeout, was logged `failed`, and was
+// never sent — the only operator mail of 692 that failed, and the one alarm the operator needed that night. The
+// cooldown is armed before the send, so without a retry a failed mail also silences the next one for the same key.
+// Pinned by TestOperatorMailRetry_FailedOperatorMailIsRetried.
+var operatorRetryDelays = []time.Duration{1 * time.Minute, 5 * time.Minute, 15 * time.Minute}
+
 // NewDispatcher creates a new notification dispatcher.
 func NewDispatcher(s *store.Store, resendAPIKey, fromEmail, operatorEmail string, operatorOn bool, logger *log.Logger) *Dispatcher {
 	d := &Dispatcher{
@@ -52,9 +63,31 @@ func NewDispatcher(s *store.Store, resendAPIKey, fromEmail, operatorEmail string
 		custCooldowns: make(map[string]time.Time),
 	}
 	d.sendEmailFn = d.sendEmail
+	d.afterFn = func(wait time.Duration, f func()) { time.AfterFunc(wait, f) }
 	return d
 }
 
+// retryOperatorEmail tries a failed operator mail again after operatorRetryDelays[attempt], then the next delay,
+// until one send succeeds or the delays run out. Each try is logged and recorded in the notification log (`sent`
+// with "after retry N", or `failed` with "retry N: …"); giving up is an ERROR line.
+func (d *Dispatcher) retryOperatorEmail(customerID, eventType, severity, message, subject, body string, headers map[string]string, attempt int) {
+	if attempt >= len(operatorRetryDelays) {
+		d.logger.Printf("[ERROR] Operator email for %s/%s GAVE UP after %d retries — it was never sent", customerID, eventType, attempt)
+		return
+	}
+	d.afterFn(operatorRetryDelays[attempt], func() {
+		n := attempt + 1
+		if err := d.sendEmailFn(d.operatorEmail, subject, body, headers); err != nil {
+			d.logger.Printf("[ERROR] Operator email retry %d failed for %s/%s: %v", n, customerID, eventType, err)
+			d.store.LogNotification(customerID, eventType, severity, message, "failed", fmt.Sprintf("retry %d: %v", n, err), "operator")
+			d.retryOperatorEmail(customerID, eventType, severity, message, subject, body, headers, n)
+			return
+		}
+		d.logger.Printf("[INFO] Operator email sent for %s/%s after retry %d", customerID, eventType, n)
+		d.store.LogNotification(customerID, eventType, severity, message, "sent", fmt.Sprintf("after retry %d", n), "operator")
+	})
+}
+
 // priorityHeaders returns the Resend custom headers that nudge mail clients toward attention for
 // error/critical mails (X-Priority + Importance; v0.71.0, audit F14-light: delivered ≠ noticed).
 // Everything else gets nil — a warning or info mail must NOT masquerade as urgent. Pure → tested.
@@ -573,9 +606,11 @@ func (d *Dispatcher) processOperator(customerID, eventType, severity, message, d
 
 	subject, body := FormatOperatorEmail(customerID, eventType, severity, message, detailsJSON)
 
-	if err := d.sendEmailFn(d.operatorEmail, subject, body, priorityHeaders(severity)); err != nil {
-		d.logger.Printf("[ERROR] Operator email failed for %s/%s: %v", customerID, eventType, err)
+	hdrs := priorityHeaders(severity)
+	if err := d.sendEmailFn(d.operatorEmail, subject, body, hdrs); err != nil {
+		d.logger.Printf("[ERROR] Operator email failed for %s/%s: %v — retrying in %v", customerID, eventType, err, operatorRetryDelays[0])
 		d.store.LogNotification(customerID, eventType, severity, message, "failed", err.Error(), "operator")
+		d.retryOperatorEmail(customerID, eventType, severity, message, subject, body, hdrs, 0)
 		return
 	}
 	d.logger.Printf("[INFO] Operator email sent for %s/%s", customerID, eventType)
@@ -689,11 +724,11 @@ var operatorOnlyEvents = map[string]bool{
 	// hub v0.133.0 (`11` §5.3.1): a TEST approval cancelled at a start without the override.
 	"os_release_cancelled": true,
 	// R-840 (hub v0.133.0): a box's root-owned config bundle behind the vouched one for 7 days.
-	"os_config_bundle_behind":    true,
+	"os_config_bundle_behind": true,
 	// hub v0.135.0: R-530 (a box behind the vouched agent for 7 days) and R-604 (a global floor raise that did not
 	// move every box). Fleet facts only the operator can act on — listed in the SAME commit that mints them.
-	"agent_behind":        true,
-	"floor_raise_skipped": true,
+	"agent_behind":               true,
+	"floor_raise_skipped":        true,
 	"os_update_settings_changed": true,
 	// R-841 (hub v0.131.0): the tunnel alarm — a box fact the household can do nothing about from inside.
 	"tunnel_down":      true,

hub/internal/notify/operator_mail_retry_test.go

@@ -0,0 +1,84 @@
+package notify
+
+import (
+	"errors"
+	"io"
+	"log"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The lost alarm of 2026-10-05 (fixed without a row, `audits/readback-2026-10-07/C/`): a failed operator mail is tried again (operatorRetryDelays) instead of being dropped. The consequence
+// asserted is that the mail is SENT and recorded `sent`.
+//
+// COMPANION RED-PROOF (REPORT): drop the retryOperatorEmail call from processOperator → "the failed mail was never
+// sent again".
+func TestOperatorMailRetry_FailedOperatorMailIsRetried(t *testing.T) {
+	st := newDispStore(t)
+	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
+	var mu sync.Mutex
+	tries, waits := 0, []time.Duration{}
+	d.afterFn = func(w time.Duration, f func()) { mu.Lock(); waits = append(waits, w); mu.Unlock(); f() }
+	d.sendEmailFn = func(string, string, string, map[string]string) error {
+		mu.Lock()
+		defer mu.Unlock()
+		tries++
+		if tries <= 2 { // the first send and the first retry time out
+			return errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")
+		}
+		return nil
+	}
+	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED", "", "box")
+	if tries != 3 {
+		t.Fatalf("the failed mail was never sent again (tries=%d)", tries)
+	}
+	if len(waits) != 2 || waits[0] != operatorRetryDelays[0] || waits[1] != operatorRetryDelays[1] {
+		t.Fatalf("retries not on the schedule: %v", waits)
+	}
+	rows, err := st.GetRecentNotifications("demo-hp", 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var sent, failed int
+	for _, r := range rows {
+		if r.Channel != "operator" {
+			continue
+		}
+		switch r.Status {
+		case "sent":
+			sent++
+		case "failed":
+			failed++
+		}
+	}
+	if sent != 1 || failed != 2 {
+		t.Fatalf("notification log: sent=%d failed=%d, want 1 and 2", sent, failed)
+	}
+}
+
+// The retries stop: a send that never succeeds is tried len(operatorRetryDelays) more times, then given up.
+func TestOperatorMailRetry_RetriesAreBounded(t *testing.T) {
+	st := newDispStore(t)
+	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
+	tries := 0
+	d.afterFn = func(_ time.Duration, f func()) { f() }
+	d.sendEmailFn = func(string, string, string, map[string]string) error { tries++; return errors.New("down") }
+	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "x", "", "box")
+	if tries != 1+len(operatorRetryDelays) {
+		t.Fatalf("tries=%d, want %d", tries, 1+len(operatorRetryDelays))
+	}
+}
+
+// A mail that is sent the first time is not retried.
+func TestOperatorMailRetry_SuccessIsNotRetried(t *testing.T) {
+	st := newDispStore(t)
+	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
+	scheduled := 0
+	d.afterFn = func(_ time.Duration, f func()) { scheduled++; f() }
+	d.sendEmailFn = func(string, string, string, map[string]string) error { return nil }
+	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "x", "", "box")
+	if scheduled != 0 {
+		t.Fatalf("a sent mail scheduled %d retries", scheduled)
+	}
+}