package notify import ( "errors" "io" "log" "strings" "sync" "testing" "time" ) type sentMail struct{ to, subject, body string } func lostMailDispatcher(t *testing.T) (*Dispatcher, *[]sentMail, *bool) { t.Helper() st := newDispStore(t) d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0)) var mu sync.Mutex sent, down := &[]sentMail{}, new(bool) d.sendEmailFn = func(to, subject, body string, _ map[string]string) error { mu.Lock() defer mu.Unlock() if *down { return errors.New("context deadline exceeded") } *sent = append(*sent, sentMail{to, subject, body}) return nil } return d, sent, down } // R-1383 (D78), the case the ruling names: an operator mail failed, and the hub RESTARTED before a retry worked — the // retry timers are memory, so nothing sent it. The stored effect asserted: at the next start ONE operator mail goes // out that carries the lost mail's own sentence, one event row is left, and a second start tells nothing again. // A mail that failed and then went out on a retry is NOT lost. No real mail service: sendEmailFn is the repo's fake. // COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-lost-mail.txt). func TestLostOperatorMail_ToldAtTheNextStart(t *testing.T) { d, sent, down := lostMailDispatcher(t) // 1. a mail that fails, and the "restart" eats its retry timers *down = true d.afterFn = func(time.Duration, func()) {} // the timer never fires: the hub restarted d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box") // 2. a second alarm of the same kind inside the cooldown is suppressed — that row must not hide the failed one d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box") // 3. a mail that fails once and goes out on its first retry — NOT lost tries := 0 d.afterFn = func(_ time.Duration, f func()) { f() } d.sendEmailFn = func(to, subject, body string, _ map[string]string) error { tries++ if tries == 1 { return errors.New("timeout") } return nil } d.processOperator("c1", "tunnel_down", "error", "The tunnel of demo-hp is NOT running", "", "hub") if tries != 2 { t.Fatalf("setup: the retried mail made %d tries, want 2", tries) } // the hub starts again: a new dispatcher on the same database, the mail service works d2 := NewDispatcher(d.store, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0)) var got []sentMail d2.sendEmailFn = func(to, subject, body string, _ map[string]string) error { got = append(got, sentMail{to, subject, body}) return nil } now := time.Now().Add(2 * time.Second) time.Sleep(1100 * time.Millisecond) // the log's times have one-second grain: the "restart" is a second later if n := d2.ReportLostOperatorMails(now, now); n != 1 { t.Fatalf("lost mails told = %d, want 1 (the one whose retries the restart ate; not the one that went out on a retry)", n) } if len(got) != 1 || got[0].to != "op@felhom.eu" { t.Fatalf("mails sent at the start = %+v, want ONE to the operator", got) } for _, want := range []string{"Whole-guest backup FAILED on demo-hp", "whole_guest_backup_failed", "c1"} { if !strings.Contains(got[0].body, want) { t.Errorf("the summary mail lacks %q:\n%s", want, got[0].body) } } if strings.Contains(got[0].body, "The tunnel of demo-hp") { t.Errorf("a mail that WAS sent on a retry is listed as lost:\n%s", got[0].body) } evs, _ := d.store.GetEventsByType("", EventOperatorMailLost, time.Now().Add(-time.Hour)) if len(evs) != 1 || evs[0].Severity != "error" || !strings.Contains(evs[0].DetailsJSON, `"count":1`) { t.Fatalf("event rows = %+v, want one operator_mail_lost row", evs) } // a second restart: the same mail is not told again if n := d2.ReportLostOperatorMails(now.Add(time.Minute), now.Add(time.Minute)); n != 0 || len(got) != 1 { t.Fatalf("a second start told %d again (mails now %d)", n, len(got)) } _ = sent if !operatorOnlyEvents[EventOperatorMailLost] { t.Errorf("%s is not operator-only", EventOperatorMailLost) } } // Review of 2026-11-14: „at a start no timer is alive” is not true for a mail that failed in THIS process (the scan // runs some way into the start; a daily-check alarm or a box's event can already have failed and be waiting for its // retry). Such a mail must not be told as „never sent” — it may still arrive. func TestLostOperatorMail_AMailThatFailedInThisProcessIsNotLost(t *testing.T) { d, _, down := lostMailDispatcher(t) processStart := time.Now().Add(-2 * time.Second) // this process started two seconds ago *down = true d.afterFn = func(time.Duration, func()) {} // its retry timer is alive and has not fired yet d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box") *down = false if n := d.ReportLostOperatorMails(time.Now().Add(time.Second), processStart); n != 0 { t.Fatalf("a mail that failed AFTER this process started was told as lost (%d)", n) } // the next process (a later start) does tell it time.Sleep(1100 * time.Millisecond) later := time.Now().Add(time.Second) if n := d.ReportLostOperatorMails(later, later); n != 1 { t.Fatalf("the next start must tell it: %d", n) } } // The other half (clock list CH04-7): the hub keeps running and all three retries fail. A log line was the only // trace; now an event row says which mail was never sent. (No mail: the mail service is what failed.) func TestLostOperatorMail_GiveUpLeavesARow(t *testing.T) { d, sent, down := lostMailDispatcher(t) *down = true d.afterFn = func(_ time.Duration, f func()) { f() } d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box") evs, _ := d.store.GetEventsByType("", EventOperatorMailLost, time.Now().Add(-time.Hour)) if len(evs) != 1 || !strings.Contains(evs[0].Message, "Whole-guest backup FAILED on demo-hp") || !strings.Contains(evs[0].Message, "3 retries") { t.Fatalf("after the retries gave up: event rows = %+v, want one that carries the lost mail's text", evs) } if len(*sent) != 0 { t.Fatalf("a mail went out while the mail service was down: %+v", *sent) } } // R-1383: the other three events of the four cases are operator-only too. func TestR1383_SilentFailureEventsAreOperatorOnly(t *testing.T) { for _, e := range []string{"hub_daily_job_skipped", "signed_job_not_taken", "signed_job_refused", "operator_mail_lost", "guest_power_gave_up"} { if !operatorOnlyEvents[e] { t.Errorf("%s must be operator-only", e) } } }