gates / gates (push) Successful in 2m9s
A hub daily check skipped while the hub was down; an operator mail that was never sent; a signed job a box did not take; a signed job a box refused, and the guest-power watchdog's give-up (both from the agent). Operator-only. No new table. Not deployed. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
121 lines
5.2 KiB
Go
121 lines
5.2 KiB
Go
package notify
|
|
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type sentMail struct{ to, subject, body string }
|
|
|
|
func lostMailDispatcher(t *testing.T) (*Dispatcher, *[]sentMail, *bool) {
|
|
t.Helper()
|
|
st := newDispStore(t)
|
|
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
|
var mu sync.Mutex
|
|
sent, down := &[]sentMail{}, new(bool)
|
|
d.sendEmailFn = func(to, subject, body string, _ map[string]string) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if *down {
|
|
return errors.New("context deadline exceeded")
|
|
}
|
|
*sent = append(*sent, sentMail{to, subject, body})
|
|
return nil
|
|
}
|
|
return d, sent, down
|
|
}
|
|
|
|
// R-1383 (D78), the case the ruling names: an operator mail failed, and the hub RESTARTED before a retry worked — the
|
|
// retry timers are memory, so nothing sent it. The stored effect asserted: at the next start ONE operator mail goes
|
|
// out that carries the lost mail's own sentence, one event row is left, and a second start tells nothing again.
|
|
// A mail that failed and then went out on a retry is NOT lost. No real mail service: sendEmailFn is the repo's fake.
|
|
// COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-lost-mail.txt).
|
|
func TestLostOperatorMail_ToldAtTheNextStart(t *testing.T) {
|
|
d, sent, down := lostMailDispatcher(t)
|
|
// 1. a mail that fails, and the "restart" eats its retry timers
|
|
*down = true
|
|
d.afterFn = func(time.Duration, func()) {} // the timer never fires: the hub restarted
|
|
d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box")
|
|
// 2. a second alarm of the same kind inside the cooldown is suppressed — that row must not hide the failed one
|
|
d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box")
|
|
// 3. a mail that fails once and goes out on its first retry — NOT lost
|
|
tries := 0
|
|
d.afterFn = func(_ time.Duration, f func()) { f() }
|
|
d.sendEmailFn = func(to, subject, body string, _ map[string]string) error {
|
|
tries++
|
|
if tries == 1 {
|
|
return errors.New("timeout")
|
|
}
|
|
return nil
|
|
}
|
|
d.processOperator("c1", "tunnel_down", "error", "The tunnel of demo-hp is NOT running", "", "hub")
|
|
if tries != 2 {
|
|
t.Fatalf("setup: the retried mail made %d tries, want 2", tries)
|
|
}
|
|
|
|
// the hub starts again: a new dispatcher on the same database, the mail service works
|
|
d2 := NewDispatcher(d.store, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
|
var got []sentMail
|
|
d2.sendEmailFn = func(to, subject, body string, _ map[string]string) error {
|
|
got = append(got, sentMail{to, subject, body})
|
|
return nil
|
|
}
|
|
now := time.Now().Add(time.Minute)
|
|
if n := d2.ReportLostOperatorMails(now); n != 1 {
|
|
t.Fatalf("lost mails told = %d, want 1 (the one whose retries the restart ate; not the one that went out on a retry)", n)
|
|
}
|
|
if len(got) != 1 || got[0].to != "op@felhom.eu" {
|
|
t.Fatalf("mails sent at the start = %+v, want ONE to the operator", got)
|
|
}
|
|
for _, want := range []string{"Whole-guest backup FAILED on demo-hp", "whole_guest_backup_failed", "c1"} {
|
|
if !strings.Contains(got[0].body, want) {
|
|
t.Errorf("the summary mail lacks %q:\n%s", want, got[0].body)
|
|
}
|
|
}
|
|
if strings.Contains(got[0].body, "The tunnel of demo-hp") {
|
|
t.Errorf("a mail that WAS sent on a retry is listed as lost:\n%s", got[0].body)
|
|
}
|
|
evs, _ := d.store.GetEventsByType("", EventOperatorMailLost, time.Now().Add(-time.Hour))
|
|
if len(evs) != 1 || evs[0].Severity != "error" || !strings.Contains(evs[0].DetailsJSON, `"count":1`) {
|
|
t.Fatalf("event rows = %+v, want one operator_mail_lost row", evs)
|
|
}
|
|
// a second restart: the same mail is not told again
|
|
if n := d2.ReportLostOperatorMails(now.Add(time.Minute)); n != 0 || len(got) != 1 {
|
|
t.Fatalf("a second start told %d again (mails now %d)", n, len(got))
|
|
}
|
|
_ = sent
|
|
if !operatorOnlyEvents[EventOperatorMailLost] {
|
|
t.Errorf("%s is not operator-only", EventOperatorMailLost)
|
|
}
|
|
}
|
|
|
|
// The other half (clock list CH04-7): the hub keeps running and all three retries fail. A log line was the only
|
|
// trace; now an event row says which mail was never sent. (No mail: the mail service is what failed.)
|
|
func TestLostOperatorMail_GiveUpLeavesARow(t *testing.T) {
|
|
d, sent, down := lostMailDispatcher(t)
|
|
*down = true
|
|
d.afterFn = func(_ time.Duration, f func()) { f() }
|
|
d.processOperator("c1", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED on demo-hp", "", "box")
|
|
evs, _ := d.store.GetEventsByType("", EventOperatorMailLost, time.Now().Add(-time.Hour))
|
|
if len(evs) != 1 || !strings.Contains(evs[0].Message, "Whole-guest backup FAILED on demo-hp") || !strings.Contains(evs[0].Message, "3 retries") {
|
|
t.Fatalf("after the retries gave up: event rows = %+v, want one that carries the lost mail's text", evs)
|
|
}
|
|
if len(*sent) != 0 {
|
|
t.Fatalf("a mail went out while the mail service was down: %+v", *sent)
|
|
}
|
|
}
|
|
|
|
// R-1383: the other three events of the four cases are operator-only too.
|
|
func TestR1383_SilentFailureEventsAreOperatorOnly(t *testing.T) {
|
|
for _, e := range []string{"hub_daily_job_skipped", "signed_job_not_taken", "signed_job_refused", "operator_mail_lost", "guest_power_gave_up"} {
|
|
if !operatorOnlyEvents[e] {
|
|
t.Errorf("%s must be operator-only", e)
|
|
}
|
|
}
|
|
}
|