felhom.eu/hub/internal/notify/operator_mail_retry_test.go

package notify

import (
	"errors"
	"io"
	"log"
	"sync"
	"testing"
	"time"
)

// The lost alarm of 2026-10-05 (fixed without a row, `audits/readback-2026-10-07/C/`): a failed operator mail is tried again (operatorRetryDelays) instead of being dropped. The consequence
// asserted is that the mail is SENT and recorded `sent`.
//
// COMPANION RED-PROOF (REPORT): drop the retryOperatorEmail call from processOperator → "the failed mail was never
// sent again".
func TestOperatorMailRetry_FailedOperatorMailIsRetried(t *testing.T) {
	st := newDispStore(t)
	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
	var mu sync.Mutex
	tries, waits := 0, []time.Duration{}
	d.afterFn = func(w time.Duration, f func()) { mu.Lock(); waits = append(waits, w); mu.Unlock(); f() }
	d.sendEmailFn = func(string, string, string, map[string]string) error {
		mu.Lock()
		defer mu.Unlock()
		tries++
		if tries <= 2 { // the first send and the first retry time out
			return errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")
		}
		return nil
	}
	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "Whole-guest backup FAILED", "", "box")
	if tries != 3 {
		t.Fatalf("the failed mail was never sent again (tries=%d)", tries)
	}
	if len(waits) != 2 || waits[0] != operatorRetryDelays[0] || waits[1] != operatorRetryDelays[1] {
		t.Fatalf("retries not on the schedule: %v", waits)
	}
	rows, err := st.GetRecentNotifications("demo-hp", 50)
	if err != nil {
		t.Fatal(err)
	}
	var sent, failed int
	for _, r := range rows {
		if r.Channel != "operator" {
			continue
		}
		switch r.Status {
		case "sent":
			sent++
		case "failed":
			failed++
		}
	}
	if sent != 1 || failed != 2 {
		t.Fatalf("notification log: sent=%d failed=%d, want 1 and 2", sent, failed)
	}
}

// The retries stop: a send that never succeeds is tried len(operatorRetryDelays) more times, then given up.
func TestOperatorMailRetry_RetriesAreBounded(t *testing.T) {
	st := newDispStore(t)
	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
	tries := 0
	d.afterFn = func(_ time.Duration, f func()) { f() }
	d.sendEmailFn = func(string, string, string, map[string]string) error { tries++; return errors.New("down") }
	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "x", "", "box")
	if tries != 1+len(operatorRetryDelays) {
		t.Fatalf("tries=%d, want %d", tries, 1+len(operatorRetryDelays))
	}
}

// A mail that is sent the first time is not retried.
func TestOperatorMailRetry_SuccessIsNotRetried(t *testing.T) {
	st := newDispStore(t)
	d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
	scheduled := 0
	d.afterFn = func(_ time.Duration, f func()) { scheduled++; f() }
	d.sendEmailFn = func(string, string, string, map[string]string) error { return nil }
	d.processOperator("demo-hp", "whole_guest_backup_failed", "error", "x", "", "box")
	if scheduled != 0 {
		t.Fatalf("a sent mail scheduled %d retries", scheduled)
	}
}