package notify import ( "encoding/json" "fmt" "strings" "time" ) // EventOperatorMailLost (R-1383, D78, `09` §3 decision 299) is operator-only. const EventOperatorMailLost = "operator_mail_lost" // lostMailLookBack bounds the start-up scan: an alarm older than two days is no longer news. const lostMailLookBack = 48 * time.Hour // ReportLostOperatorMails runs ONCE at a hub start (cmd/hub/main.go). The retries of a failed operator mail are // timers in memory (operatorRetryDelays, 21 minutes in all): a hub restart inside them lost the mail, and after the // last failed retry nothing sent it either (clock list CH04-4, CH04-7). At a start no timer is alive, so every // operator mail whose newest delivery record is `failed` was never sent. They are told in ONE operator mail that // carries each lost mail's own sentence, and in one event row. The scan's own time is kept (hub settings), so a second // restart does not tell the same mails again. // // The limit, said plainly: a mail that gave up while the hub KEEPS running is told only at the next hub start (it // leaves an event row at once — see retryOperatorEmail). And this summary is an operator mail like any other: when the // mail service is still down it is retried three times and can be lost too — the next start then lists it. // Returns the number of lost mails told. Pinned by TestLostOperatorMail_*. // // processStart is when THIS hub process began (cmd/hub: a package-level time). Only mails that failed before it are // lost: one that failed since may still be retried by this process's own timers. func (d *Dispatcher) ReportLostOperatorMails(now, processStart time.Time) int { since := now.Add(-lostMailLookBack) if mark := d.store.OSAlarmRaised("operator_mail_lost_scan"); mark.After(since) { since = mark } lost, err := d.store.LostOperatorMails(since, processStart) if err != nil { d.logger.Printf("[WARN] lost operator mail scan failed: %v", err) return 0 } if err := d.store.SetOSAlarmRaised("operator_mail_lost_scan", processStart); err != nil { d.logger.Printf("[WARN] lost operator mail scan: could not record the scan time: %v", err) } if len(lost) == 0 { d.logger.Printf("[INFO] lost operator mail scan: none since %s", since.UTC().Format(time.RFC3339)) return 0 } var b strings.Builder fmt.Fprintf(&b, "%d operator mail(s) were NEVER sent: the send failed and the hub restarted or gave up before a retry worked. Each one, with its own text:", len(lost)) type item struct { At, Customer, Event, Severity string } var items []item for i, m := range lost { if i == 20 { fmt.Fprintf(&b, "\n… and %d more (the hub's notification log holds them, status „failed”).", len(lost)-20) break } msg := m.Message if r := []rune(msg); len(r) > 400 { // by rune: the texts hold „ ” — and a byte cut would split one msg = string(r[:400]) + "…" } cust := m.CustomerID if cust == "" { cust = "(hub)" } fmt.Fprintf(&b, "\n- %s · %s · %s (%s): %s", m.CreatedAt.UTC().Format("2006-01-02 15:04Z"), cust, m.EventType, m.Severity, msg) items = append(items, item{m.CreatedAt.UTC().Format(time.RFC3339), m.CustomerID, m.EventType, m.Severity}) } d.logger.Printf("[ERROR] lost operator mail scan: %d operator mail(s) were never sent — telling the operator in one mail", len(lost)) details, _ := json.Marshal(map[string]any{"count": len(lost), "since": since.UTC().Format(time.RFC3339), "lost": items}) if _, err := d.store.SaveEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub"); err != nil { d.logger.Printf("[WARN] %s event save FAILED: %v", EventOperatorMailLost, err) } d.ProcessEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub") return len(lost) }