77 lines
3.7 KiB
Go
77 lines
3.7 KiB
Go
package notify
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// EventOperatorMailLost (R-1383, D78, `09` §3 decision 299) is operator-only.
|
|
const EventOperatorMailLost = "operator_mail_lost"
|
|
|
|
// lostMailLookBack bounds the start-up scan: an alarm older than two days is no longer news.
|
|
const lostMailLookBack = 48 * time.Hour
|
|
|
|
// ReportLostOperatorMails runs ONCE at a hub start (cmd/hub/main.go). The retries of a failed operator mail are
|
|
// timers in memory (operatorRetryDelays, 21 minutes in all): a hub restart inside them lost the mail, and after the
|
|
// last failed retry nothing sent it either (clock list CH04-4, CH04-7). At a start no timer is alive, so every
|
|
// operator mail whose newest delivery record is `failed` was never sent. They are told in ONE operator mail that
|
|
// carries each lost mail's own sentence, and in one event row. The scan's own time is kept (hub settings), so a second
|
|
// restart does not tell the same mails again.
|
|
//
|
|
// The limit, said plainly: a mail that gave up while the hub KEEPS running is told only at the next hub start (it
|
|
// leaves an event row at once — see retryOperatorEmail). And this summary is an operator mail like any other: when the
|
|
// mail service is still down it is retried three times and can be lost too — the next start then lists it.
|
|
// Returns the number of lost mails told. Pinned by TestLostOperatorMail_*.
|
|
//
|
|
// processStart is when THIS hub process began (cmd/hub: a package-level time). Only mails that failed before it are
|
|
// lost: one that failed since may still be retried by this process's own timers.
|
|
func (d *Dispatcher) ReportLostOperatorMails(now, processStart time.Time) int {
|
|
since := now.Add(-lostMailLookBack)
|
|
if mark := d.store.OSAlarmRaised("operator_mail_lost_scan"); mark.After(since) {
|
|
since = mark
|
|
}
|
|
lost, err := d.store.LostOperatorMails(since, processStart)
|
|
if err != nil {
|
|
d.logger.Printf("[WARN] lost operator mail scan failed: %v", err)
|
|
return 0
|
|
}
|
|
if err := d.store.SetOSAlarmRaised("operator_mail_lost_scan", processStart); err != nil {
|
|
d.logger.Printf("[WARN] lost operator mail scan: could not record the scan time: %v", err)
|
|
}
|
|
if len(lost) == 0 {
|
|
d.logger.Printf("[INFO] lost operator mail scan: none since %s", since.UTC().Format(time.RFC3339))
|
|
return 0
|
|
}
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "%d operator mail(s) were NEVER sent: the send failed and the hub restarted or gave up before a retry worked. Each one, with its own text:", len(lost))
|
|
type item struct {
|
|
At, Customer, Event, Severity string
|
|
}
|
|
var items []item
|
|
for i, m := range lost {
|
|
if i == 20 {
|
|
fmt.Fprintf(&b, "\n… and %d more (the hub's notification log holds them, status „failed”).", len(lost)-20)
|
|
break
|
|
}
|
|
msg := m.Message
|
|
if r := []rune(msg); len(r) > 400 { // by rune: the texts hold „ ” — and a byte cut would split one
|
|
msg = string(r[:400]) + "…"
|
|
}
|
|
cust := m.CustomerID
|
|
if cust == "" {
|
|
cust = "(hub)"
|
|
}
|
|
fmt.Fprintf(&b, "\n- %s · %s · %s (%s): %s", m.CreatedAt.UTC().Format("2006-01-02 15:04Z"), cust, m.EventType, m.Severity, msg)
|
|
items = append(items, item{m.CreatedAt.UTC().Format(time.RFC3339), m.CustomerID, m.EventType, m.Severity})
|
|
}
|
|
d.logger.Printf("[ERROR] lost operator mail scan: %d operator mail(s) were never sent — telling the operator in one mail", len(lost))
|
|
details, _ := json.Marshal(map[string]any{"count": len(lost), "since": since.UTC().Format(time.RFC3339), "lost": items})
|
|
if _, err := d.store.SaveEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub"); err != nil {
|
|
d.logger.Printf("[WARN] %s event save FAILED: %v", EventOperatorMailLost, err)
|
|
}
|
|
d.ProcessEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub")
|
|
return len(lost)
|
|
}
|