Files
felhom.eu/hub/internal/notify/lost_operator_mail.go
T
admin 1886030782
gates / gates (push) Successful in 2m9s
hub (unreleased): four silent failures tell the operator (R-1383, D78)
A hub daily check skipped while the hub was down; an operator mail that was never sent; a signed job a box did not
take; a signed job a box refused, and the guest-power watchdog's give-up (both from the agent). Operator-only.
No new table. Not deployed.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-11-14 17:29:53 +01:00

74 lines
3.4 KiB
Go

package notify
import (
"encoding/json"
"fmt"
"strings"
"time"
)
// EventOperatorMailLost (R-1383, D78, `09` §3 decision 299) is operator-only.
const EventOperatorMailLost = "operator_mail_lost"
// lostMailLookBack bounds the start-up scan: an alarm older than two days is no longer news.
const lostMailLookBack = 48 * time.Hour
// ReportLostOperatorMails runs ONCE at a hub start (cmd/hub/main.go). The retries of a failed operator mail are
// timers in memory (operatorRetryDelays, 21 minutes in all): a hub restart inside them lost the mail, and after the
// last failed retry nothing sent it either (clock list CH04-4, CH04-7). At a start no timer is alive, so every
// operator mail whose newest delivery record is `failed` was never sent. They are told in ONE operator mail that
// carries each lost mail's own sentence, and in one event row. The scan's own time is kept (hub settings), so a second
// restart does not tell the same mails again.
//
// The limit, said plainly: a mail that gave up while the hub KEEPS running is told only at the next hub start (it
// leaves an event row at once — see retryOperatorEmail). And this summary is an operator mail like any other: when the
// mail service is still down it is retried three times and can be lost too — the next start then lists it.
// Returns the number of lost mails told. Pinned by TestLostOperatorMail_*.
func (d *Dispatcher) ReportLostOperatorMails(now time.Time) int {
since := now.Add(-lostMailLookBack)
if mark := d.store.OSAlarmRaised("operator_mail_lost_scan"); mark.After(since) {
since = mark
}
lost, err := d.store.LostOperatorMails(since)
if err != nil {
d.logger.Printf("[WARN] lost operator mail scan failed: %v", err)
return 0
}
if err := d.store.SetOSAlarmRaised("operator_mail_lost_scan", now); err != nil {
d.logger.Printf("[WARN] lost operator mail scan: could not record the scan time: %v", err)
}
if len(lost) == 0 {
d.logger.Printf("[INFO] lost operator mail scan: none since %s", since.UTC().Format(time.RFC3339))
return 0
}
var b strings.Builder
fmt.Fprintf(&b, "%d operator mail(s) were NEVER sent: the send failed and the hub restarted or gave up before a retry worked. Each one, with its own text:", len(lost))
type item struct {
At, Customer, Event, Severity string
}
var items []item
for i, m := range lost {
if i == 20 {
fmt.Fprintf(&b, "\n… and %d more (the hub's notification log holds them, status „failed”).", len(lost)-20)
break
}
msg := m.Message
if len(msg) > 400 {
msg = msg[:400] + "…"
}
cust := m.CustomerID
if cust == "" {
cust = "(hub)"
}
fmt.Fprintf(&b, "\n- %s · %s · %s (%s): %s", m.CreatedAt.UTC().Format("2006-01-02 15:04Z"), cust, m.EventType, m.Severity, msg)
items = append(items, item{m.CreatedAt.UTC().Format(time.RFC3339), m.CustomerID, m.EventType, m.Severity})
}
d.logger.Printf("[ERROR] lost operator mail scan: %d operator mail(s) were never sent — telling the operator in one mail", len(lost))
details, _ := json.Marshal(map[string]any{"count": len(lost), "since": since.UTC().Format(time.RFC3339), "lost": items})
if _, err := d.store.SaveEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub"); err != nil {
d.logger.Printf("[WARN] %s event save FAILED: %v", EventOperatorMailLost, err)
}
d.ProcessEvent("", EventOperatorMailLost, "error", b.String(), string(details), "hub")
return len(lost)
}