hub v0.71.0: paired recovery mails (F11), prefs seeding at claim + empty-email no-clobber (F12), priority headers + operator test leg (F14-light)
This commit is contained in:
@@ -29,8 +29,10 @@ type Dispatcher struct {
|
||||
custCooldowns map[string]time.Time // "customerID:eventType" → last customer notify
|
||||
|
||||
// sendEmailFn is the email sender, seam-injected so tests exercise routing without real HTTP.
|
||||
// Defaults to (*Dispatcher).sendEmail (Resend) in NewDispatcher.
|
||||
sendEmailFn func(to, subject, textBody string) error
|
||||
// Defaults to (*Dispatcher).sendEmail (Resend) in NewDispatcher. headers (nil = none) become
|
||||
// Resend custom headers — used for the high-priority nudge on error/critical mails (v0.71.0,
|
||||
// audit F14-light).
|
||||
sendEmailFn func(to, subject, textBody string, headers map[string]string) error
|
||||
}
|
||||
|
||||
// NewDispatcher creates a new notification dispatcher.
|
||||
@@ -50,6 +52,26 @@ func NewDispatcher(s *store.Store, resendAPIKey, fromEmail, operatorEmail string
|
||||
return d
|
||||
}
|
||||
|
||||
// priorityHeaders returns the Resend custom headers that nudge mail clients toward attention for
|
||||
// error/critical mails (X-Priority + Importance; v0.71.0, audit F14-light: delivered ≠ noticed).
|
||||
// Everything else gets nil — a warning or info mail must NOT masquerade as urgent. Pure → tested.
|
||||
func priorityHeaders(severity string) map[string]string {
|
||||
switch severity {
|
||||
case "error", "critical":
|
||||
return map[string]string{"X-Priority": "1", "Importance": "high"}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// recoveredPairedDownTypes maps a *_recovered eventType to the stale/down set whose customer-channel
|
||||
// "sent" evidence licenses the customer recovery mail (v0.71.0, audit F11): recovery notifies
|
||||
// exactly whoever the down notified.
|
||||
var recoveredPairedDownTypes = map[string][]string{
|
||||
"node_recovered": {"node_stale", "node_down"},
|
||||
"host_recovered": {"host_stale", "host_down"},
|
||||
}
|
||||
|
||||
// severityNotifies reports whether a severity triggers email notifications. warning / error / critical
|
||||
// notify; everything else (info, recovery/status, or an unrecognized value) does not. Pure → unit-tested.
|
||||
// (Before v0.24.0 a "critical" severity was silently dropped here — the host_disk-class bug.)
|
||||
@@ -75,6 +97,14 @@ func (d *Dispatcher) ProcessEvent(customerID, eventType, severity, message, deta
|
||||
return
|
||||
}
|
||||
|
||||
// Recovery branch (v0.71.0, audit F11) — BEFORE the severity gate, as an explicit eventType
|
||||
// branch: *_recovered stays severity "info" (semantics frozen), but is no longer silent.
|
||||
// Operator always hears both edges; the customer hears recovery iff they heard the down.
|
||||
if _, isRecovery := recoveredPairedDownTypes[eventType]; isRecovery {
|
||||
d.processRecovery(customerID, eventType, severity, message, detailsJSON, source)
|
||||
return
|
||||
}
|
||||
|
||||
// warning / error / critical trigger notifications. "info" is an intentional non-notify (status/
|
||||
// recovery events). Anything else is UNRECOGNIZED — log it (don't silently drop), so a bad severity
|
||||
// surfaces instead of vanishing (the felhom-pve-class lesson: a critical event must never be lost).
|
||||
@@ -93,22 +123,106 @@ func (d *Dispatcher) ProcessEvent(customerID, eventType, severity, message, deta
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendTestEmail(customerID string) {
|
||||
// nil-prefs guard (v0.71.0): GetNotificationPrefs returns (nil, nil) for a customer with no
|
||||
// notification row — dereferencing prefs.Email here panicked the dispatcher goroutine for such
|
||||
// a customer (latent since the test leg shipped; found while adding the operator copy).
|
||||
prefs, err := d.store.GetNotificationPrefs(customerID)
|
||||
if err != nil || prefs.Email == "" {
|
||||
if err != nil || prefs == nil || prefs.Email == "" {
|
||||
d.logger.Printf("[WARN] Test email: no email configured for %s", customerID)
|
||||
} else {
|
||||
subject := "[Felhom] Teszt értesítés"
|
||||
body := "Kedves Ügyfél!\n\nEz egy teszt értesítés a Felhom monitoring rendszerből.\nAz értesítések megfelelően működnek.\n\nÜdvözlettel,\nFelhom.eu monitoring"
|
||||
|
||||
if err := d.sendEmailFn(prefs.Email, subject, body, nil); err != nil {
|
||||
d.logger.Printf("[ERROR] Test email to %s failed: %v", prefs.Email, err)
|
||||
d.store.LogNotification(customerID, "test", "info", "Teszt értesítés", "failed", err.Error(), "customer")
|
||||
} else {
|
||||
d.logger.Printf("[INFO] Test email sent to %s for %s", prefs.Email, customerID)
|
||||
d.store.LogNotification(customerID, "test", "info", "Teszt értesítés", "sent", "", "customer")
|
||||
}
|
||||
}
|
||||
|
||||
// Operator copy (v0.71.0, audit F14-light): one test click proves the customer channel, the
|
||||
// operator channel AND the high-priority header rendering in a single shot.
|
||||
if !d.operatorOn || d.operatorEmail == "" {
|
||||
return
|
||||
}
|
||||
opSubject := fmt.Sprintf("[Felhom] ✅ %s: teszt / operator channel OK", customerID)
|
||||
opBody := fmt.Sprintf(`Operator copy of the customer notification test for %s.
|
||||
|
||||
If this mail shows as high priority in your client, the X-Priority/Importance
|
||||
headers render correctly. The customer test mail result is recorded in the
|
||||
notification log.
|
||||
|
||||
Dashboard: https://hub.felhom.eu/customers/%s`, customerID, customerID)
|
||||
if err := d.sendEmailFn(d.operatorEmail, opSubject, opBody, priorityHeaders("critical")); err != nil {
|
||||
d.logger.Printf("[ERROR] Operator test email failed for %s: %v", customerID, err)
|
||||
d.store.LogNotification(customerID, "test", "info", "operator test copy", "failed", err.Error(), "operator")
|
||||
return
|
||||
}
|
||||
d.logger.Printf("[INFO] Operator test email sent for %s", customerID)
|
||||
d.store.LogNotification(customerID, "test", "info", "operator test copy", "sent", "", "operator")
|
||||
}
|
||||
|
||||
// processRecovery routes a *_recovered event (v0.71.0, audit F11). Severity semantics stay frozen
|
||||
// ("info" everywhere else remains non-notify) — this is an explicit eventType branch.
|
||||
// - Operator leg: always wanted (both edges), gated only by operatorOn + the 1h per-type
|
||||
// cooldown — exactly processOperator.
|
||||
// - Customer leg: gated by the PAIRING rule, not enabled_events — "recovery notifies exactly
|
||||
// whoever the down notified." Evidence = a customer-channel status=sent row for the paired
|
||||
// stale/down set newer than the last customer-channel sent recovery of this type.
|
||||
func (d *Dispatcher) processRecovery(customerID, eventType, severity, message, detailsJSON, source string) {
|
||||
d.processOperator(customerID, eventType, severity, message, detailsJSON, source)
|
||||
|
||||
if d.store.IsCustomerBlocked(customerID) {
|
||||
return
|
||||
}
|
||||
prefs, err := d.store.GetNotificationPrefs(customerID)
|
||||
if err != nil || prefs == nil || prefs.Email == "" {
|
||||
return
|
||||
}
|
||||
|
||||
subject := "[Felhom] Teszt értesítés"
|
||||
body := "Kedves Ügyfél!\n\nEz egy teszt értesítés a Felhom monitoring rendszerből.\nAz értesítések megfelelően működnek.\n\nÜdvözlettel,\nFelhom.eu monitoring"
|
||||
|
||||
if err := d.sendEmailFn(prefs.Email, subject, body); err != nil {
|
||||
d.logger.Printf("[ERROR] Test email to %s failed: %v", prefs.Email, err)
|
||||
d.store.LogNotification(customerID, "test", "info", "Teszt értesítés", "failed", err.Error(), "customer")
|
||||
// Pairing check — the customer gate. enabled_events is deliberately ignored here: a customer
|
||||
// who was told "down" must be told "recovered", and one who wasn't must not be.
|
||||
lastDown, downOk, err := d.store.LastCustomerSentAt(customerID, recoveredPairedDownTypes[eventType])
|
||||
if err != nil {
|
||||
d.logger.Printf("[ERROR] Recovery pairing query failed for %s/%s: %v", customerID, eventType, err)
|
||||
return
|
||||
}
|
||||
d.logger.Printf("[INFO] Test email sent to %s for %s", prefs.Email, customerID)
|
||||
d.store.LogNotification(customerID, "test", "info", "Teszt értesítés", "sent", "", "customer")
|
||||
lastRecovered, recOk, err := d.store.LastCustomerSentAt(customerID, []string{eventType})
|
||||
if err != nil {
|
||||
d.logger.Printf("[ERROR] Recovery pairing query failed for %s/%s: %v", customerID, eventType, err)
|
||||
return
|
||||
}
|
||||
// Second-granularity ties resolve to NOT-after → no mail (flap-safe direction).
|
||||
if !downOk || (recOk && !lastDown.After(lastRecovered)) {
|
||||
d.logger.Printf("[INFO] Recovery %s for %s: customer mail skipped — no unanswered customer down mail (pairing miss)", eventType, customerID)
|
||||
return
|
||||
}
|
||||
|
||||
// Prefs cooldown keyed on the recovered eventType (belt over the pairing braces).
|
||||
cooldownHours := prefs.CooldownHours
|
||||
if cooldownHours <= 0 {
|
||||
cooldownHours = 6
|
||||
}
|
||||
cooldownKey := customerID + ":" + eventType
|
||||
d.mu.Lock()
|
||||
if last, ok := d.custCooldowns[cooldownKey]; ok && time.Since(last) < time.Duration(cooldownHours)*time.Hour {
|
||||
d.mu.Unlock()
|
||||
d.logger.Printf("[INFO] Recovery %s for %s: customer mail skipped — cooldown", eventType, customerID)
|
||||
return
|
||||
}
|
||||
d.custCooldowns[cooldownKey] = time.Now()
|
||||
d.mu.Unlock()
|
||||
|
||||
subject, body := FormatCustomerEmail(customerID, eventType, severity, message, detailsJSON)
|
||||
if err := d.sendEmailFn(prefs.Email, subject, body, priorityHeaders(severity)); err != nil {
|
||||
d.logger.Printf("[ERROR] Customer recovery email failed for %s/%s: %v", customerID, eventType, err)
|
||||
d.store.LogNotification(customerID, eventType, severity, message, "failed", err.Error(), "customer")
|
||||
return
|
||||
}
|
||||
d.logger.Printf("[INFO] Customer recovery email sent to %s for %s/%s", prefs.Email, customerID, eventType)
|
||||
d.store.LogNotification(customerID, eventType, severity, message, "sent", "", "customer")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) processOperator(customerID, eventType, severity, message, detailsJSON, source string) {
|
||||
@@ -127,7 +241,7 @@ func (d *Dispatcher) processOperator(customerID, eventType, severity, message, d
|
||||
|
||||
subject, body := FormatOperatorEmail(customerID, eventType, severity, message, detailsJSON)
|
||||
|
||||
if err := d.sendEmailFn(d.operatorEmail, subject, body); err != nil {
|
||||
if err := d.sendEmailFn(d.operatorEmail, subject, body, priorityHeaders(severity)); err != nil {
|
||||
d.logger.Printf("[ERROR] Operator email failed for %s/%s: %v", customerID, eventType, err)
|
||||
d.store.LogNotification(customerID, eventType, severity, message, "failed", err.Error(), "operator")
|
||||
return
|
||||
@@ -173,7 +287,7 @@ func (d *Dispatcher) processCustomer(customerID, eventType, severity, message, d
|
||||
|
||||
subject, body := FormatCustomerEmail(customerID, eventType, severity, message, detailsJSON)
|
||||
|
||||
if err := d.sendEmailFn(prefs.Email, subject, body); err != nil {
|
||||
if err := d.sendEmailFn(prefs.Email, subject, body, priorityHeaders(severity)); err != nil {
|
||||
d.logger.Printf("[ERROR] Customer email failed for %s/%s: %v", customerID, eventType, err)
|
||||
d.store.LogNotification(customerID, eventType, severity, message, "failed", err.Error(), "customer")
|
||||
return
|
||||
@@ -182,13 +296,16 @@ func (d *Dispatcher) processCustomer(customerID, eventType, severity, message, d
|
||||
d.store.LogNotification(customerID, eventType, severity, message, "sent", "", "customer")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmail(to, subject, textBody string) error {
|
||||
func (d *Dispatcher) sendEmail(to, subject, textBody string, headers map[string]string) error {
|
||||
payload := map[string]interface{}{
|
||||
"from": d.fromEmail,
|
||||
"to": []string{to},
|
||||
"subject": subject,
|
||||
"text": textBody,
|
||||
}
|
||||
if len(headers) > 0 {
|
||||
payload["headers"] = headers
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
@@ -236,7 +353,7 @@ func (d *Dispatcher) SendClaimEmail(kind, customerID, email, domain, code string
|
||||
}
|
||||
subject, body := FormatClaimEmail(kind, customerID, domain, code)
|
||||
eventType := "claim_" + kind
|
||||
if err := d.sendEmailFn(email, subject, body); err != nil {
|
||||
if err := d.sendEmailFn(email, subject, body, nil); err != nil {
|
||||
d.logger.Printf("[ERROR] claim %s email to customer %s failed: %v", kind, customerID, err)
|
||||
d.store.LogNotification(customerID, eventType, "info", subject, "failed", err.Error(), "customer")
|
||||
return err
|
||||
@@ -257,7 +374,7 @@ func (d *Dispatcher) SendSelfBindEmail(customerID, email, link string) error {
|
||||
return fmt.Errorf("notify: no resend api key")
|
||||
}
|
||||
subject, body := FormatSelfBindEmail(customerID, link)
|
||||
if err := d.sendEmailFn(email, subject, body); err != nil {
|
||||
if err := d.sendEmailFn(email, subject, body, nil); err != nil {
|
||||
d.logger.Printf("[ERROR] self-bind link email to customer %s failed: %v", customerID, err)
|
||||
d.store.LogNotification(customerID, "selfbind_link", "info", subject, "failed", err.Error(), "customer")
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// capturedMail is one seam-captured send: recipient, subject and headers.
|
||||
type capturedMail struct {
|
||||
to string
|
||||
subject string
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
// captureSeam installs a capturing sendEmailFn and returns the capture slice pointer.
|
||||
func captureSeam(d *Dispatcher) *[]capturedMail {
|
||||
var mu sync.Mutex
|
||||
sent := &[]capturedMail{}
|
||||
d.sendEmailFn = func(to, subject, _ string, headers map[string]string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
*sent = append(*sent, capturedMail{to: to, subject: subject, headers: headers})
|
||||
return nil
|
||||
}
|
||||
return sent
|
||||
}
|
||||
|
||||
func mailsFor(sent []capturedMail, to string) []capturedMail {
|
||||
var out []capturedMail
|
||||
for _, m := range sent {
|
||||
if m.to == to {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestRecovery_PairedCustomerMail (Scenario A — the doodoo21 case, fixed): a customer who was
|
||||
// mailed node_down gets the node_recovered mail; the operator gets the ✅ mail; both are logged.
|
||||
func TestRecovery_PairedCustomerMail(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"node_down"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
// The down edge: customer (enabled) + operator both mailed.
|
||||
d.ProcessEvent("c1", "node_down", "error", "No report received for 1h", "{}", "hub")
|
||||
if len(*sent) != 2 {
|
||||
t.Fatalf("down edge must mail operator + customer, got %d sends", len(*sent))
|
||||
}
|
||||
|
||||
// The recovery edge.
|
||||
d.ProcessEvent("c1", "node_recovered", "info", "Reports resumed (was down for 3h48m)", "{}", "hub")
|
||||
|
||||
cust := mailsFor(*sent, "cust@example.com")
|
||||
if len(cust) != 2 {
|
||||
t.Fatalf("customer must get down + recovery, got %d customer mails", len(cust))
|
||||
}
|
||||
recMail := cust[1]
|
||||
if !strings.Contains(recMail.subject, "Információ: A szerver újra elérhető.") {
|
||||
t.Fatalf("customer recovery subject wrong: %q", recMail.subject)
|
||||
}
|
||||
op := mailsFor(*sent, "op@felhom.eu")
|
||||
if len(op) != 2 {
|
||||
t.Fatalf("operator must get down + recovery, got %d", len(op))
|
||||
}
|
||||
if !strings.Contains(op[1].subject, "✅") || !strings.Contains(op[1].subject, "node_recovered") {
|
||||
t.Fatalf("operator recovery subject must carry ✅ + node_recovered: %q", op[1].subject)
|
||||
}
|
||||
|
||||
// notification_log holds sent rows on both channels for node_recovered.
|
||||
entries, err := st.GetRecentNotifications("c1", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var gotCust, gotOp bool
|
||||
for _, e := range entries {
|
||||
if e.EventType == "node_recovered" && e.Status == "sent" {
|
||||
switch e.Channel {
|
||||
case "customer":
|
||||
gotCust = true
|
||||
case "operator":
|
||||
gotOp = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !gotCust || !gotOp {
|
||||
t.Fatalf("node_recovered sent rows missing (customer=%v operator=%v)", gotCust, gotOp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecovery_UnpairedStaysCustomerSilent (Scenario B): no customer-channel down evidence → the
|
||||
// operator is mailed, the customer is NOT — even with node_recovered in enabled_events (the
|
||||
// pairing rule is THE customer gate; enabled_events is deliberately ignored for recovery).
|
||||
// Companion red-proof: removing the pairing check in processRecovery makes this fail.
|
||||
func TestRecovery_UnpairedStaysCustomerSilent(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"node_down", "node_recovered"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
d.ProcessEvent("c1", "node_recovered", "info", "Reports resumed", "{}", "hub")
|
||||
|
||||
if n := len(mailsFor(*sent, "cust@example.com")); n != 0 {
|
||||
t.Fatalf("unpaired recovery must NOT mail the customer, got %d", n)
|
||||
}
|
||||
if n := len(mailsFor(*sent, "op@felhom.eu")); n != 1 {
|
||||
t.Fatalf("operator must still get the recovery mail, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecovery_FlapDamping (Scenario C): down mailed → recovery mailed → second down SUPPRESSED by
|
||||
// the customer cooldown → second recovery must NOT mail the customer (the suppressed down left no
|
||||
// fresh pairing evidence), and the operator recovery mail obeys the 1h per-type cooldown.
|
||||
func TestRecovery_FlapDamping(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"node_down"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
d.ProcessEvent("c1", "node_down", "error", "down", "{}", "hub") // down #1: cust+op mailed
|
||||
d.ProcessEvent("c1", "node_recovered", "info", "up", "{}", "hub") // recovery #1: cust+op mailed
|
||||
d.ProcessEvent("c1", "node_down", "error", "down", "{}", "hub") // down #2 10min later: customer
|
||||
// cooldown suppresses (op too, 1h)
|
||||
// Clear the recovery CUSTOMER cooldown so the pairing gate — not the cooldown — decides #2:
|
||||
d.mu.Lock()
|
||||
delete(d.custCooldowns, "c1:node_recovered")
|
||||
d.mu.Unlock()
|
||||
d.ProcessEvent("c1", "node_recovered", "info", "up", "{}", "hub") // recovery #2
|
||||
|
||||
cust := mailsFor(*sent, "cust@example.com")
|
||||
if len(cust) != 2 { // down #1 + recovery #1 only
|
||||
t.Fatalf("second recovery must not mail the customer (no fresh pairing), customer mails=%d: %+v", len(cust), cust)
|
||||
}
|
||||
// Operator: down #1 + recovery #1; down #2 and recovery #2 suppressed by the 1h op cooldown.
|
||||
if op := mailsFor(*sent, "op@felhom.eu"); len(op) != 2 {
|
||||
t.Fatalf("operator recovery must obey the 1h cooldown, op mails=%d", len(op))
|
||||
}
|
||||
}
|
||||
|
||||
// TestTestEvent_BothChannels (Scenario F): a "test" event mails the customer (existing copy) AND
|
||||
// the operator, the operator mail carrying both priority headers; both logged.
|
||||
func TestTestEvent_BothChannels(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", nil, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
d.ProcessEvent("c1", "test", "info", "", "{}", "hub")
|
||||
|
||||
cust := mailsFor(*sent, "cust@example.com")
|
||||
if len(cust) != 1 || !strings.Contains(cust[0].subject, "Teszt értesítés") {
|
||||
t.Fatalf("customer test mail wrong: %+v", cust)
|
||||
}
|
||||
op := mailsFor(*sent, "op@felhom.eu")
|
||||
if len(op) != 1 || !strings.Contains(op[0].subject, "✅") || !strings.Contains(op[0].subject, "operator channel OK") {
|
||||
t.Fatalf("operator test mail wrong: %+v", op)
|
||||
}
|
||||
if op[0].headers["X-Priority"] != "1" || op[0].headers["Importance"] != "high" {
|
||||
t.Fatalf("operator test mail must carry priority headers, got %v", op[0].headers)
|
||||
}
|
||||
entries, _ := st.GetRecentNotifications("c1", 10)
|
||||
var chans []string
|
||||
for _, e := range entries {
|
||||
if e.EventType == "test" && e.Status == "sent" {
|
||||
chans = append(chans, e.Channel)
|
||||
}
|
||||
}
|
||||
if len(chans) != 2 {
|
||||
t.Fatalf("both test sends must be logged, got channels %v", chans)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTestEvent_NoPrefsRow_NoPanic: a test event for a customer WITHOUT a notification row must
|
||||
// not panic (the nil-prefs deref found in v0.71.0 — GetNotificationPrefs returns (nil, nil)) and
|
||||
// the operator copy still proves the operator channel.
|
||||
func TestTestEvent_NoPrefsRow_NoPanic(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
d.ProcessEvent("c1", "test", "info", "", "{}", "hub") // must not panic
|
||||
|
||||
if n := len(mailsFor(*sent, "op@felhom.eu")); n != 1 {
|
||||
t.Fatalf("operator test copy must send even without customer prefs, got %d", n)
|
||||
}
|
||||
if n := len(*sent); n != 1 {
|
||||
t.Fatalf("no customer mail possible without prefs, total sends=%d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPriorityHeaders (Scenario G): error/critical carry the two headers; warning/info/unknown nil.
|
||||
// The negative half is the red-proof target: adding headers unconditionally fails the nil cases.
|
||||
func TestPriorityHeaders(t *testing.T) {
|
||||
for _, sev := range []string{"error", "critical"} {
|
||||
h := priorityHeaders(sev)
|
||||
if h["X-Priority"] != "1" || h["Importance"] != "high" || len(h) != 2 {
|
||||
t.Errorf("%s: headers = %v", sev, h)
|
||||
}
|
||||
}
|
||||
for _, sev := range []string{"warning", "info", "", "frobnicate"} {
|
||||
if h := priorityHeaders(sev); h != nil {
|
||||
t.Errorf("%s must have NO priority headers, got %v", sev, h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPriorityHeaders_EndToEnd (Scenario G, seam level): an error mail carries headers on both
|
||||
// channels; a warning mail carries none.
|
||||
func TestPriorityHeaders_EndToEnd(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"node_down", "node_stale"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
|
||||
d.ProcessEvent("c1", "node_down", "error", "down", "{}", "hub")
|
||||
for _, m := range *sent {
|
||||
if m.headers["X-Priority"] != "1" || m.headers["Importance"] != "high" {
|
||||
t.Fatalf("error mail to %s must carry priority headers, got %v", m.to, m.headers)
|
||||
}
|
||||
}
|
||||
|
||||
*sent = (*sent)[:0]
|
||||
d.ProcessEvent("c1", "node_stale", "warning", "stale", "{}", "hub")
|
||||
if len(*sent) == 0 {
|
||||
t.Fatal("warning must still send")
|
||||
}
|
||||
for _, m := range *sent {
|
||||
if m.headers != nil {
|
||||
t.Fatalf("warning mail to %s must carry NO headers, got %v", m.to, m.headers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecovery_SeverityStaysInfo guards the frozen semantics: the recovery branch must not have
|
||||
// widened severityNotifies — a non-recovery info event is still silent.
|
||||
func TestRecovery_SeverityStaysInfo(t *testing.T) {
|
||||
if severityNotifies("info") {
|
||||
t.Fatal("info must remain non-notify — recovery is an eventType branch, not a severity change")
|
||||
}
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"controller_started"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
d.ProcessEvent("c1", "controller_started", "info", "started", "{}", "hub")
|
||||
if len(*sent) != 0 {
|
||||
t.Fatalf("non-recovery info must stay silent, got %d sends", len(*sent))
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestProcessEvent_PoolBoxScopeOperatorOnly(t *testing.T) {
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
d.sendEmailFn = func(to, _, _ string, _ map[string]string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
|
||||
d.ProcessEvent("pool-box", "offsite_box_fill", "warning", "Offsite pool box 82% full", `{"scope":"pool-box"}`, "hub")
|
||||
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
|
||||
@@ -71,7 +71,7 @@ func TestProcessEvent_PBSDRBoxScopeOperatorOnly(t *testing.T) {
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
d.sendEmailFn = func(to, _, _ string, _ map[string]string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
|
||||
d.ProcessEvent("pbsdr-box", "pbsdr_box_fill", "warning", "PBS DR datastore 82% full", `{"scope":"pbsdr-box"}`, "hub")
|
||||
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
|
||||
@@ -85,7 +85,7 @@ func TestProcessEvent_CriticalRoutes(t *testing.T) {
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
d.sendEmailFn = func(to, _, _ string, _ map[string]string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
|
||||
|
||||
d.ProcessEvent("c1", "host_disk_critical", "critical", "root full", "{}", "hub")
|
||||
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
|
||||
@@ -100,7 +100,7 @@ func TestProcessEvent_UnknownSeverityLogged(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
d := NewDispatcher(st, "test-key", "from", "op@felhom.eu", true, log.New(&buf, "", 0))
|
||||
sent := 0
|
||||
d.sendEmailFn = func(_, _, _ string) error { sent++; return nil }
|
||||
d.sendEmailFn = func(_, _, _ string, _ map[string]string) error { sent++; return nil }
|
||||
|
||||
d.ProcessEvent("c1", "weird_event", "frobnicate", "msg", "{}", "hub")
|
||||
if sent != 0 {
|
||||
@@ -118,7 +118,7 @@ func TestProcessEvent_InfoSilent(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
d := NewDispatcher(st, "test-key", "from", "op@felhom.eu", true, log.New(&buf, "", 0))
|
||||
sent := 0
|
||||
d.sendEmailFn = func(_, _, _ string) error { sent++; return nil }
|
||||
d.sendEmailFn = func(_, _, _ string, _ map[string]string) error { sent++; return nil }
|
||||
|
||||
d.ProcessEvent("c1", "controller_started", "info", "msg", "{}", "hub")
|
||||
if sent != 0 {
|
||||
|
||||
@@ -2,6 +2,7 @@ package notify
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -22,8 +23,12 @@ func init() {
|
||||
|
||||
// FormatOperatorEmail returns (subject, textBody) for the operator channel.
|
||||
func FormatOperatorEmail(customerID, eventType, severity, message, detailsJSON string) (string, string) {
|
||||
// Icon is eventType-aware for recovery (v0.71.0, audit F11); severity stays the fallback.
|
||||
icon := "⚠️"
|
||||
if severity == "error" || severity == "critical" {
|
||||
switch {
|
||||
case strings.HasSuffix(eventType, "_recovered"):
|
||||
icon = "✅"
|
||||
case severity == "error" || severity == "critical":
|
||||
icon = "🔴"
|
||||
}
|
||||
|
||||
@@ -86,6 +91,7 @@ var customerMessages = map[string]string{
|
||||
"node_stale": "A szerver nem küldött jelentést az elmúlt időszakban.",
|
||||
"node_down": "A szerver nem elérhető!",
|
||||
"node_recovered": "A szerver újra elérhető.",
|
||||
"host_recovered": "A házszerver alaprendszere (Proxmox-gazda) újra elérhető.",
|
||||
|
||||
// Health events
|
||||
"health_degraded": "A rendszer állapota romlott.",
|
||||
|
||||
Reference in New Issue
Block a user