fix(disk-health): the alert that never sent — severity, a real Hiba level, and a memory that survives a restart
Three defects made the disk-health feature silent in exactly the case it
exists for. Evidence: felhom.eu documentation/audits/DIAG-smart-passed-trap-2026-08-14.md
1. SEVERITY (the one that changes whether anything arrives at all).
NotifyDiskHealthDegraded emitted severity "warn", which is NOT in the
hub's accepted set {info,warning,error,critical}. The hub coerced it to
"info" (hub/internal/api/handler.go) and severityNotifies dropped it
(hub/internal/notify/dispatcher.go), so every Figyelmeztetes-level disk
alert was filed as an informational notice and emailed to NOBODY, on the
customer and the operator leg alike. Now "warning". DiskAlertKind.Severity()
is exported so the contract is checkable from any package.
2. NO LEVEL ABOVE "worth an eye". smart_status.passed CANNOT fail on
unreadable sectors (attrs 187/197/198 all carry thresh 0 and a normalized
value floors at 1), so Hiba was unreachable for this whole fault class.
DiskVerdictFor now takes a DiskPrior and implements a 14-row top-down
ladder: sustained unreadable sectors, a count too large to be a blip (64),
unreadable+remapping together, overheating, NVMe critical flag or spent
endurance all reach Hiba. No fourth label — predicted failure is "Hiba".
3. IT SPOKE ONCE, AND FORGOT ON RESTART. The baseline was in-memory, so a box
that rebooted while a disk was failing never alerted again; and between 8
and 352 sectors nothing was emitted at all. State is now persisted
(disk-health-state.json, atomic tmp+rename), the decision compares against
the last ALERTED verdict (collapsing flaps to one alert while letting a
genuine escalation fire immediately), and a disk already at Hiba re-alerts
once it has BOTH doubled its count and waited out a 24h cooldown.
The card replays the same prior the check used (diskRecord.PriorSawUncorrectable)
so the chip and the email cannot disagree — the property the shared verdict
function exists to guarantee, now pinned rather than asserted.
Tests: 12 scenario groups A-L. Group L builds the Server through web.NewServer,
the same call main.go makes, over a real file.
This commit is contained in:
@@ -5,24 +5,81 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// NotifyDiskHealthDegraded emits the right event type + severity + a message carrying the required
|
||||
// "Lemez állapot romlás: <label>" phrase and the triggering attribute(s). Uses the pushFn seam.
|
||||
func TestNotifyDiskHealthDegraded(t *testing.T) {
|
||||
// hubAcceptedSeverities is the hub's EXACT severity vocabulary, transcribed from
|
||||
// felhom.eu/hub/internal/api/handler.go — the event-ingest handler's `switch payload.Severity` —
|
||||
// where anything outside this set is silently coerced to "info".
|
||||
//
|
||||
// Of these, only warning/error/critical actually reach an email:
|
||||
// felhom.eu/hub/internal/notify/dispatcher.go, func severityNotifies.
|
||||
//
|
||||
// This set is duplicated here ON PURPOSE. A test asserting only `== "warning"` would keep passing if
|
||||
// the hub's vocabulary changed underneath it; pinning the set means the assertion is about the WIRE
|
||||
// CONTRACT, and a reader can check both named locations rather than take this on trust.
|
||||
var hubAcceptedSeverities = map[string]bool{"info": true, "warning": true, "error": true, "critical": true}
|
||||
|
||||
// hubSeverityNotifies mirrors felhom.eu/hub/internal/notify/dispatcher.go:severityNotifies.
|
||||
func hubSeverityNotifies(sev string) bool {
|
||||
switch sev {
|
||||
case "warning", "error", "critical":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Group K — Scenario K: the severity actually routes.
|
||||
//
|
||||
// This is the highest-value assertion in the change. Before v0.215.0 the Figyelmeztetés-level disk
|
||||
// alert carried severity "warn", which is NOT in the hub's accepted set, so the hub coerced it to
|
||||
// "info" and severityNotifies dropped it — the alert reached no email on either the customer or the
|
||||
// operator leg.
|
||||
//
|
||||
// Red-proof: restore `return "warn"` in DiskAlertKind.severity() → the exact-match assertion, the
|
||||
// accepted-set assertion AND the routes-to-email assertion all fail.
|
||||
func TestNotifyDiskHealthDegraded_SeverityRoutes(t *testing.T) {
|
||||
n := &Notifier{}
|
||||
var gotType, gotSev, gotMsg string
|
||||
var gotDetails interface{}
|
||||
n.pushFn = func(eventType, severity, message string, details interface{}) {
|
||||
gotType, gotSev, gotMsg, gotDetails = eventType, severity, message, details
|
||||
var gotSev string
|
||||
n.pushFn = func(eventType, severity, message string, details interface{}) { gotSev = severity }
|
||||
|
||||
n.NotifyDiskHealthDegraded(DiskAlert{Label: "sdb", Kind: DiskAlertWarn,
|
||||
Attributes: []string{"függőben lévő szektorok"}})
|
||||
|
||||
if gotSev != "warning" {
|
||||
t.Errorf("Figyelmeztetés severity = %q, want %q", gotSev, "warning")
|
||||
}
|
||||
if !hubAcceptedSeverities[gotSev] {
|
||||
t.Errorf("severity %q is NOT in the hub's accepted set — the hub coerces it to \"info\"", gotSev)
|
||||
}
|
||||
if !hubSeverityNotifies(gotSev) {
|
||||
t.Errorf("severity %q does not route to email — the alert would reach nobody", gotSev)
|
||||
}
|
||||
|
||||
// Warn (Figyelmeztetés): severity warn, attributes named.
|
||||
n.NotifyDiskHealthDegraded("sdb (lassú)", []string{"függőben lévő szektorok"}, false)
|
||||
// Every Hiba kind must also carry a routing severity.
|
||||
for _, k := range []DiskAlertKind{DiskAlertFailSelfReported, DiskAlertFailSectors, DiskAlertFailTemperature, DiskAlertFailWorsened} {
|
||||
n.NotifyDiskHealthDegraded(DiskAlert{Label: "sdc", Kind: k, Sectors: 352, TemperatureC: 61})
|
||||
if gotSev != "critical" {
|
||||
t.Errorf("kind %d severity = %q, want critical", k, gotSev)
|
||||
}
|
||||
if !hubAcceptedSeverities[gotSev] || !hubSeverityNotifies(gotSev) {
|
||||
t.Errorf("kind %d severity %q does not route", k, gotSev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The event type + details payload + the Figyelmeztetés copy (unchanged wording, now delivered).
|
||||
func TestNotifyDiskHealthDegraded_WarnShape(t *testing.T) {
|
||||
n := &Notifier{}
|
||||
var gotType, gotMsg string
|
||||
var gotDetails interface{}
|
||||
n.pushFn = func(eventType, severity, message string, details interface{}) {
|
||||
gotType, gotMsg, gotDetails = eventType, message, details
|
||||
}
|
||||
|
||||
n.NotifyDiskHealthDegraded(DiskAlert{Label: "sdb (lassú)", Kind: DiskAlertWarn,
|
||||
Attributes: []string{"függőben lévő szektorok"}})
|
||||
|
||||
if gotType != "disk_health_degraded" {
|
||||
t.Errorf("event type = %q, want disk_health_degraded", gotType)
|
||||
}
|
||||
if gotSev != "warn" {
|
||||
t.Errorf("severity = %q, want warn", gotSev)
|
||||
}
|
||||
if !strings.Contains(gotMsg, "Lemez állapot romlás: sdb (lassú)") {
|
||||
t.Errorf("message missing the required subject phrase: %q", gotMsg)
|
||||
}
|
||||
@@ -32,13 +89,81 @@ func TestNotifyDiskHealthDegraded(t *testing.T) {
|
||||
if d, ok := gotDetails.(DiskHealthDetails); !ok || d.Disk != "sdb (lassú)" || d.Critical {
|
||||
t.Errorf("details = %+v, want DiskHealthDetails{Disk:sdb (lassú), Critical:false}", gotDetails)
|
||||
}
|
||||
}
|
||||
|
||||
// Critical (Hiba/FAILING): severity critical.
|
||||
n.NotifyDiskHealthDegraded("sdc", nil, true)
|
||||
if gotSev != "critical" {
|
||||
t.Errorf("critical severity = %q, want critical", gotSev)
|
||||
// Each Hiba kind produces its OWN message, and each names the fact the customer needs to act on.
|
||||
// A customer told "the drive overheated" must not be told to arrange a replacement, and vice versa.
|
||||
//
|
||||
// Red-proof: collapse the switch so every Fail kind emits the self-reported sentence → the sector
|
||||
// count and the temperature disappear from their messages and the substring assertions fail.
|
||||
func TestNotifyDiskHealthDegraded_FailShapes(t *testing.T) {
|
||||
n := &Notifier{}
|
||||
var gotMsg string
|
||||
var gotDetails interface{}
|
||||
n.pushFn = func(eventType, severity, message string, details interface{}) { gotMsg, gotDetails = message, details }
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
alert DiskAlert
|
||||
wantSubstrs []string
|
||||
notSubstrs []string
|
||||
}{
|
||||
{
|
||||
name: "drive self-reports FAILING",
|
||||
alert: DiskAlert{Label: "sdc", Kind: DiskAlertFailSelfReported},
|
||||
wantSubstrs: []string{"Lemez állapot romlás: sdc", "SMART önellenőrzése hibát jelez"},
|
||||
},
|
||||
{
|
||||
name: "reached from sector counters",
|
||||
alert: DiskAlert{Label: "ST3000VX010-2E3166", Kind: DiskAlertFailSectors, Sectors: 352},
|
||||
wantSubstrs: []string{"Lemez hiba: ST3000VX010-2E3166", "352 olvashatatlan szektor", "meghajtó cseréjéhez"},
|
||||
notSubstrs: []string{"SMART önellenőrzése"}, // the drive said PASSED — must not claim otherwise
|
||||
},
|
||||
{
|
||||
name: "reached from heat",
|
||||
alert: DiskAlert{Label: "sdd", Kind: DiskAlertFailTemperature, TemperatureC: 61},
|
||||
wantSubstrs: []string{"Lemez hiba: sdd", "túlmelegedett (61 °C)", "szellőzését"},
|
||||
notSubstrs: []string{"olvashatatlan szektor"},
|
||||
},
|
||||
{
|
||||
name: "already reported, still worsening",
|
||||
alert: DiskAlert{Label: "sdg", Kind: DiskAlertFailWorsened, Sectors: 352},
|
||||
wantSubstrs: []string{"Lemez hiba: sdg", "tovább romlott", "352 olvashatatlan szektor"},
|
||||
},
|
||||
}
|
||||
if !strings.Contains(gotMsg, "SMART önellenőrzése hibát jelez") {
|
||||
t.Errorf("critical message wrong: %q", gotMsg)
|
||||
for _, c := range cases {
|
||||
n.NotifyDiskHealthDegraded(c.alert)
|
||||
for _, w := range c.wantSubstrs {
|
||||
if !strings.Contains(gotMsg, w) {
|
||||
t.Errorf("%s: message %q missing %q", c.name, gotMsg, w)
|
||||
}
|
||||
}
|
||||
for _, bad := range c.notSubstrs {
|
||||
if strings.Contains(gotMsg, bad) {
|
||||
t.Errorf("%s: message %q must not contain %q", c.name, gotMsg, bad)
|
||||
}
|
||||
}
|
||||
if d, ok := gotDetails.(DiskHealthDetails); !ok || !d.Critical {
|
||||
t.Errorf("%s: details %+v, want Critical:true", c.name, gotDetails)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No customer-facing disk copy may carry an emoji (design-system hard rule) and every shape must
|
||||
// name the disk. Cheap, and it covers shapes added later by someone who did not read the rule.
|
||||
func TestNotifyDiskHealthDegraded_CopyDiscipline(t *testing.T) {
|
||||
n := &Notifier{}
|
||||
var gotMsg string
|
||||
n.pushFn = func(eventType, severity, message string, details interface{}) { gotMsg = message }
|
||||
for _, k := range []DiskAlertKind{DiskAlertWarn, DiskAlertFailSelfReported, DiskAlertFailSectors, DiskAlertFailTemperature, DiskAlertFailWorsened} {
|
||||
n.NotifyDiskHealthDegraded(DiskAlert{Label: "TESTDISK", Kind: k, Sectors: 8, TemperatureC: 61})
|
||||
if !strings.Contains(gotMsg, "TESTDISK") {
|
||||
t.Errorf("kind %d: message does not name the disk: %q", k, gotMsg)
|
||||
}
|
||||
for _, r := range gotMsg {
|
||||
if r > 0x2100 { // emoji / symbol block — Hungarian accents are all well below this
|
||||
t.Errorf("kind %d: message contains a non-text rune %q: %s", k, r, gotMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -556,24 +556,78 @@ type DiskHealthDetails struct {
|
||||
Critical bool `json:"critical"`
|
||||
}
|
||||
|
||||
// NotifyDiskHealthDegraded fires a disk_health_degraded hub event on a disk-health DEGRADATION only
|
||||
// (the controller's 6h check owns the transition logic — never first-run/recovery/UNKNOWN). severity
|
||||
// is warn for a Figyelmeztetés, critical for a Hiba (FAILING). The hub applies its own per-event-type
|
||||
// cooldown. NOTE: the event type "disk_health_degraded" MUST be in the hub's allowedEventTypes (else
|
||||
// the hub 400s the POST) — added in the hub's v0.x bump alongside this.
|
||||
func (n *Notifier) NotifyDiskHealthDegraded(label string, attrs []string, critical bool) {
|
||||
severity := "warn"
|
||||
var msg string
|
||||
switch {
|
||||
case critical:
|
||||
severity = "critical"
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez SMART önellenőrzése hibát jelez. Kérjük, mentse az adatait, és vegye fel velünk a kapcsolatot.", label)
|
||||
case len(attrs) > 0:
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — romló érték: %s. Javasolt figyelemmel kísérni.", label, strings.Join(attrs, ", "))
|
||||
default:
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez állapota romlott. Javasolt figyelemmel kísérni.", label)
|
||||
// DiskAlertKind selects the customer-facing message shape. A customer's ACTION differs by kind —
|
||||
// "back up and call us for a replacement" is not "check the ventilation" — so the kind travels with
|
||||
// the alert rather than being flattened into a single sentence.
|
||||
type DiskAlertKind int
|
||||
|
||||
const (
|
||||
DiskAlertWarn DiskAlertKind = iota // Figyelmeztetés — worth keeping an eye on
|
||||
DiskAlertFailSelfReported // Hiba — the drive's own SMART verdict says FAILING
|
||||
DiskAlertFailSectors // Hiba — reached from unreadable-sector counters
|
||||
DiskAlertFailTemperature // Hiba — reached from heat
|
||||
DiskAlertFailWorsened // Hiba — already reported, and still getting worse
|
||||
)
|
||||
|
||||
// DiskAlert is the payload for one disk-health alert. It carries enough for the notifier to pick a
|
||||
// message shape and fill in the counts; message CONSTRUCTION stays here because the notifier owns
|
||||
// customer copy, and moving it to the caller would scatter Hungarian across packages.
|
||||
type DiskAlert struct {
|
||||
Label string // customer-facing disk label (device model where known)
|
||||
Attributes []string // Hungarian attribute names behind the verdict (nil for a self-reported FAILING)
|
||||
Kind DiskAlertKind
|
||||
Sectors int // max(pending, offline_uncorrectable) — quoted in the sector/worsened shapes
|
||||
TemperatureC int // °C — quoted in the temperature shape
|
||||
}
|
||||
|
||||
// Severity is the hub-accepted severity string for this alert.
|
||||
//
|
||||
// THE VOCABULARY IS EXACT AND IT IS THE HUB'S, NOT OURS. The hub accepts only
|
||||
// {"info","warning","error","critical"} and silently COERCES anything else to "info"
|
||||
// (felhom.eu/hub/internal/api/handler.go, the severity switch in the event-ingest handler); "info" is
|
||||
// then dropped by severityNotifies (felhom.eu/hub/internal/notify/dispatcher.go), which routes only
|
||||
// warning/error/critical. So a severity outside that set is stored and emailed to NOBODY — neither
|
||||
// the customer nor the operator leg.
|
||||
//
|
||||
// Exported so any caller — and any test in any package — can check the contract against the two
|
||||
// named hub locations instead of duplicating the literal.
|
||||
//
|
||||
// Until v0.215.0 this function emitted "warn", which is not in the set. Every Figyelmeztetés-level
|
||||
// disk alert the product ever produced was filed as an informational notice and delivered to no one.
|
||||
func (k DiskAlertKind) Severity() string {
|
||||
if k == DiskAlertWarn {
|
||||
return "warning"
|
||||
}
|
||||
n.emit("disk_health_degraded", severity, msg, DiskHealthDetails{Disk: label, Attributes: attrs, Critical: critical})
|
||||
return "critical"
|
||||
}
|
||||
|
||||
// NotifyDiskHealthDegraded fires a disk_health_degraded hub event. The controller's periodic check
|
||||
// owns the decision to call this at all (transitions, flap damping, the re-alert cooldown) — never
|
||||
// first-run, never recovery, never UNKNOWN.
|
||||
//
|
||||
// The hub applies its own per-event-type cooldown ON TOP of ours. NOTE: the event type
|
||||
// "disk_health_degraded" MUST be in the hub's allowedEventTypes (else the hub 400s the POST) — it is.
|
||||
func (n *Notifier) NotifyDiskHealthDegraded(a DiskAlert) {
|
||||
critical := a.Kind != DiskAlertWarn
|
||||
var msg string
|
||||
switch a.Kind {
|
||||
case DiskAlertFailSelfReported:
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez SMART önellenőrzése hibát jelez. Kérjük, mentse az adatait, és vegye fel velünk a kapcsolatot.", a.Label)
|
||||
case DiskAlertFailSectors:
|
||||
msg = fmt.Sprintf("Lemez hiba: %s — a meghajtón %d olvashatatlan szektor van. Mentse az adatait, és keressen meg minket a meghajtó cseréjéhez.", a.Label, a.Sectors)
|
||||
case DiskAlertFailTemperature:
|
||||
msg = fmt.Sprintf("Lemez hiba: %s — a meghajtó túlmelegedett (%d °C). Ellenőrizze a gép szellőzését, és keressen meg minket.", a.Label, a.TemperatureC)
|
||||
case DiskAlertFailWorsened:
|
||||
msg = fmt.Sprintf("Lemez hiba: %s — a meghajtó állapota tovább romlott, már %d olvashatatlan szektor van. Ha még nem tette meg, mentse az adatait.", a.Label, a.Sectors)
|
||||
default: // DiskAlertWarn
|
||||
if len(a.Attributes) > 0 {
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — romló érték: %s. Javasolt figyelemmel kísérni.", a.Label, strings.Join(a.Attributes, ", "))
|
||||
} else {
|
||||
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez állapota romlott. Javasolt figyelemmel kísérni.", a.Label)
|
||||
}
|
||||
}
|
||||
n.emit("disk_health_degraded", a.Kind.Severity(), msg,
|
||||
DiskHealthDetails{Disk: a.Label, Attributes: a.Attributes, Critical: critical})
|
||||
}
|
||||
|
||||
// emit sends an event through the test seam if set, else the real async PushEvent.
|
||||
|
||||
Reference in New Issue
Block a user