fix(disk-health): the alert that never sent — severity, a real Hiba level, and a memory that survives a restart
Three defects made the disk-health feature silent in exactly the case it
exists for. Evidence: felhom.eu documentation/audits/DIAG-smart-passed-trap-2026-08-14.md
1. SEVERITY (the one that changes whether anything arrives at all).
NotifyDiskHealthDegraded emitted severity "warn", which is NOT in the
hub's accepted set {info,warning,error,critical}. The hub coerced it to
"info" (hub/internal/api/handler.go) and severityNotifies dropped it
(hub/internal/notify/dispatcher.go), so every Figyelmeztetes-level disk
alert was filed as an informational notice and emailed to NOBODY, on the
customer and the operator leg alike. Now "warning". DiskAlertKind.Severity()
is exported so the contract is checkable from any package.
2. NO LEVEL ABOVE "worth an eye". smart_status.passed CANNOT fail on
unreadable sectors (attrs 187/197/198 all carry thresh 0 and a normalized
value floors at 1), so Hiba was unreachable for this whole fault class.
DiskVerdictFor now takes a DiskPrior and implements a 14-row top-down
ladder: sustained unreadable sectors, a count too large to be a blip (64),
unreadable+remapping together, overheating, NVMe critical flag or spent
endurance all reach Hiba. No fourth label — predicted failure is "Hiba".
3. IT SPOKE ONCE, AND FORGOT ON RESTART. The baseline was in-memory, so a box
that rebooted while a disk was failing never alerted again; and between 8
and 352 sectors nothing was emitted at all. State is now persisted
(disk-health-state.json, atomic tmp+rename), the decision compares against
the last ALERTED verdict (collapsing flaps to one alert while letting a
genuine escalation fire immediately), and a disk already at Hiba re-alerts
once it has BOTH doubled its count and waited out a 24h cooldown.
The card replays the same prior the check used (diskRecord.PriorSawUncorrectable)
so the chip and the email cannot disagree — the property the shared verdict
function exists to guarantee, now pinned rather than asserted.
Tests: 12 scenario groups A-L. Group L builds the Server through web.NewServer,
the same call main.go makes, over a real file.
This commit is contained in:
@@ -2,122 +2,628 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// Disk-health severity ladder, event half (v0.215.0). Scenario letters refer to the task spec.
|
||||
// The verdict half is pinned in internal/agentapi/diskverdict_ladder_test.go.
|
||||
|
||||
func smartPtr(v int) *int { return &v }
|
||||
|
||||
func physDisk(name string, sm *agentapi.SmartSummary) agentapi.DiskInfo {
|
||||
return agentapi.DiskInfo{Name: name, BackingDevice: "/dev/" + name, DurableID: "uuid:" + name, Smart: sm}
|
||||
}
|
||||
|
||||
// diskCheckHarness wires a Server with the disks source + notify sink seams and returns a captured
|
||||
// list of emitted disk labels.
|
||||
func diskCheckHarness(t *testing.T) (*Server, *[]string, *[]agentapi.DiskInfo) {
|
||||
// realDriveSmart is the failing drive as captured on 2026-08-14 — PASSED, 352 unreadable sectors.
|
||||
// felhom.eu/documentation/audits/fixtures/smart-ST3000VX010-failing-2026-08-14.json
|
||||
func realDriveSmart() *agentapi.SmartSummary {
|
||||
return &agentapi.SmartSummary{
|
||||
Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(352),
|
||||
OfflineUncorrectable: smartPtr(352),
|
||||
ReallocatedSectors: smartPtr(0),
|
||||
TemperatureC: smartPtr(40),
|
||||
}
|
||||
}
|
||||
|
||||
// diskHarness wires a Server with the disks source + notify sink seams, a REAL data dir (so the
|
||||
// persisted state is exercised, not bypassed) and a controllable clock.
|
||||
type diskHarness struct {
|
||||
s *Server
|
||||
fired *[]notify.DiskAlert
|
||||
payload *[]agentapi.DiskInfo
|
||||
dir string
|
||||
clock *time.Time
|
||||
}
|
||||
|
||||
func newDiskHarness(t *testing.T) *diskHarness {
|
||||
t.Helper()
|
||||
s := testServer(t)
|
||||
var fired []string
|
||||
payload := &[]agentapi.DiskInfo{}
|
||||
s.diskNotifyFn = func(label string, attrs []string, critical bool) { fired = append(fired, label) }
|
||||
dir := t.TempDir()
|
||||
lg := log.New(io.Discard, "", 0)
|
||||
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
||||
if err != nil {
|
||||
t.Fatalf("settings: %v", err)
|
||||
}
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = dir
|
||||
s := &Server{settings: sett, logger: lg, cfg: cfg}
|
||||
|
||||
now := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
||||
h := &diskHarness{s: s, dir: dir, clock: &now, fired: &[]notify.DiskAlert{}, payload: &[]agentapi.DiskInfo{}}
|
||||
s.diskHealth.clock = func() time.Time { return *h.clock }
|
||||
s.diskNotifyFn = func(a notify.DiskAlert) { *h.fired = append(*h.fired, a) }
|
||||
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
||||
return agentapi.DisksResponse{Disks: *payload}, nil
|
||||
return agentapi.DisksResponse{Disks: *h.payload}, nil
|
||||
}
|
||||
return s, &fired, payload
|
||||
return h
|
||||
}
|
||||
|
||||
// Scenario B — first run baselines silently; a real degradation (OK→Warn) emits exactly once; a
|
||||
// steady-state re-check does not re-emit. Red-proof: remove the `firstRun || !had` guard → the first
|
||||
// run emits and the "no notify on first run" assertion fails.
|
||||
func TestDiskHealthCheck_DegradationOnce(t *testing.T) {
|
||||
s, fired, payload := diskCheckHarness(t)
|
||||
ctx := context.Background()
|
||||
func (h *diskHarness) set(d ...agentapi.DiskInfo) { *h.payload = d }
|
||||
func (h *diskHarness) run(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
||||
t.Fatalf("RunDiskHealthCheck: %v", err)
|
||||
}
|
||||
}
|
||||
func (h *diskHarness) advance(d time.Duration) { *h.clock = h.clock.Add(d) }
|
||||
func (h *diskHarness) events() []notify.DiskAlert {
|
||||
return *h.fired
|
||||
}
|
||||
func (h *diskHarness) reset() { *h.fired = nil }
|
||||
|
||||
// First check: disk PASSED clean (verdict OK). Baseline only.
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed})}
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
if len(*fired) != 0 {
|
||||
t.Fatalf("first run must not notify, got %v", *fired)
|
||||
// record reads the persisted state file straight off disk — asserting the FILE, not the in-memory
|
||||
// map, because Scenario L depends on what actually landed there.
|
||||
func (h *diskHarness) record(t *testing.T, key string) *diskRecord {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(h.dir, diskStateFileName))
|
||||
if err != nil {
|
||||
t.Fatalf("reading state file: %v", err)
|
||||
}
|
||||
var f diskStateFile
|
||||
if err := json.Unmarshal(data, &f); err != nil {
|
||||
t.Fatalf("parsing state file: %v", err)
|
||||
}
|
||||
return f.Disks[key]
|
||||
}
|
||||
|
||||
// ── Group A — Scenario A: the real drive, second observation ────────────────────────────────────
|
||||
|
||||
// The captured failing drive reaches Hiba, renders the crit chip, and emits exactly ONE event at
|
||||
// severity "critical".
|
||||
//
|
||||
// Red-proof: remove truth-table row 6 (prior.SawUncorrectable) → 352 still trips row 8, so use
|
||||
// TestDiskLadder_SustainDrivesTheEscalation (Group C) to isolate row 6.
|
||||
func TestDiskCheck_RealDrive_HibaAndOneCriticalEvent(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
h.set(physDisk("sdg", realDriveSmart()))
|
||||
|
||||
h.run(t) // first ever observation: baselines silently
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Fatalf("first observation must be silent, got %d event(s)", n)
|
||||
}
|
||||
|
||||
// Degrade: pending sectors 0→5 (OK→Figyelmeztetés).
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(5)})}
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
if len(*fired) != 1 {
|
||||
t.Fatalf("degradation must emit exactly once, got %v", *fired)
|
||||
h.run(t) // second observation, now with prior
|
||||
ev := h.events()
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("want exactly ONE event, got %d: %+v", len(ev), ev)
|
||||
}
|
||||
if ev[0].Kind.Severity() != "critical" {
|
||||
t.Errorf("severity = %q, want %q", ev[0].Kind.Severity(), "critical")
|
||||
}
|
||||
if ev[0].Kind != notify.DiskAlertFailSectors {
|
||||
t.Errorf("kind = %d, want DiskAlertFailSectors (the drive says PASSED — we must not claim it self-reported)", ev[0].Kind)
|
||||
}
|
||||
if ev[0].Sectors != 352 {
|
||||
t.Errorf("sectors = %d, want 352", ev[0].Sectors)
|
||||
}
|
||||
|
||||
// Steady state: still Figyelmeztetés — no repeat.
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
if len(*fired) != 1 {
|
||||
t.Fatalf("steady-state degraded must not re-emit, got %v", *fired)
|
||||
// The card must agree with the alert — same verdict function, same prior.
|
||||
rows := h.s.diskHealthRows(context.Background())
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("want 1 card row, got %d", len(rows))
|
||||
}
|
||||
if rows[0].ChipLabel != "Hiba" {
|
||||
t.Errorf("chip label = %q, want %q", rows[0].ChipLabel, "Hiba")
|
||||
}
|
||||
if rows[0].ChipClass != "state-text-crit" {
|
||||
t.Errorf("chip class = %q, want %q", rows[0].ChipClass, "state-text-crit")
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario B (cont.) — recovery (Figyelmeztetés→Rendben) notifies nothing.
|
||||
func TestDiskHealthCheck_RecoverySilent(t *testing.T) {
|
||||
s, fired, payload := diskCheckHarness(t)
|
||||
ctx := context.Background()
|
||||
// ── Group B — Scenario B: the transient that cleared (11 August) ────────────────────────────────
|
||||
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // baseline OK
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(5)})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // OK→Warn: emits
|
||||
if len(*fired) != 1 {
|
||||
t.Fatalf("expected 1 emit on degradation, got %v", *fired)
|
||||
// A first-ever sighting of 8 sectors is Figyelmeztetés, not Hiba. The real drive did exactly this on
|
||||
// 11 Aug 12:28 and had cleared completely by 13:28.
|
||||
//
|
||||
// Red-proof: make truth-table row 9 return Fail → the chip reads Hiba and the kind assertion fails.
|
||||
func TestDiskCheck_FirstSightingIsWarnOnly(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
excursion := &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)}
|
||||
h.set(physDisk("sdg", excursion))
|
||||
h.run(t)
|
||||
|
||||
rows := h.s.diskHealthRows(context.Background())
|
||||
if len(rows) != 1 || rows[0].ChipLabel != "Figyelmeztetés" {
|
||||
t.Fatalf("first sighting chip = %+v, want Figyelmeztetés", rows)
|
||||
}
|
||||
// Recover back to clean.
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed})}
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
if len(*fired) != 1 {
|
||||
t.Errorf("recovery must not notify, got %v", *fired)
|
||||
rec := h.record(t, "uuid:sdg")
|
||||
if rec == nil {
|
||||
t.Fatal("no persisted record for the disk")
|
||||
}
|
||||
if !rec.SawUncorrectable {
|
||||
t.Error("persisted state must record SawUncorrectable=true so the NEXT check can sustain")
|
||||
}
|
||||
if rec.Sectors != 8 {
|
||||
t.Errorf("persisted sectors = %d, want 8", rec.Sectors)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario C — UNKNOWN is excluded both directions: a UNKNOWN disk never baselines/emits, and an
|
||||
// OK→UNKNOWN→Warn sequence fires on the real OK→Warn (the UNKNOWN blip is ignored, not treated as a
|
||||
// transition). Red-proof: the truth of "excluded both directions" — if UNKNOWN were recorded as a
|
||||
// verdict, UNKNOWN→Warn would look like a degradation from a low baseline.
|
||||
func TestDiskHealthCheck_UnknownExcluded(t *testing.T) {
|
||||
s, fired, payload := diskCheckHarness(t)
|
||||
ctx := context.Background()
|
||||
// ── Group C — Scenario C: the same disk one check later, still bad ──────────────────────────────
|
||||
|
||||
// A purely-UNKNOWN disk: first run + repeat, never notifies, never records.
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartUnknown})}
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
_ = s.RunDiskHealthCheck(ctx)
|
||||
if len(*fired) != 0 {
|
||||
t.Fatalf("UNKNOWN disk must never notify, got %v", *fired)
|
||||
// The count does NOT grow — only the fact that it persisted. This isolates truth-table row 6: at 8
|
||||
// sectors the count backstop (64) cannot be what fires.
|
||||
//
|
||||
// Red-proof: drop the prior argument's effect (always pass a zero DiskPrior in RunDiskHealthCheck)
|
||||
// → the disk stays at Figyelmeztetés forever and no event is emitted.
|
||||
func TestDiskLadder_SustainDrivesTheEscalation(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
excursion := func() agentapi.DiskInfo {
|
||||
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)})
|
||||
}
|
||||
h.set(excursion())
|
||||
h.run(t) // first sighting → silent baseline at Figyelmeztetés
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Fatalf("first sighting must be silent, got %d", n)
|
||||
}
|
||||
|
||||
// OK baseline, then a UNKNOWN blip, then Warn — must fire once (OK→Warn), the blip ignored.
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdc", &agentapi.SmartSummary{Health: agentapi.SmartPassed})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // baseline OK
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdc", &agentapi.SmartSummary{Health: agentapi.SmartUnknown})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // UNKNOWN blip: no change, no emit
|
||||
if len(*fired) != 0 {
|
||||
t.Fatalf("UNKNOWN blip must not emit, got %v", *fired)
|
||||
h.set(excursion())
|
||||
h.run(t) // SAME counters, now sustained → Hiba
|
||||
ev := h.events()
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("sustained excursion must emit exactly ONE event, got %d: %+v", len(ev), ev)
|
||||
}
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdc", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(3)})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // OK→Warn (the blip was ignored): fire once
|
||||
if len(*fired) != 1 {
|
||||
t.Fatalf("real OK→Warn after a UNKNOWN blip must fire once, got %v", *fired)
|
||||
if ev[0].Kind.Severity() != "critical" {
|
||||
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
||||
}
|
||||
if ev[0].Sectors != 8 {
|
||||
t.Errorf("sectors = %d, want 8 — the count did not grow; SUSTAIN is what fired", ev[0].Sectors)
|
||||
}
|
||||
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
||||
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario C — the card renders gracefully with nil SMART (old agent / no data): the row shows
|
||||
// "Nincs adat" and never errors; a physical disk with no smart field is still listed.
|
||||
// ── Group D — Scenario D: recovered ─────────────────────────────────────────────────────────────
|
||||
|
||||
// Red-proof: make recovery emit → an event appears where zero are expected.
|
||||
func TestDiskCheck_RecoveryIsSilentAndClearsState(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
h.reset()
|
||||
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(0), OfflineUncorrectable: smartPtr(0), ReallocatedSectors: smartPtr(0)}))
|
||||
h.run(t)
|
||||
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("recovery must be silent, got %d event(s): %+v", n, h.events())
|
||||
}
|
||||
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Rendben" {
|
||||
t.Errorf("chip = %q, want Rendben", rows[0].ChipLabel)
|
||||
}
|
||||
rec := h.record(t, "uuid:sdg")
|
||||
if rec.SawUncorrectable {
|
||||
t.Error("recovery must clear SawUncorrectable, else the disk re-escalates on the next blip")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group E — Scenario E: the flap must not spam ────────────────────────────────────────────────
|
||||
|
||||
// Warn (alerted) → OK → Warn again, all inside 24h: exactly ONE event across all three checks.
|
||||
// Today's transition-only logic emits twice on that zero-crossing, which is how a customer learns to
|
||||
// ignore the message.
|
||||
//
|
||||
// Red-proof: compare against the last OBSERVED verdict instead of the last ALERTED one → two events.
|
||||
func TestDiskCheck_FlapDamping(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
warn := func() agentapi.DiskInfo {
|
||||
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)})
|
||||
}
|
||||
clean := func() agentapi.DiskInfo {
|
||||
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)})
|
||||
}
|
||||
// Baseline clean first so the Warn below is a real transition rather than a first sighting.
|
||||
h.set(clean())
|
||||
h.run(t)
|
||||
|
||||
h.set(warn())
|
||||
h.run(t) // OK→Warn: the one legitimate alert
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Fatalf("first degradation must emit once, got %d", n)
|
||||
}
|
||||
h.advance(2 * time.Hour)
|
||||
h.set(clean())
|
||||
h.run(t) // recovery: silent
|
||||
h.advance(2 * time.Hour)
|
||||
h.set(warn())
|
||||
h.run(t) // re-degradation to the SAME level within 24h: damped
|
||||
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Errorf("a flap inside 24h must produce exactly ONE event, got %d: %+v", n, h.events())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group F — Scenario F: escalation always beats damping ───────────────────────────────────────
|
||||
|
||||
// Red-proof: let damping cover escalations (return false whenever a verdict was ever alerted) →
|
||||
// zero events, and a drive that has started actually failing says nothing.
|
||||
func TestDiskCheck_EscalationBeatsDamping(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)}))
|
||||
h.run(t)
|
||||
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)}))
|
||||
h.run(t) // OK→Warn, alerted
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Fatalf("setup: want 1 event, got %d", n)
|
||||
}
|
||||
h.reset()
|
||||
|
||||
// Two hours later — well inside the damping window — it reaches Hiba.
|
||||
h.advance(2 * time.Hour)
|
||||
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartFailing, ReallocatedSectors: smartPtr(1)}))
|
||||
h.run(t)
|
||||
|
||||
ev := h.events()
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("an escalation inside the damping window MUST emit, got %d", len(ev))
|
||||
}
|
||||
if ev[0].Kind.Severity() != "critical" {
|
||||
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
||||
}
|
||||
if ev[0].Kind != notify.DiskAlertFailSelfReported {
|
||||
t.Errorf("kind = %d, want DiskAlertFailSelfReported", ev[0].Kind)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group G — Scenario G: still getting worse ───────────────────────────────────────────────────
|
||||
|
||||
// A disk already at Hiba, last alerted at 64 sectors, now at 352, 24h later: a SECOND event naming
|
||||
// the current count. Today's code emits nothing at all between 8 and 352 sectors.
|
||||
//
|
||||
// Red-proof: remove the re-alert branch from diskAlertDecision → one event only.
|
||||
func TestDiskCheck_RealertWhenStillWorsening(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
at := func(n int) agentapi.DiskInfo {
|
||||
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
||||
}
|
||||
h.set(at(64))
|
||||
h.run(t) // first sighting → silent (already Hiba via row 8, but baselined)
|
||||
h.set(at(64))
|
||||
h.run(t) // sustained → Hiba, alerted at 64
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Fatalf("setup: want 1 event, got %d", n)
|
||||
}
|
||||
h.reset()
|
||||
|
||||
h.advance(25 * time.Hour)
|
||||
h.set(at(352))
|
||||
h.run(t)
|
||||
|
||||
ev := h.events()
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("a doubling after the cooldown must re-alert, got %d event(s)", len(ev))
|
||||
}
|
||||
if ev[0].Kind != notify.DiskAlertFailWorsened {
|
||||
t.Errorf("kind = %d, want DiskAlertFailWorsened", ev[0].Kind)
|
||||
}
|
||||
if ev[0].Sectors != 352 {
|
||||
t.Errorf("re-alert must name the CURRENT count, got %d", ev[0].Sectors)
|
||||
}
|
||||
if ev[0].Kind.Severity() != "critical" {
|
||||
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group H — Scenario H: growth below the bar, and inside the cooldown ─────────────────────────
|
||||
|
||||
// Red-proof: make the cooldown and the doubling bar an OR instead of an AND → an event appears.
|
||||
func TestDiskCheck_NoRealertBelowBothBars(t *testing.T) {
|
||||
at := func(n int) agentapi.DiskInfo {
|
||||
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
||||
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
||||
}
|
||||
// Neither bar cleared: 352 → 360 after 2h.
|
||||
h := newDiskHarness(t)
|
||||
h.set(at(352))
|
||||
h.run(t)
|
||||
h.set(at(352))
|
||||
h.run(t) // alerted at 352
|
||||
h.reset()
|
||||
h.advance(2 * time.Hour)
|
||||
h.set(at(360))
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("neither bar cleared → no event, got %d: %+v", n, h.events())
|
||||
}
|
||||
|
||||
// ONLY the cooldown cleared (25h) but growth is tiny → still silent. Pins the AND.
|
||||
h.advance(25 * time.Hour)
|
||||
h.set(at(400))
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("cooldown alone must not re-alert (AND, not OR), got %d: %+v", n, h.events())
|
||||
}
|
||||
|
||||
// ONLY the doubling cleared (2h later) → still silent. Pins the other half of the AND.
|
||||
h2 := newDiskHarness(t)
|
||||
h2.set(at(100))
|
||||
h2.run(t)
|
||||
h2.set(at(100))
|
||||
h2.run(t) // alerted at 100
|
||||
h2.reset()
|
||||
h2.advance(2 * time.Hour)
|
||||
h2.set(at(400))
|
||||
h2.run(t)
|
||||
if n := len(h2.events()); n != 0 {
|
||||
t.Errorf("doubling alone inside the cooldown must not re-alert, got %d: %+v", n, h2.events())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group I — Scenario I: heat ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// The event half of the temperature ladder: a hot disk alerts with the temperature SHAPE, so the
|
||||
// customer is told to check ventilation rather than to arrange a replacement.
|
||||
//
|
||||
// Red-proof: remove truth-table rows 3 and 13 → the disk reads Rendben and nothing is emitted.
|
||||
func TestDiskCheck_TemperatureShape(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
cool := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(40)})
|
||||
hot := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(61)})
|
||||
h.set(cool)
|
||||
h.run(t)
|
||||
h.set(hot)
|
||||
h.run(t)
|
||||
|
||||
ev := h.events()
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("overheating must emit once, got %d", len(ev))
|
||||
}
|
||||
if ev[0].Kind != notify.DiskAlertFailTemperature {
|
||||
t.Errorf("kind = %d, want DiskAlertFailTemperature", ev[0].Kind)
|
||||
}
|
||||
if ev[0].TemperatureC != 61 {
|
||||
t.Errorf("temperature = %d, want 61", ev[0].TemperatureC)
|
||||
}
|
||||
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
||||
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group J — Scenario J: no data never alarms, and never erases history ────────────────────────
|
||||
|
||||
// Red-proof: let UNKNOWN write or delete the record → the disk re-baselines and its history is lost,
|
||||
// so the sustained escalation afterwards never fires.
|
||||
func TestDiskCheck_UnknownNeverAlarmsNorErasesPrior(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
// Establish a real record with SawUncorrectable set.
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
before := h.record(t, "uuid:sdg")
|
||||
if !before.SawUncorrectable {
|
||||
t.Fatal("setup: expected SawUncorrectable")
|
||||
}
|
||||
h.reset()
|
||||
|
||||
// A transient unreadable SMART, and separately a nil one.
|
||||
for _, sm := range []*agentapi.SmartSummary{{Health: agentapi.SmartUnknown}, nil} {
|
||||
h.set(physDisk("sdg", sm))
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Fatalf("no-data disk must never alarm, got %d", n)
|
||||
}
|
||||
after := h.record(t, "uuid:sdg")
|
||||
if after == nil {
|
||||
t.Fatal("UNKNOWN deleted the prior record — a transient blip must not erase history")
|
||||
}
|
||||
if after.SawUncorrectable != before.SawUncorrectable || after.Verdict != before.Verdict {
|
||||
t.Errorf("UNKNOWN overwrote the prior: before=%+v after=%+v", before, after)
|
||||
}
|
||||
if rows := h.s.diskHealthRows(context.Background()); len(rows) != 1 || rows[0].ChipLabel != "Nincs adat" {
|
||||
t.Errorf("chip = %+v, want Nincs adat", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// History survived, so the real sustained escalation still fires.
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Errorf("the sustained escalation must still fire after a no-data blip, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group L — Scenario L: state survives a restart (SEAM TEST) ──────────────────────────────────
|
||||
|
||||
// Constructed through the PRODUCTION path — web.NewServer, exactly as cmd/controller/main.go builds
|
||||
// it — over the same data dir, reading and writing a real file. An injected-seam test proves the
|
||||
// component; it never proves the caller. This project has shipped seven built-but-never-wired
|
||||
// defects, and the severity bug being fixed here is arguably the seventh.
|
||||
//
|
||||
// Red-proof: skip loadDiskStateLocked in RunDiskHealthCheck → the restarted controller silently
|
||||
// re-baselines a disk it has already reported, and never alerts on it again. That is today's bug.
|
||||
func TestDiskCheck_StateSurvivesRestart_ProductionPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
lg := log.New(io.Discard, "", 0)
|
||||
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
||||
if err != nil {
|
||||
t.Fatalf("settings: %v", err)
|
||||
}
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = dir
|
||||
|
||||
failing := []agentapi.DiskInfo{physDisk("sdg", realDriveSmart())}
|
||||
newProdServer := func() (*Server, *[]notify.DiskAlert) {
|
||||
// The same call cmd/controller/main.go makes.
|
||||
s := NewServer(cfg, nil, nil, nil, nil, sett, nil, nil, nil, lg, "test")
|
||||
fired := &[]notify.DiskAlert{}
|
||||
s.diskNotifyFn = func(a notify.DiskAlert) { *fired = append(*fired, a) }
|
||||
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
||||
return agentapi.DisksResponse{Disks: failing}, nil
|
||||
}
|
||||
return s, fired
|
||||
}
|
||||
|
||||
s1, fired1 := newProdServer()
|
||||
_ = s1.RunDiskHealthCheck(context.Background()) // baseline
|
||||
_ = s1.RunDiskHealthCheck(context.Background()) // sustained → Hiba, one alert
|
||||
if len(*fired1) != 1 {
|
||||
t.Fatalf("setup: want 1 alert before the restart, got %d", len(*fired1))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, diskStateFileName)); err != nil {
|
||||
t.Fatalf("the production path must have WRITTEN the state file: %v", err)
|
||||
}
|
||||
|
||||
// Restart: a brand-new Server over the same data dir, disk still failing.
|
||||
s2, fired2 := newProdServer()
|
||||
_ = s2.RunDiskHealthCheck(context.Background())
|
||||
if n := len(*fired2); n != 0 {
|
||||
t.Errorf("a restarted controller must NOT re-alert an already-reported disk, got %d: %+v", n, *fired2)
|
||||
}
|
||||
|
||||
// And it must still know the disk is bad — the chip cannot silently drop to Figyelmeztetés.
|
||||
rows := s2.diskHealthRows(context.Background())
|
||||
if len(rows) != 1 || rows[0].ChipLabel != "Hiba" {
|
||||
t.Errorf("after restart the card = %+v, want Hiba", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A corrupt state file must LOG and fall back to "no prior" — never crash the check and never take
|
||||
// the box down. The fail-safe direction is a duplicate alert, not a missing one.
|
||||
func TestDiskState_CorruptFileFallsBackToNoPrior(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
if err := os.WriteFile(filepath.Join(h.dir, diskStateFileName), []byte("{not json"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.set(physDisk("sdg", realDriveSmart()))
|
||||
h.run(t) // must not panic; treated as first-ever → silent
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("corrupt state → treated as no prior → silent baseline, got %d", n)
|
||||
}
|
||||
// And the next check works normally, proving the check was not left wedged.
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 1 {
|
||||
t.Errorf("check must keep working after a corrupt state file, got %d events", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A disk that disappears from the report is forgotten, so a reappearance re-baselines silently
|
||||
// (behaviour preserved from v0.169.0 — persisting the state must not change it).
|
||||
func TestDiskCheck_DisappearedDiskIsForgotten(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
h.set() // drive removed
|
||||
h.run(t)
|
||||
if r := h.record(t, "uuid:sdg"); r != nil {
|
||||
t.Errorf("a disappeared disk must be dropped, got %+v", r)
|
||||
}
|
||||
// Reappears, still bad: first sighting again → silent, NOT an immediate Hiba.
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("a reappearing disk must re-baseline silently, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// An unreachable agent changes nothing at all — no state churn, no alarm, no error.
|
||||
func TestDiskCheck_UnreachableAgentIsInert(t *testing.T) {
|
||||
h := newDiskHarness(t)
|
||||
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
||||
h.run(t)
|
||||
before := h.record(t, "uuid:sdg")
|
||||
|
||||
h.s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
||||
return agentapi.DisksResponse{}, context.DeadlineExceeded
|
||||
}
|
||||
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
||||
t.Errorf("an unreachable agent must return nil, got %v", err)
|
||||
}
|
||||
after := h.record(t, "uuid:sdg")
|
||||
if after == nil || after.SawUncorrectable != before.SawUncorrectable {
|
||||
t.Errorf("an unreachable agent must not churn state: before=%+v after=%+v", before, after)
|
||||
}
|
||||
if n := len(h.events()); n != 0 {
|
||||
t.Errorf("an unreachable agent must not alarm, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// diskAlertDecision is pure — pin its table directly, including the boundaries the scenarios above
|
||||
// only reach indirectly.
|
||||
func TestDiskAlertDecision_Table(t *testing.T) {
|
||||
base := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
||||
alertedFail := func(sectors int, ago time.Duration) *diskRecord {
|
||||
return &diskRecord{Verdict: int(agentapi.DiskVerdictFail), Alerted: true,
|
||||
AlertedVerdict: int(agentapi.DiskVerdictFail), AlertedSectors: sectors, AlertedAt: base.Add(-ago)}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
prev *diskRecord
|
||||
v agentapi.DiskVerdict
|
||||
sectors int
|
||||
wantEmit bool
|
||||
wantWorsened bool
|
||||
}{
|
||||
{"first ever, Warn", nil, agentapi.DiskVerdictWarn, 8, false, false},
|
||||
{"first ever, Fail", nil, agentapi.DiskVerdictFail, 352, false, false},
|
||||
{"OK is always silent", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn)}, agentapi.DiskVerdictOK, 0, false, false},
|
||||
{"never alerted, now Warn", &diskRecord{Verdict: int(agentapi.DiskVerdictOK)}, agentapi.DiskVerdictWarn, 8, true, false},
|
||||
{"escalate Warn→Fail", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn), Alerted: true,
|
||||
AlertedVerdict: int(agentapi.DiskVerdictWarn), AlertedAt: base}, agentapi.DiskVerdictFail, 8, true, false},
|
||||
{"steady Fail, no growth, no time", alertedFail(352, time.Hour), agentapi.DiskVerdictFail, 352, false, false},
|
||||
{"exactly 2x but only 23h", alertedFail(64, 23 * time.Hour), agentapi.DiskVerdictFail, 128, false, false},
|
||||
{"exactly 2x at exactly 24h", alertedFail(64, 24 * time.Hour), agentapi.DiskVerdictFail, 128, true, true},
|
||||
{"25h but only 1.9x", alertedFail(64, 25 * time.Hour), agentapi.DiskVerdictFail, 121, false, false},
|
||||
{"improved Fail→Warn is silent", alertedFail(352, 48 * time.Hour), agentapi.DiskVerdictWarn, 8, false, false},
|
||||
{"Fail from heat (0 sectors) never re-alerts on the doubling rule",
|
||||
alertedFail(0, 48 * time.Hour), agentapi.DiskVerdictFail, 0, false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
emit, worsened := diskAlertDecision(c.prev, c.v, c.sectors, base)
|
||||
if emit != c.wantEmit || worsened != c.wantWorsened {
|
||||
t.Errorf("%s: got (emit=%v worsened=%v), want (emit=%v worsened=%v)",
|
||||
c.name, emit, worsened, c.wantEmit, c.wantWorsened)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Preserved v0.169.0/v0.171.0 coverage ────────────────────────────────────────────────────────
|
||||
|
||||
// The card renders gracefully with nil SMART (old agent / no data): the row shows "Nincs adat" and
|
||||
// never errors; a physical disk with no smart field is still listed.
|
||||
func TestDiskHealthRows_NilSmart(t *testing.T) {
|
||||
s, _, payload := diskCheckHarness(t)
|
||||
*payload = []agentapi.DiskInfo{
|
||||
{Name: "sdb", BackingDevice: "/dev/sdb", Smart: nil}, // physical, no smart → Nincs adat
|
||||
h := newDiskHarness(t)
|
||||
h.set(
|
||||
agentapi.DiskInfo{Name: "sdb", BackingDevice: "/dev/sdb", Smart: nil}, // physical, no smart → Nincs adat
|
||||
// PBS/LVM carry a default UNKNOWN SMART from the agent — must STILL be excluded (not disks).
|
||||
{Name: "felhom-pbs", Type: "pbs", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
||||
{Name: "local-lvm", Type: "lvmthin", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
||||
{Name: "sdc", BackingDevice: "/dev/sdc", Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(31)}}, // Rendben, 31°C
|
||||
}
|
||||
rows := s.diskHealthRows(context.Background())
|
||||
agentapi.DiskInfo{Name: "felhom-pbs", Type: "pbs", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
||||
agentapi.DiskInfo{Name: "local-lvm", Type: "lvmthin", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
||||
agentapi.DiskInfo{Name: "sdc", BackingDevice: "/dev/sdc", Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(31)}},
|
||||
)
|
||||
rows := h.s.diskHealthRows(context.Background())
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 physical-disk rows (pbs+lvm excluded), got %d: %+v", len(rows), rows)
|
||||
}
|
||||
@@ -133,26 +639,7 @@ func TestDiskHealthRows_NilSmart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A degraded verdict is FAILING → critical (Scenario B, Hiba→critical path).
|
||||
func TestDiskHealthCheck_FailingCritical(t *testing.T) {
|
||||
s := testServer(t)
|
||||
var crit []bool
|
||||
s.diskNotifyFn = func(label string, attrs []string, critical bool) { crit = append(crit, critical) }
|
||||
payload := &[]agentapi.DiskInfo{}
|
||||
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
||||
return agentapi.DisksResponse{Disks: *payload}, nil
|
||||
}
|
||||
ctx := context.Background()
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // baseline OK
|
||||
*payload = []agentapi.DiskInfo{physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartFailing})}
|
||||
_ = s.RunDiskHealthCheck(ctx) // OK→Fail
|
||||
if len(crit) != 1 || !crit[0] {
|
||||
t.Fatalf("OK→FAILING must emit one critical event, got %v", crit)
|
||||
}
|
||||
}
|
||||
|
||||
// TTL cache (Scenario A): two card fetches inside 60s hit the agent once.
|
||||
// TTL cache: two card fetches inside 60s hit the agent once.
|
||||
func TestCachedDisks_TTL(t *testing.T) {
|
||||
s := testServer(t)
|
||||
calls := 0
|
||||
|
||||
Reference in New Issue
Block a user