admin
c24f1920d9
One check cannot distinguish a loaded state from a silent re-baseline — a forgetful controller is also silent on its first check. It betrays itself on the second, when the rebuilt prior makes the disk look newly sustained and it alerts all over again. Caught while building the companion red-proof: with the state load skipped, the single-check version still passed.
686 lines
28 KiB
Go
686 lines
28 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// Disk-health severity ladder, event half (v0.215.0). Scenario letters refer to the task spec.
|
|
// The verdict half is pinned in internal/agentapi/diskverdict_ladder_test.go.
|
|
|
|
func smartPtr(v int) *int { return &v }
|
|
|
|
func physDisk(name string, sm *agentapi.SmartSummary) agentapi.DiskInfo {
|
|
return agentapi.DiskInfo{Name: name, BackingDevice: "/dev/" + name, DurableID: "uuid:" + name, Smart: sm}
|
|
}
|
|
|
|
// realDriveSmart is the failing drive as captured on 2026-08-14 — PASSED, 352 unreadable sectors.
|
|
// felhom.eu/documentation/audits/fixtures/smart-ST3000VX010-failing-2026-08-14.json
|
|
func realDriveSmart() *agentapi.SmartSummary {
|
|
return &agentapi.SmartSummary{
|
|
Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(352),
|
|
OfflineUncorrectable: smartPtr(352),
|
|
ReallocatedSectors: smartPtr(0),
|
|
TemperatureC: smartPtr(40),
|
|
}
|
|
}
|
|
|
|
// diskHarness wires a Server with the disks source + notify sink seams, a REAL data dir (so the
|
|
// persisted state is exercised, not bypassed) and a controllable clock.
|
|
type diskHarness struct {
|
|
s *Server
|
|
fired *[]notify.DiskAlert
|
|
payload *[]agentapi.DiskInfo
|
|
dir string
|
|
clock *time.Time
|
|
}
|
|
|
|
func newDiskHarness(t *testing.T) *diskHarness {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
lg := log.New(io.Discard, "", 0)
|
|
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
|
if err != nil {
|
|
t.Fatalf("settings: %v", err)
|
|
}
|
|
cfg := &config.Config{}
|
|
cfg.Paths.DataDir = dir
|
|
s := &Server{settings: sett, logger: lg, cfg: cfg}
|
|
|
|
now := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
|
h := &diskHarness{s: s, dir: dir, clock: &now, fired: &[]notify.DiskAlert{}, payload: &[]agentapi.DiskInfo{}}
|
|
s.diskHealth.clock = func() time.Time { return *h.clock }
|
|
s.diskNotifyFn = func(a notify.DiskAlert) { *h.fired = append(*h.fired, a) }
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{Disks: *h.payload}, nil
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (h *diskHarness) set(d ...agentapi.DiskInfo) { *h.payload = d }
|
|
func (h *diskHarness) run(t *testing.T) {
|
|
t.Helper()
|
|
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
|
t.Fatalf("RunDiskHealthCheck: %v", err)
|
|
}
|
|
}
|
|
func (h *diskHarness) advance(d time.Duration) { *h.clock = h.clock.Add(d) }
|
|
func (h *diskHarness) events() []notify.DiskAlert {
|
|
return *h.fired
|
|
}
|
|
func (h *diskHarness) reset() { *h.fired = nil }
|
|
|
|
// record reads the persisted state file straight off disk — asserting the FILE, not the in-memory
|
|
// map, because Scenario L depends on what actually landed there.
|
|
func (h *diskHarness) record(t *testing.T, key string) *diskRecord {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join(h.dir, diskStateFileName))
|
|
if err != nil {
|
|
t.Fatalf("reading state file: %v", err)
|
|
}
|
|
var f diskStateFile
|
|
if err := json.Unmarshal(data, &f); err != nil {
|
|
t.Fatalf("parsing state file: %v", err)
|
|
}
|
|
return f.Disks[key]
|
|
}
|
|
|
|
// ── Group A — Scenario A: the real drive, second observation ────────────────────────────────────
|
|
|
|
// The captured failing drive reaches Hiba, renders the crit chip, and emits exactly ONE event at
|
|
// severity "critical".
|
|
//
|
|
// Red-proof: remove truth-table row 6 (prior.SawUncorrectable) → 352 still trips row 8, so use
|
|
// TestDiskLadder_SustainDrivesTheEscalation (Group C) to isolate row 6.
|
|
func TestDiskCheck_RealDrive_HibaAndOneCriticalEvent(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", realDriveSmart()))
|
|
|
|
h.run(t) // first ever observation: baselines silently
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("first observation must be silent, got %d event(s)", n)
|
|
}
|
|
|
|
h.run(t) // second observation, now with prior
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("want exactly ONE event, got %d: %+v", len(ev), ev)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want %q", ev[0].Kind.Severity(), "critical")
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailSectors {
|
|
t.Errorf("kind = %d, want DiskAlertFailSectors (the drive says PASSED — we must not claim it self-reported)", ev[0].Kind)
|
|
}
|
|
if ev[0].Sectors != 352 {
|
|
t.Errorf("sectors = %d, want 352", ev[0].Sectors)
|
|
}
|
|
|
|
// The card must agree with the alert — same verdict function, same prior.
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 1 {
|
|
t.Fatalf("want 1 card row, got %d", len(rows))
|
|
}
|
|
if rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip label = %q, want %q", rows[0].ChipLabel, "Hiba")
|
|
}
|
|
if rows[0].ChipClass != "state-text-crit" {
|
|
t.Errorf("chip class = %q, want %q", rows[0].ChipClass, "state-text-crit")
|
|
}
|
|
}
|
|
|
|
// ── Group B — Scenario B: the transient that cleared (11 August) ────────────────────────────────
|
|
|
|
// A first-ever sighting of 8 sectors is Figyelmeztetés, not Hiba. The real drive did exactly this on
|
|
// 11 Aug 12:28 and had cleared completely by 13:28.
|
|
//
|
|
// Red-proof: make truth-table row 9 return Fail → the chip reads Hiba and the kind assertion fails.
|
|
func TestDiskCheck_FirstSightingIsWarnOnly(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
excursion := &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)}
|
|
h.set(physDisk("sdg", excursion))
|
|
h.run(t)
|
|
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 1 || rows[0].ChipLabel != "Figyelmeztetés" {
|
|
t.Fatalf("first sighting chip = %+v, want Figyelmeztetés", rows)
|
|
}
|
|
rec := h.record(t, "uuid:sdg")
|
|
if rec == nil {
|
|
t.Fatal("no persisted record for the disk")
|
|
}
|
|
if !rec.SawUncorrectable {
|
|
t.Error("persisted state must record SawUncorrectable=true so the NEXT check can sustain")
|
|
}
|
|
if rec.Sectors != 8 {
|
|
t.Errorf("persisted sectors = %d, want 8", rec.Sectors)
|
|
}
|
|
}
|
|
|
|
// ── Group C — Scenario C: the same disk one check later, still bad ──────────────────────────────
|
|
|
|
// The count does NOT grow — only the fact that it persisted. This isolates truth-table row 6: at 8
|
|
// sectors the count backstop (64) cannot be what fires.
|
|
//
|
|
// Red-proof: drop the prior argument's effect (always pass a zero DiskPrior in RunDiskHealthCheck)
|
|
// → the disk stays at Figyelmeztetés forever and no event is emitted.
|
|
func TestDiskLadder_SustainDrivesTheEscalation(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
excursion := func() agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
h.set(excursion())
|
|
h.run(t) // first sighting → silent baseline at Figyelmeztetés
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("first sighting must be silent, got %d", n)
|
|
}
|
|
|
|
h.set(excursion())
|
|
h.run(t) // SAME counters, now sustained → Hiba
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("sustained excursion must emit exactly ONE event, got %d: %+v", len(ev), ev)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
if ev[0].Sectors != 8 {
|
|
t.Errorf("sectors = %d, want 8 — the count did not grow; SUSTAIN is what fired", ev[0].Sectors)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
|
}
|
|
}
|
|
|
|
// ── Group D — Scenario D: recovered ─────────────────────────────────────────────────────────────
|
|
|
|
// Red-proof: make recovery emit → an event appears where zero are expected.
|
|
func TestDiskCheck_RecoveryIsSilentAndClearsState(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
h.reset()
|
|
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(0), OfflineUncorrectable: smartPtr(0), ReallocatedSectors: smartPtr(0)}))
|
|
h.run(t)
|
|
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("recovery must be silent, got %d event(s): %+v", n, h.events())
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Rendben" {
|
|
t.Errorf("chip = %q, want Rendben", rows[0].ChipLabel)
|
|
}
|
|
rec := h.record(t, "uuid:sdg")
|
|
if rec.SawUncorrectable {
|
|
t.Error("recovery must clear SawUncorrectable, else the disk re-escalates on the next blip")
|
|
}
|
|
}
|
|
|
|
// ── Group E — Scenario E: the flap must not spam ────────────────────────────────────────────────
|
|
|
|
// Warn (alerted) → OK → Warn again, all inside 24h: exactly ONE event across all three checks.
|
|
// Today's transition-only logic emits twice on that zero-crossing, which is how a customer learns to
|
|
// ignore the message.
|
|
//
|
|
// Red-proof: compare against the last OBSERVED verdict instead of the last ALERTED one → two events.
|
|
func TestDiskCheck_FlapDamping(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
warn := func() agentapi.DiskInfo {
|
|
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)})
|
|
}
|
|
clean := func() agentapi.DiskInfo {
|
|
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
// Baseline clean first so the Warn below is a real transition rather than a first sighting.
|
|
h.set(clean())
|
|
h.run(t)
|
|
|
|
h.set(warn())
|
|
h.run(t) // OK→Warn: the one legitimate alert
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("first degradation must emit once, got %d", n)
|
|
}
|
|
h.advance(2 * time.Hour)
|
|
h.set(clean())
|
|
h.run(t) // recovery: silent
|
|
h.advance(2 * time.Hour)
|
|
h.set(warn())
|
|
h.run(t) // re-degradation to the SAME level within 24h: damped
|
|
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("a flap inside 24h must produce exactly ONE event, got %d: %+v", n, h.events())
|
|
}
|
|
}
|
|
|
|
// ── Group F — Scenario F: escalation always beats damping ───────────────────────────────────────
|
|
|
|
// Red-proof: let damping cover escalations (return false whenever a verdict was ever alerted) →
|
|
// zero events, and a drive that has started actually failing says nothing.
|
|
func TestDiskCheck_EscalationBeatsDamping(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)}))
|
|
h.run(t)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)}))
|
|
h.run(t) // OK→Warn, alerted
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("setup: want 1 event, got %d", n)
|
|
}
|
|
h.reset()
|
|
|
|
// Two hours later — well inside the damping window — it reaches Hiba.
|
|
h.advance(2 * time.Hour)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartFailing, ReallocatedSectors: smartPtr(1)}))
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("an escalation inside the damping window MUST emit, got %d", len(ev))
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailSelfReported {
|
|
t.Errorf("kind = %d, want DiskAlertFailSelfReported", ev[0].Kind)
|
|
}
|
|
}
|
|
|
|
// ── Group G — Scenario G: still getting worse ───────────────────────────────────────────────────
|
|
|
|
// A disk already at Hiba, last alerted at 64 sectors, now at 352, 24h later: a SECOND event naming
|
|
// the current count. Today's code emits nothing at all between 8 and 352 sectors.
|
|
//
|
|
// Red-proof: remove the re-alert branch from diskAlertDecision → one event only.
|
|
func TestDiskCheck_RealertWhenStillWorsening(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
at := func(n int) agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
h.set(at(64))
|
|
h.run(t) // first sighting → silent (already Hiba via row 8, but baselined)
|
|
h.set(at(64))
|
|
h.run(t) // sustained → Hiba, alerted at 64
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("setup: want 1 event, got %d", n)
|
|
}
|
|
h.reset()
|
|
|
|
h.advance(25 * time.Hour)
|
|
h.set(at(352))
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("a doubling after the cooldown must re-alert, got %d event(s)", len(ev))
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailWorsened {
|
|
t.Errorf("kind = %d, want DiskAlertFailWorsened", ev[0].Kind)
|
|
}
|
|
if ev[0].Sectors != 352 {
|
|
t.Errorf("re-alert must name the CURRENT count, got %d", ev[0].Sectors)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
}
|
|
|
|
// ── Group H — Scenario H: growth below the bar, and inside the cooldown ─────────────────────────
|
|
|
|
// Red-proof: make the cooldown and the doubling bar an OR instead of an AND → an event appears.
|
|
func TestDiskCheck_NoRealertBelowBothBars(t *testing.T) {
|
|
at := func(n int) agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
// Neither bar cleared: 352 → 360 after 2h.
|
|
h := newDiskHarness(t)
|
|
h.set(at(352))
|
|
h.run(t)
|
|
h.set(at(352))
|
|
h.run(t) // alerted at 352
|
|
h.reset()
|
|
h.advance(2 * time.Hour)
|
|
h.set(at(360))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("neither bar cleared → no event, got %d: %+v", n, h.events())
|
|
}
|
|
|
|
// ONLY the cooldown cleared (25h) but growth is tiny → still silent. Pins the AND.
|
|
h.advance(25 * time.Hour)
|
|
h.set(at(400))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("cooldown alone must not re-alert (AND, not OR), got %d: %+v", n, h.events())
|
|
}
|
|
|
|
// ONLY the doubling cleared (2h later) → still silent. Pins the other half of the AND.
|
|
h2 := newDiskHarness(t)
|
|
h2.set(at(100))
|
|
h2.run(t)
|
|
h2.set(at(100))
|
|
h2.run(t) // alerted at 100
|
|
h2.reset()
|
|
h2.advance(2 * time.Hour)
|
|
h2.set(at(400))
|
|
h2.run(t)
|
|
if n := len(h2.events()); n != 0 {
|
|
t.Errorf("doubling alone inside the cooldown must not re-alert, got %d: %+v", n, h2.events())
|
|
}
|
|
}
|
|
|
|
// ── Group I — Scenario I: heat ──────────────────────────────────────────────────────────────────
|
|
|
|
// The event half of the temperature ladder: a hot disk alerts with the temperature SHAPE, so the
|
|
// customer is told to check ventilation rather than to arrange a replacement.
|
|
//
|
|
// Red-proof: remove truth-table rows 3 and 13 → the disk reads Rendben and nothing is emitted.
|
|
func TestDiskCheck_TemperatureShape(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
cool := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(40)})
|
|
hot := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(61)})
|
|
h.set(cool)
|
|
h.run(t)
|
|
h.set(hot)
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("overheating must emit once, got %d", len(ev))
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailTemperature {
|
|
t.Errorf("kind = %d, want DiskAlertFailTemperature", ev[0].Kind)
|
|
}
|
|
if ev[0].TemperatureC != 61 {
|
|
t.Errorf("temperature = %d, want 61", ev[0].TemperatureC)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
|
}
|
|
}
|
|
|
|
// ── Group J — Scenario J: no data never alarms, and never erases history ────────────────────────
|
|
|
|
// Red-proof: let UNKNOWN write or delete the record → the disk re-baselines and its history is lost,
|
|
// so the sustained escalation afterwards never fires.
|
|
func TestDiskCheck_UnknownNeverAlarmsNorErasesPrior(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
// Establish a real record with SawUncorrectable set.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
before := h.record(t, "uuid:sdg")
|
|
if !before.SawUncorrectable {
|
|
t.Fatal("setup: expected SawUncorrectable")
|
|
}
|
|
h.reset()
|
|
|
|
// A transient unreadable SMART, and separately a nil one.
|
|
for _, sm := range []*agentapi.SmartSummary{{Health: agentapi.SmartUnknown}, nil} {
|
|
h.set(physDisk("sdg", sm))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("no-data disk must never alarm, got %d", n)
|
|
}
|
|
after := h.record(t, "uuid:sdg")
|
|
if after == nil {
|
|
t.Fatal("UNKNOWN deleted the prior record — a transient blip must not erase history")
|
|
}
|
|
if after.SawUncorrectable != before.SawUncorrectable || after.Verdict != before.Verdict {
|
|
t.Errorf("UNKNOWN overwrote the prior: before=%+v after=%+v", before, after)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); len(rows) != 1 || rows[0].ChipLabel != "Nincs adat" {
|
|
t.Errorf("chip = %+v, want Nincs adat", rows)
|
|
}
|
|
}
|
|
|
|
// History survived, so the real sustained escalation still fires.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("the sustained escalation must still fire after a no-data blip, got %d", n)
|
|
}
|
|
}
|
|
|
|
// ── Group L — Scenario L: state survives a restart (SEAM TEST) ──────────────────────────────────
|
|
|
|
// Constructed through the PRODUCTION path — web.NewServer, exactly as cmd/controller/main.go builds
|
|
// it — over the same data dir, reading and writing a real file. An injected-seam test proves the
|
|
// component; it never proves the caller. This project has shipped seven built-but-never-wired
|
|
// defects, and the severity bug being fixed here is arguably the seventh.
|
|
//
|
|
// Red-proof: skip loadDiskStateLocked in RunDiskHealthCheck → the restarted controller silently
|
|
// re-baselines a disk it has already reported, and never alerts on it again. That is today's bug.
|
|
func TestDiskCheck_StateSurvivesRestart_ProductionPath(t *testing.T) {
|
|
dir := t.TempDir()
|
|
lg := log.New(io.Discard, "", 0)
|
|
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
|
if err != nil {
|
|
t.Fatalf("settings: %v", err)
|
|
}
|
|
cfg := &config.Config{}
|
|
cfg.Paths.DataDir = dir
|
|
|
|
failing := []agentapi.DiskInfo{physDisk("sdg", realDriveSmart())}
|
|
newProdServer := func() (*Server, *[]notify.DiskAlert) {
|
|
// The same call cmd/controller/main.go makes.
|
|
s := NewServer(cfg, nil, nil, nil, nil, sett, nil, nil, nil, lg, "test")
|
|
fired := &[]notify.DiskAlert{}
|
|
s.diskNotifyFn = func(a notify.DiskAlert) { *fired = append(*fired, a) }
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{Disks: failing}, nil
|
|
}
|
|
return s, fired
|
|
}
|
|
|
|
s1, fired1 := newProdServer()
|
|
_ = s1.RunDiskHealthCheck(context.Background()) // baseline
|
|
_ = s1.RunDiskHealthCheck(context.Background()) // sustained → Hiba, one alert
|
|
if len(*fired1) != 1 {
|
|
t.Fatalf("setup: want 1 alert before the restart, got %d", len(*fired1))
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, diskStateFileName)); err != nil {
|
|
t.Fatalf("the production path must have WRITTEN the state file: %v", err)
|
|
}
|
|
|
|
// Restart: a brand-new Server over the same data dir, disk still failing.
|
|
//
|
|
// TWO checks, deliberately. One is not enough to tell a loaded state from a silent re-baseline:
|
|
// a forgetful controller is also silent on its first check. It betrays itself on the SECOND,
|
|
// when the rebuilt prior makes the disk look newly sustained and it alerts all over again.
|
|
s2, fired2 := newProdServer()
|
|
_ = s2.RunDiskHealthCheck(context.Background())
|
|
_ = s2.RunDiskHealthCheck(context.Background())
|
|
if n := len(*fired2); n != 0 {
|
|
t.Errorf("a restarted controller must NOT re-alert an already-reported disk, got %d: %+v", n, *fired2)
|
|
}
|
|
|
|
// And it must still know the disk is bad — the chip cannot silently drop to Figyelmeztetés.
|
|
rows := s2.diskHealthRows(context.Background())
|
|
if len(rows) != 1 || rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("after restart the card = %+v, want Hiba", rows)
|
|
}
|
|
}
|
|
|
|
// A corrupt state file must LOG and fall back to "no prior" — never crash the check and never take
|
|
// the box down. The fail-safe direction is a duplicate alert, not a missing one.
|
|
func TestDiskState_CorruptFileFallsBackToNoPrior(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
if err := os.WriteFile(filepath.Join(h.dir, diskStateFileName), []byte("{not json"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.set(physDisk("sdg", realDriveSmart()))
|
|
h.run(t) // must not panic; treated as first-ever → silent
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("corrupt state → treated as no prior → silent baseline, got %d", n)
|
|
}
|
|
// And the next check works normally, proving the check was not left wedged.
|
|
h.run(t)
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("check must keep working after a corrupt state file, got %d events", n)
|
|
}
|
|
}
|
|
|
|
// A disk that disappears from the report is forgotten, so a reappearance re-baselines silently
|
|
// (behaviour preserved from v0.169.0 — persisting the state must not change it).
|
|
func TestDiskCheck_DisappearedDiskIsForgotten(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
h.set() // drive removed
|
|
h.run(t)
|
|
if r := h.record(t, "uuid:sdg"); r != nil {
|
|
t.Errorf("a disappeared disk must be dropped, got %+v", r)
|
|
}
|
|
// Reappears, still bad: first sighting again → silent, NOT an immediate Hiba.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("a reappearing disk must re-baseline silently, got %d", n)
|
|
}
|
|
}
|
|
|
|
// An unreachable agent changes nothing at all — no state churn, no alarm, no error.
|
|
func TestDiskCheck_UnreachableAgentIsInert(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
before := h.record(t, "uuid:sdg")
|
|
|
|
h.s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{}, context.DeadlineExceeded
|
|
}
|
|
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
|
t.Errorf("an unreachable agent must return nil, got %v", err)
|
|
}
|
|
after := h.record(t, "uuid:sdg")
|
|
if after == nil || after.SawUncorrectable != before.SawUncorrectable {
|
|
t.Errorf("an unreachable agent must not churn state: before=%+v after=%+v", before, after)
|
|
}
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("an unreachable agent must not alarm, got %d", n)
|
|
}
|
|
}
|
|
|
|
// diskAlertDecision is pure — pin its table directly, including the boundaries the scenarios above
|
|
// only reach indirectly.
|
|
func TestDiskAlertDecision_Table(t *testing.T) {
|
|
base := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
|
alertedFail := func(sectors int, ago time.Duration) *diskRecord {
|
|
return &diskRecord{Verdict: int(agentapi.DiskVerdictFail), Alerted: true,
|
|
AlertedVerdict: int(agentapi.DiskVerdictFail), AlertedSectors: sectors, AlertedAt: base.Add(-ago)}
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
prev *diskRecord
|
|
v agentapi.DiskVerdict
|
|
sectors int
|
|
wantEmit bool
|
|
wantWorsened bool
|
|
}{
|
|
{"first ever, Warn", nil, agentapi.DiskVerdictWarn, 8, false, false},
|
|
{"first ever, Fail", nil, agentapi.DiskVerdictFail, 352, false, false},
|
|
{"OK is always silent", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn)}, agentapi.DiskVerdictOK, 0, false, false},
|
|
{"never alerted, now Warn", &diskRecord{Verdict: int(agentapi.DiskVerdictOK)}, agentapi.DiskVerdictWarn, 8, true, false},
|
|
{"escalate Warn→Fail", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn), Alerted: true,
|
|
AlertedVerdict: int(agentapi.DiskVerdictWarn), AlertedAt: base}, agentapi.DiskVerdictFail, 8, true, false},
|
|
{"steady Fail, no growth, no time", alertedFail(352, time.Hour), agentapi.DiskVerdictFail, 352, false, false},
|
|
{"exactly 2x but only 23h", alertedFail(64, 23 * time.Hour), agentapi.DiskVerdictFail, 128, false, false},
|
|
{"exactly 2x at exactly 24h", alertedFail(64, 24 * time.Hour), agentapi.DiskVerdictFail, 128, true, true},
|
|
{"25h but only 1.9x", alertedFail(64, 25 * time.Hour), agentapi.DiskVerdictFail, 121, false, false},
|
|
{"improved Fail→Warn is silent", alertedFail(352, 48 * time.Hour), agentapi.DiskVerdictWarn, 8, false, false},
|
|
{"Fail from heat (0 sectors) never re-alerts on the doubling rule",
|
|
alertedFail(0, 48 * time.Hour), agentapi.DiskVerdictFail, 0, false, false},
|
|
}
|
|
for _, c := range cases {
|
|
emit, worsened := diskAlertDecision(c.prev, c.v, c.sectors, base)
|
|
if emit != c.wantEmit || worsened != c.wantWorsened {
|
|
t.Errorf("%s: got (emit=%v worsened=%v), want (emit=%v worsened=%v)",
|
|
c.name, emit, worsened, c.wantEmit, c.wantWorsened)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Preserved v0.169.0/v0.171.0 coverage ────────────────────────────────────────────────────────
|
|
|
|
// The card renders gracefully with nil SMART (old agent / no data): the row shows "Nincs adat" and
|
|
// never errors; a physical disk with no smart field is still listed.
|
|
func TestDiskHealthRows_NilSmart(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(
|
|
agentapi.DiskInfo{Name: "sdb", BackingDevice: "/dev/sdb", Smart: nil}, // physical, no smart → Nincs adat
|
|
// PBS/LVM carry a default UNKNOWN SMART from the agent — must STILL be excluded (not disks).
|
|
agentapi.DiskInfo{Name: "felhom-pbs", Type: "pbs", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
|
agentapi.DiskInfo{Name: "local-lvm", Type: "lvmthin", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
|
agentapi.DiskInfo{Name: "sdc", BackingDevice: "/dev/sdc", Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(31)}},
|
|
)
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 2 {
|
|
t.Fatalf("want 2 physical-disk rows (pbs+lvm excluded), got %d: %+v", len(rows), rows)
|
|
}
|
|
byLabel := map[string]DiskHealthRow{}
|
|
for _, r := range rows {
|
|
byLabel[r.Label] = r
|
|
}
|
|
if byLabel["sdb"].ChipLabel != "Nincs adat" {
|
|
t.Errorf("nil-smart disk chip = %q, want Nincs adat", byLabel["sdb"].ChipLabel)
|
|
}
|
|
if byLabel["sdc"].ChipLabel != "Rendben" || byLabel["sdc"].Temp != "31" {
|
|
t.Errorf("sdc row = %+v, want Rendben / 31", byLabel["sdc"])
|
|
}
|
|
}
|
|
|
|
// TTL cache: two card fetches inside 60s hit the agent once.
|
|
func TestCachedDisks_TTL(t *testing.T) {
|
|
s := testServer(t)
|
|
calls := 0
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
calls++
|
|
return agentapi.DisksResponse{}, nil
|
|
}
|
|
ctx := context.Background()
|
|
_, _ = s.cachedDisks(ctx)
|
|
_, _ = s.cachedDisks(ctx)
|
|
if calls != 1 {
|
|
t.Errorf("two fetches within the TTL should call the agent once, got %d", calls)
|
|
}
|
|
}
|
|
|
|
func sptr(s string) *string { return &s }
|
|
|
|
// v0.171.0: the card label prefers the device model (agent v0.95.0) over the raw name/UUID; it falls
|
|
// back to Name (+ speed hint) on an older agent or a modelless disk.
|
|
// Red-proof: drop the fallback (always return ModelName) → the nil-model/old-agent cases return "" and
|
|
// TestDiskDisplayLabel_PrefersModel fails (A4 — old-payload tolerance).
|
|
func TestDiskDisplayLabel_PrefersModel(t *testing.T) {
|
|
withModel := agentapi.DiskInfo{Name: "47a3361a-uuid", Class: "slow",
|
|
Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, ModelName: sptr("TOSHIBA MQ04ABF100")}}
|
|
if got := diskDisplayLabel(withModel); got != "TOSHIBA MQ04ABF100" {
|
|
t.Errorf("label = %q, want the model name", got)
|
|
}
|
|
// A4: an old (v0.94.0) agent carries no model → Name (+ speed hint), byte-identical to before.
|
|
oldAgent := agentapi.DiskInfo{Name: "local", Class: "fast",
|
|
Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}}
|
|
if got := diskDisplayLabel(oldAgent); got != "local (gyors)" {
|
|
t.Errorf("old-agent label = %q, want 'local (gyors)'", got)
|
|
}
|
|
// No SMART at all → bare Name.
|
|
if got := diskDisplayLabel(agentapi.DiskInfo{Name: "sdb"}); got != "sdb" {
|
|
t.Errorf("nil-smart label = %q, want 'sdb'", got)
|
|
}
|
|
}
|