admin
90f2545679
gates / gates (push) Successful in 9s
Found on live hardware two hours after the v0.215.0 deploy, by noticing the release's own positive observable disagreed with its own persisted artefact: the check logged '3 disk(s) evaluated' while disk-health-state.json held two records. demo-hp's c11-scratch and felhom-backup are the same NVMe and share a durable id, so one disk was walked twice per run. Not cosmetic. The loop writes a disk's record before the next entry reads it, so the second copy of an aliased disk consumed the FIRST copy's write as its prior: the disk sustained against ITSELF and reached Hiba on a first sighting, defeating truth-table row 6 — the rule that separates a one-hour benign excursion from a false critical. It would also have emitted two identical events for one drive. Latent on demo-hp only because all counters are zero. Each diskKey is now evaluated once per run. Both entries stay marked seen so neither looks like a disappeared disk, and the card still renders both rows — the dedup is about state and alerts, not display. Red-proof run and reverted: deleting the guard makes the first sighting emit Kind:2 (Hiba-from-sectors) at 8 sectors.
742 lines
31 KiB
Go
742 lines
31 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// Disk-health severity ladder, event half (v0.215.0). Scenario letters refer to the task spec.
|
|
// The verdict half is pinned in internal/agentapi/diskverdict_ladder_test.go.
|
|
|
|
func smartPtr(v int) *int { return &v }
|
|
|
|
func physDisk(name string, sm *agentapi.SmartSummary) agentapi.DiskInfo {
|
|
return agentapi.DiskInfo{Name: name, BackingDevice: "/dev/" + name, DurableID: "uuid:" + name, Smart: sm}
|
|
}
|
|
|
|
// realDriveSmart is the failing drive as captured on 2026-08-14 — PASSED, 352 unreadable sectors.
|
|
// felhom.eu/documentation/audits/fixtures/smart-ST3000VX010-failing-2026-08-14.json
|
|
func realDriveSmart() *agentapi.SmartSummary {
|
|
return &agentapi.SmartSummary{
|
|
Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(352),
|
|
OfflineUncorrectable: smartPtr(352),
|
|
ReallocatedSectors: smartPtr(0),
|
|
TemperatureC: smartPtr(40),
|
|
}
|
|
}
|
|
|
|
// diskHarness wires a Server with the disks source + notify sink seams, a REAL data dir (so the
|
|
// persisted state is exercised, not bypassed) and a controllable clock.
|
|
type diskHarness struct {
|
|
s *Server
|
|
fired *[]notify.DiskAlert
|
|
payload *[]agentapi.DiskInfo
|
|
dir string
|
|
clock *time.Time
|
|
}
|
|
|
|
func newDiskHarness(t *testing.T) *diskHarness {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
lg := log.New(io.Discard, "", 0)
|
|
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
|
if err != nil {
|
|
t.Fatalf("settings: %v", err)
|
|
}
|
|
cfg := &config.Config{}
|
|
cfg.Paths.DataDir = dir
|
|
s := &Server{settings: sett, logger: lg, cfg: cfg}
|
|
|
|
now := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
|
h := &diskHarness{s: s, dir: dir, clock: &now, fired: &[]notify.DiskAlert{}, payload: &[]agentapi.DiskInfo{}}
|
|
s.diskHealth.clock = func() time.Time { return *h.clock }
|
|
s.diskNotifyFn = func(a notify.DiskAlert) { *h.fired = append(*h.fired, a) }
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{Disks: *h.payload}, nil
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (h *diskHarness) set(d ...agentapi.DiskInfo) { *h.payload = d }
|
|
func (h *diskHarness) run(t *testing.T) {
|
|
t.Helper()
|
|
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
|
t.Fatalf("RunDiskHealthCheck: %v", err)
|
|
}
|
|
}
|
|
func (h *diskHarness) advance(d time.Duration) { *h.clock = h.clock.Add(d) }
|
|
func (h *diskHarness) events() []notify.DiskAlert {
|
|
return *h.fired
|
|
}
|
|
func (h *diskHarness) reset() { *h.fired = nil }
|
|
|
|
// record reads the persisted state file straight off disk — asserting the FILE, not the in-memory
|
|
// map, because Scenario L depends on what actually landed there.
|
|
func (h *diskHarness) record(t *testing.T, key string) *diskRecord {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join(h.dir, diskStateFileName))
|
|
if err != nil {
|
|
t.Fatalf("reading state file: %v", err)
|
|
}
|
|
var f diskStateFile
|
|
if err := json.Unmarshal(data, &f); err != nil {
|
|
t.Fatalf("parsing state file: %v", err)
|
|
}
|
|
return f.Disks[key]
|
|
}
|
|
|
|
// ── Group A — Scenario A: the real drive, second observation ────────────────────────────────────
|
|
|
|
// The captured failing drive reaches Hiba, renders the crit chip, and emits exactly ONE event at
|
|
// severity "critical".
|
|
//
|
|
// Red-proof: remove truth-table row 6 (prior.SawUncorrectable) → 352 still trips row 8, so use
|
|
// TestDiskLadder_SustainDrivesTheEscalation (Group C) to isolate row 6.
|
|
func TestDiskCheck_RealDrive_HibaAndOneCriticalEvent(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", realDriveSmart()))
|
|
|
|
h.run(t) // first ever observation: baselines silently
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("first observation must be silent, got %d event(s)", n)
|
|
}
|
|
|
|
h.run(t) // second observation, now with prior
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("want exactly ONE event, got %d: %+v", len(ev), ev)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want %q", ev[0].Kind.Severity(), "critical")
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailSectors {
|
|
t.Errorf("kind = %d, want DiskAlertFailSectors (the drive says PASSED — we must not claim it self-reported)", ev[0].Kind)
|
|
}
|
|
if ev[0].Sectors != 352 {
|
|
t.Errorf("sectors = %d, want 352", ev[0].Sectors)
|
|
}
|
|
|
|
// The card must agree with the alert — same verdict function, same prior.
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 1 {
|
|
t.Fatalf("want 1 card row, got %d", len(rows))
|
|
}
|
|
if rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip label = %q, want %q", rows[0].ChipLabel, "Hiba")
|
|
}
|
|
if rows[0].ChipClass != "state-text-crit" {
|
|
t.Errorf("chip class = %q, want %q", rows[0].ChipClass, "state-text-crit")
|
|
}
|
|
}
|
|
|
|
// ── Group B — Scenario B: the transient that cleared (11 August) ────────────────────────────────
|
|
|
|
// A first-ever sighting of 8 sectors is Figyelmeztetés, not Hiba. The real drive did exactly this on
|
|
// 11 Aug 12:28 and had cleared completely by 13:28.
|
|
//
|
|
// Red-proof: make truth-table row 9 return Fail → the chip reads Hiba and the kind assertion fails.
|
|
func TestDiskCheck_FirstSightingIsWarnOnly(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
excursion := &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)}
|
|
h.set(physDisk("sdg", excursion))
|
|
h.run(t)
|
|
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 1 || rows[0].ChipLabel != "Figyelmeztetés" {
|
|
t.Fatalf("first sighting chip = %+v, want Figyelmeztetés", rows)
|
|
}
|
|
rec := h.record(t, "uuid:sdg")
|
|
if rec == nil {
|
|
t.Fatal("no persisted record for the disk")
|
|
}
|
|
if !rec.SawUncorrectable {
|
|
t.Error("persisted state must record SawUncorrectable=true so the NEXT check can sustain")
|
|
}
|
|
if rec.Sectors != 8 {
|
|
t.Errorf("persisted sectors = %d, want 8", rec.Sectors)
|
|
}
|
|
}
|
|
|
|
// ── Group C — Scenario C: the same disk one check later, still bad ──────────────────────────────
|
|
|
|
// The count does NOT grow — only the fact that it persisted. This isolates truth-table row 6: at 8
|
|
// sectors the count backstop (64) cannot be what fires.
|
|
//
|
|
// Red-proof: drop the prior argument's effect (always pass a zero DiskPrior in RunDiskHealthCheck)
|
|
// → the disk stays at Figyelmeztetés forever and no event is emitted.
|
|
func TestDiskLadder_SustainDrivesTheEscalation(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
excursion := func() agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
h.set(excursion())
|
|
h.run(t) // first sighting → silent baseline at Figyelmeztetés
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("first sighting must be silent, got %d", n)
|
|
}
|
|
|
|
h.set(excursion())
|
|
h.run(t) // SAME counters, now sustained → Hiba
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("sustained excursion must emit exactly ONE event, got %d: %+v", len(ev), ev)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
if ev[0].Sectors != 8 {
|
|
t.Errorf("sectors = %d, want 8 — the count did not grow; SUSTAIN is what fired", ev[0].Sectors)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
|
}
|
|
}
|
|
|
|
// ── Group D — Scenario D: recovered ─────────────────────────────────────────────────────────────
|
|
|
|
// Red-proof: make recovery emit → an event appears where zero are expected.
|
|
func TestDiskCheck_RecoveryIsSilentAndClearsState(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
h.reset()
|
|
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(0), OfflineUncorrectable: smartPtr(0), ReallocatedSectors: smartPtr(0)}))
|
|
h.run(t)
|
|
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("recovery must be silent, got %d event(s): %+v", n, h.events())
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Rendben" {
|
|
t.Errorf("chip = %q, want Rendben", rows[0].ChipLabel)
|
|
}
|
|
rec := h.record(t, "uuid:sdg")
|
|
if rec.SawUncorrectable {
|
|
t.Error("recovery must clear SawUncorrectable, else the disk re-escalates on the next blip")
|
|
}
|
|
}
|
|
|
|
// ── Group E — Scenario E: the flap must not spam ────────────────────────────────────────────────
|
|
|
|
// Warn (alerted) → OK → Warn again, all inside 24h: exactly ONE event across all three checks.
|
|
// Today's transition-only logic emits twice on that zero-crossing, which is how a customer learns to
|
|
// ignore the message.
|
|
//
|
|
// Red-proof: compare against the last OBSERVED verdict instead of the last ALERTED one → two events.
|
|
func TestDiskCheck_FlapDamping(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
warn := func() agentapi.DiskInfo {
|
|
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)})
|
|
}
|
|
clean := func() agentapi.DiskInfo {
|
|
return physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
// Baseline clean first so the Warn below is a real transition rather than a first sighting.
|
|
h.set(clean())
|
|
h.run(t)
|
|
|
|
h.set(warn())
|
|
h.run(t) // OK→Warn: the one legitimate alert
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("first degradation must emit once, got %d", n)
|
|
}
|
|
h.advance(2 * time.Hour)
|
|
h.set(clean())
|
|
h.run(t) // recovery: silent
|
|
h.advance(2 * time.Hour)
|
|
h.set(warn())
|
|
h.run(t) // re-degradation to the SAME level within 24h: damped
|
|
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("a flap inside 24h must produce exactly ONE event, got %d: %+v", n, h.events())
|
|
}
|
|
}
|
|
|
|
// ── Group F — Scenario F: escalation always beats damping ───────────────────────────────────────
|
|
|
|
// Red-proof: let damping cover escalations (return false whenever a verdict was ever alerted) →
|
|
// zero events, and a drive that has started actually failing says nothing.
|
|
func TestDiskCheck_EscalationBeatsDamping(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(0)}))
|
|
h.run(t)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartPassed, ReallocatedSectors: smartPtr(1)}))
|
|
h.run(t) // OK→Warn, alerted
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("setup: want 1 event, got %d", n)
|
|
}
|
|
h.reset()
|
|
|
|
// Two hours later — well inside the damping window — it reaches Hiba.
|
|
h.advance(2 * time.Hour)
|
|
h.set(physDisk("sdb", &agentapi.SmartSummary{Health: agentapi.SmartFailing, ReallocatedSectors: smartPtr(1)}))
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("an escalation inside the damping window MUST emit, got %d", len(ev))
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailSelfReported {
|
|
t.Errorf("kind = %d, want DiskAlertFailSelfReported", ev[0].Kind)
|
|
}
|
|
}
|
|
|
|
// ── Group G — Scenario G: still getting worse ───────────────────────────────────────────────────
|
|
|
|
// A disk already at Hiba, last alerted at 64 sectors, now at 352, 24h later: a SECOND event naming
|
|
// the current count. Today's code emits nothing at all between 8 and 352 sectors.
|
|
//
|
|
// Red-proof: remove the re-alert branch from diskAlertDecision → one event only.
|
|
func TestDiskCheck_RealertWhenStillWorsening(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
at := func(n int) agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
h.set(at(64))
|
|
h.run(t) // first sighting → silent (already Hiba via row 8, but baselined)
|
|
h.set(at(64))
|
|
h.run(t) // sustained → Hiba, alerted at 64
|
|
if n := len(h.events()); n != 1 {
|
|
t.Fatalf("setup: want 1 event, got %d", n)
|
|
}
|
|
h.reset()
|
|
|
|
h.advance(25 * time.Hour)
|
|
h.set(at(352))
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("a doubling after the cooldown must re-alert, got %d event(s)", len(ev))
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailWorsened {
|
|
t.Errorf("kind = %d, want DiskAlertFailWorsened", ev[0].Kind)
|
|
}
|
|
if ev[0].Sectors != 352 {
|
|
t.Errorf("re-alert must name the CURRENT count, got %d", ev[0].Sectors)
|
|
}
|
|
if ev[0].Kind.Severity() != "critical" {
|
|
t.Errorf("severity = %q, want critical", ev[0].Kind.Severity())
|
|
}
|
|
}
|
|
|
|
// ── Group H — Scenario H: growth below the bar, and inside the cooldown ─────────────────────────
|
|
|
|
// Red-proof: make the cooldown and the doubling bar an OR instead of an AND → an event appears.
|
|
func TestDiskCheck_NoRealertBelowBothBars(t *testing.T) {
|
|
at := func(n int) agentapi.DiskInfo {
|
|
return physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(n), OfflineUncorrectable: smartPtr(n), ReallocatedSectors: smartPtr(0)})
|
|
}
|
|
// Neither bar cleared: 352 → 360 after 2h.
|
|
h := newDiskHarness(t)
|
|
h.set(at(352))
|
|
h.run(t)
|
|
h.set(at(352))
|
|
h.run(t) // alerted at 352
|
|
h.reset()
|
|
h.advance(2 * time.Hour)
|
|
h.set(at(360))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("neither bar cleared → no event, got %d: %+v", n, h.events())
|
|
}
|
|
|
|
// ONLY the cooldown cleared (25h) but growth is tiny → still silent. Pins the AND.
|
|
h.advance(25 * time.Hour)
|
|
h.set(at(400))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("cooldown alone must not re-alert (AND, not OR), got %d: %+v", n, h.events())
|
|
}
|
|
|
|
// ONLY the doubling cleared (2h later) → still silent. Pins the other half of the AND.
|
|
h2 := newDiskHarness(t)
|
|
h2.set(at(100))
|
|
h2.run(t)
|
|
h2.set(at(100))
|
|
h2.run(t) // alerted at 100
|
|
h2.reset()
|
|
h2.advance(2 * time.Hour)
|
|
h2.set(at(400))
|
|
h2.run(t)
|
|
if n := len(h2.events()); n != 0 {
|
|
t.Errorf("doubling alone inside the cooldown must not re-alert, got %d: %+v", n, h2.events())
|
|
}
|
|
}
|
|
|
|
// ── Group I — Scenario I: heat ──────────────────────────────────────────────────────────────────
|
|
|
|
// The event half of the temperature ladder: a hot disk alerts with the temperature SHAPE, so the
|
|
// customer is told to check ventilation rather than to arrange a replacement.
|
|
//
|
|
// Red-proof: remove truth-table rows 3 and 13 → the disk reads Rendben and nothing is emitted.
|
|
func TestDiskCheck_TemperatureShape(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
cool := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(40)})
|
|
hot := physDisk("sdd", &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(61)})
|
|
h.set(cool)
|
|
h.run(t)
|
|
h.set(hot)
|
|
h.run(t)
|
|
|
|
ev := h.events()
|
|
if len(ev) != 1 {
|
|
t.Fatalf("overheating must emit once, got %d", len(ev))
|
|
}
|
|
if ev[0].Kind != notify.DiskAlertFailTemperature {
|
|
t.Errorf("kind = %d, want DiskAlertFailTemperature", ev[0].Kind)
|
|
}
|
|
if ev[0].TemperatureC != 61 {
|
|
t.Errorf("temperature = %d, want 61", ev[0].TemperatureC)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("chip = %q, want Hiba", rows[0].ChipLabel)
|
|
}
|
|
}
|
|
|
|
// ── Group J — Scenario J: no data never alarms, and never erases history ────────────────────────
|
|
|
|
// Red-proof: let UNKNOWN write or delete the record → the disk re-baselines and its history is lost,
|
|
// so the sustained escalation afterwards never fires.
|
|
func TestDiskCheck_UnknownNeverAlarmsNorErasesPrior(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
// Establish a real record with SawUncorrectable set.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
before := h.record(t, "uuid:sdg")
|
|
if !before.SawUncorrectable {
|
|
t.Fatal("setup: expected SawUncorrectable")
|
|
}
|
|
h.reset()
|
|
|
|
// A transient unreadable SMART, and separately a nil one.
|
|
for _, sm := range []*agentapi.SmartSummary{{Health: agentapi.SmartUnknown}, nil} {
|
|
h.set(physDisk("sdg", sm))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("no-data disk must never alarm, got %d", n)
|
|
}
|
|
after := h.record(t, "uuid:sdg")
|
|
if after == nil {
|
|
t.Fatal("UNKNOWN deleted the prior record — a transient blip must not erase history")
|
|
}
|
|
if after.SawUncorrectable != before.SawUncorrectable || after.Verdict != before.Verdict {
|
|
t.Errorf("UNKNOWN overwrote the prior: before=%+v after=%+v", before, after)
|
|
}
|
|
if rows := h.s.diskHealthRows(context.Background()); len(rows) != 1 || rows[0].ChipLabel != "Nincs adat" {
|
|
t.Errorf("chip = %+v, want Nincs adat", rows)
|
|
}
|
|
}
|
|
|
|
// History survived, so the real sustained escalation still fires.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("the sustained escalation must still fire after a no-data blip, got %d", n)
|
|
}
|
|
}
|
|
|
|
// ── Group L — Scenario L: state survives a restart (SEAM TEST) ──────────────────────────────────
|
|
|
|
// Constructed through the PRODUCTION path — web.NewServer, exactly as cmd/controller/main.go builds
|
|
// it — over the same data dir, reading and writing a real file. An injected-seam test proves the
|
|
// component; it never proves the caller. This project has shipped seven built-but-never-wired
|
|
// defects, and the severity bug being fixed here is arguably the seventh.
|
|
//
|
|
// Red-proof: skip loadDiskStateLocked in RunDiskHealthCheck → the restarted controller silently
|
|
// re-baselines a disk it has already reported, and never alerts on it again. That is today's bug.
|
|
func TestDiskCheck_StateSurvivesRestart_ProductionPath(t *testing.T) {
|
|
dir := t.TempDir()
|
|
lg := log.New(io.Discard, "", 0)
|
|
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
|
|
if err != nil {
|
|
t.Fatalf("settings: %v", err)
|
|
}
|
|
cfg := &config.Config{}
|
|
cfg.Paths.DataDir = dir
|
|
|
|
failing := []agentapi.DiskInfo{physDisk("sdg", realDriveSmart())}
|
|
newProdServer := func() (*Server, *[]notify.DiskAlert) {
|
|
// The same call cmd/controller/main.go makes.
|
|
s := NewServer(cfg, nil, nil, nil, nil, sett, nil, nil, nil, lg, "test")
|
|
fired := &[]notify.DiskAlert{}
|
|
s.diskNotifyFn = func(a notify.DiskAlert) { *fired = append(*fired, a) }
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{Disks: failing}, nil
|
|
}
|
|
return s, fired
|
|
}
|
|
|
|
s1, fired1 := newProdServer()
|
|
_ = s1.RunDiskHealthCheck(context.Background()) // baseline
|
|
_ = s1.RunDiskHealthCheck(context.Background()) // sustained → Hiba, one alert
|
|
if len(*fired1) != 1 {
|
|
t.Fatalf("setup: want 1 alert before the restart, got %d", len(*fired1))
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, diskStateFileName)); err != nil {
|
|
t.Fatalf("the production path must have WRITTEN the state file: %v", err)
|
|
}
|
|
|
|
// Restart: a brand-new Server over the same data dir, disk still failing.
|
|
//
|
|
// TWO checks, deliberately. One is not enough to tell a loaded state from a silent re-baseline:
|
|
// a forgetful controller is also silent on its first check. It betrays itself on the SECOND,
|
|
// when the rebuilt prior makes the disk look newly sustained and it alerts all over again.
|
|
s2, fired2 := newProdServer()
|
|
_ = s2.RunDiskHealthCheck(context.Background())
|
|
_ = s2.RunDiskHealthCheck(context.Background())
|
|
if n := len(*fired2); n != 0 {
|
|
t.Errorf("a restarted controller must NOT re-alert an already-reported disk, got %d: %+v", n, *fired2)
|
|
}
|
|
|
|
// And it must still know the disk is bad — the chip cannot silently drop to Figyelmeztetés.
|
|
rows := s2.diskHealthRows(context.Background())
|
|
if len(rows) != 1 || rows[0].ChipLabel != "Hiba" {
|
|
t.Errorf("after restart the card = %+v, want Hiba", rows)
|
|
}
|
|
}
|
|
|
|
// A corrupt state file must LOG and fall back to "no prior" — never crash the check and never take
|
|
// the box down. The fail-safe direction is a duplicate alert, not a missing one.
|
|
func TestDiskState_CorruptFileFallsBackToNoPrior(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
if err := os.WriteFile(filepath.Join(h.dir, diskStateFileName), []byte("{not json"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.set(physDisk("sdg", realDriveSmart()))
|
|
h.run(t) // must not panic; treated as first-ever → silent
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("corrupt state → treated as no prior → silent baseline, got %d", n)
|
|
}
|
|
// And the next check works normally, proving the check was not left wedged.
|
|
h.run(t)
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("check must keep working after a corrupt state file, got %d events", n)
|
|
}
|
|
}
|
|
|
|
// A disk that disappears from the report is forgotten, so a reappearance re-baselines silently
|
|
// (behaviour preserved from v0.169.0 — persisting the state must not change it).
|
|
func TestDiskCheck_DisappearedDiskIsForgotten(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
h.set() // drive removed
|
|
h.run(t)
|
|
if r := h.record(t, "uuid:sdg"); r != nil {
|
|
t.Errorf("a disappeared disk must be dropped, got %+v", r)
|
|
}
|
|
// Reappears, still bad: first sighting again → silent, NOT an immediate Hiba.
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("a reappearing disk must re-baseline silently, got %d", n)
|
|
}
|
|
}
|
|
|
|
// ── One physical disk, several storage entries ──────────────────────────────────────────────────
|
|
|
|
// FOUND ON LIVE HARDWARE (demo-hp, 2026-08-14): the check reported "3 disk(s) evaluated" while the
|
|
// persisted state held only TWO records, because `c11-scratch` and `felhom-backup` are the same NVMe
|
|
// and resolve to the same durable id.
|
|
//
|
|
// That aliasing was a real defect, not a cosmetic one. The loop writes this run's record before the
|
|
// next entry reads it, so the second copy of a disk consumed the FIRST copy's write as its prior —
|
|
// the disk sustained against ITSELF and reached Hiba on a first sighting, defeating truth-table row 6
|
|
// entirely, and emitted two identical events while doing it.
|
|
//
|
|
// Red-proof: delete the `if seen[key] { continue }` guard in RunDiskHealthCheck → the first run
|
|
// reaches Fail and emits, and both assertions below fail.
|
|
func TestDiskCheck_SameDiskTwiceIsEvaluatedOnce(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
// Two storage entries, ONE physical device — identical durable id, as the agent really reports it.
|
|
twice := func() []agentapi.DiskInfo {
|
|
sm := &agentapi.SmartSummary{Health: agentapi.SmartPassed,
|
|
PendingSectors: smartPtr(8), OfflineUncorrectable: smartPtr(8), ReallocatedSectors: smartPtr(0)}
|
|
return []agentapi.DiskInfo{
|
|
{Name: "c11-scratch", Type: "local-dir", BackingDevice: "/dev/nvme0n1", DurableID: "uuid:nvme-1", Smart: sm},
|
|
{Name: "felhom-backup", Type: "local-dir", BackingDevice: "/dev/nvme0n1", DurableID: "uuid:nvme-1", Smart: sm},
|
|
}
|
|
}
|
|
h.set(twice()...)
|
|
h.run(t)
|
|
|
|
// FIRST sighting: silent, and Figyelmeztetés — a disk must not sustain against its own alias.
|
|
if n := len(h.events()); n != 0 {
|
|
t.Fatalf("first sighting of an aliased disk must be silent, got %d: %+v", n, h.events())
|
|
}
|
|
rec := h.record(t, "uuid:nvme-1")
|
|
if rec == nil {
|
|
t.Fatal("no record for the aliased disk")
|
|
}
|
|
if agentapi.DiskVerdict(rec.Verdict) != agentapi.DiskVerdictWarn {
|
|
t.Errorf("first sighting verdict = %d, want Warn — the disk sustained against its own alias",
|
|
rec.Verdict)
|
|
}
|
|
if rec.PriorSawUncorrectable {
|
|
t.Error("the prior used on a first sighting must be empty, not this run's own write")
|
|
}
|
|
|
|
// SECOND run: now genuinely sustained → exactly ONE event, not one per storage entry.
|
|
h.set(twice()...)
|
|
h.run(t)
|
|
if n := len(h.events()); n != 1 {
|
|
t.Errorf("one physical disk must produce ONE event, got %d: %+v", n, h.events())
|
|
}
|
|
|
|
// Both entries still render on the card — dedup is about state and alerts, not display.
|
|
if rows := h.s.diskHealthRows(context.Background()); len(rows) != 2 {
|
|
t.Errorf("card must still show both storage entries, got %d rows", len(rows))
|
|
}
|
|
}
|
|
|
|
// An unreachable agent changes nothing at all — no state churn, no alarm, no error.
|
|
func TestDiskCheck_UnreachableAgentIsInert(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(physDisk("sdg", &agentapi.SmartSummary{Health: agentapi.SmartPassed, PendingSectors: smartPtr(8)}))
|
|
h.run(t)
|
|
before := h.record(t, "uuid:sdg")
|
|
|
|
h.s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
return agentapi.DisksResponse{}, context.DeadlineExceeded
|
|
}
|
|
if err := h.s.RunDiskHealthCheck(context.Background()); err != nil {
|
|
t.Errorf("an unreachable agent must return nil, got %v", err)
|
|
}
|
|
after := h.record(t, "uuid:sdg")
|
|
if after == nil || after.SawUncorrectable != before.SawUncorrectable {
|
|
t.Errorf("an unreachable agent must not churn state: before=%+v after=%+v", before, after)
|
|
}
|
|
if n := len(h.events()); n != 0 {
|
|
t.Errorf("an unreachable agent must not alarm, got %d", n)
|
|
}
|
|
}
|
|
|
|
// diskAlertDecision is pure — pin its table directly, including the boundaries the scenarios above
|
|
// only reach indirectly.
|
|
func TestDiskAlertDecision_Table(t *testing.T) {
|
|
base := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
|
alertedFail := func(sectors int, ago time.Duration) *diskRecord {
|
|
return &diskRecord{Verdict: int(agentapi.DiskVerdictFail), Alerted: true,
|
|
AlertedVerdict: int(agentapi.DiskVerdictFail), AlertedSectors: sectors, AlertedAt: base.Add(-ago)}
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
prev *diskRecord
|
|
v agentapi.DiskVerdict
|
|
sectors int
|
|
wantEmit bool
|
|
wantWorsened bool
|
|
}{
|
|
{"first ever, Warn", nil, agentapi.DiskVerdictWarn, 8, false, false},
|
|
{"first ever, Fail", nil, agentapi.DiskVerdictFail, 352, false, false},
|
|
{"OK is always silent", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn)}, agentapi.DiskVerdictOK, 0, false, false},
|
|
{"never alerted, now Warn", &diskRecord{Verdict: int(agentapi.DiskVerdictOK)}, agentapi.DiskVerdictWarn, 8, true, false},
|
|
{"escalate Warn→Fail", &diskRecord{Verdict: int(agentapi.DiskVerdictWarn), Alerted: true,
|
|
AlertedVerdict: int(agentapi.DiskVerdictWarn), AlertedAt: base}, agentapi.DiskVerdictFail, 8, true, false},
|
|
{"steady Fail, no growth, no time", alertedFail(352, time.Hour), agentapi.DiskVerdictFail, 352, false, false},
|
|
{"exactly 2x but only 23h", alertedFail(64, 23 * time.Hour), agentapi.DiskVerdictFail, 128, false, false},
|
|
{"exactly 2x at exactly 24h", alertedFail(64, 24 * time.Hour), agentapi.DiskVerdictFail, 128, true, true},
|
|
{"25h but only 1.9x", alertedFail(64, 25 * time.Hour), agentapi.DiskVerdictFail, 121, false, false},
|
|
{"improved Fail→Warn is silent", alertedFail(352, 48 * time.Hour), agentapi.DiskVerdictWarn, 8, false, false},
|
|
{"Fail from heat (0 sectors) never re-alerts on the doubling rule",
|
|
alertedFail(0, 48 * time.Hour), agentapi.DiskVerdictFail, 0, false, false},
|
|
}
|
|
for _, c := range cases {
|
|
emit, worsened := diskAlertDecision(c.prev, c.v, c.sectors, base)
|
|
if emit != c.wantEmit || worsened != c.wantWorsened {
|
|
t.Errorf("%s: got (emit=%v worsened=%v), want (emit=%v worsened=%v)",
|
|
c.name, emit, worsened, c.wantEmit, c.wantWorsened)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Preserved v0.169.0/v0.171.0 coverage ────────────────────────────────────────────────────────
|
|
|
|
// The card renders gracefully with nil SMART (old agent / no data): the row shows "Nincs adat" and
|
|
// never errors; a physical disk with no smart field is still listed.
|
|
func TestDiskHealthRows_NilSmart(t *testing.T) {
|
|
h := newDiskHarness(t)
|
|
h.set(
|
|
agentapi.DiskInfo{Name: "sdb", BackingDevice: "/dev/sdb", Smart: nil}, // physical, no smart → Nincs adat
|
|
// PBS/LVM carry a default UNKNOWN SMART from the agent — must STILL be excluded (not disks).
|
|
agentapi.DiskInfo{Name: "felhom-pbs", Type: "pbs", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
|
agentapi.DiskInfo{Name: "local-lvm", Type: "lvmthin", Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}},
|
|
agentapi.DiskInfo{Name: "sdc", BackingDevice: "/dev/sdc", Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, TemperatureC: smartPtr(31)}},
|
|
)
|
|
rows := h.s.diskHealthRows(context.Background())
|
|
if len(rows) != 2 {
|
|
t.Fatalf("want 2 physical-disk rows (pbs+lvm excluded), got %d: %+v", len(rows), rows)
|
|
}
|
|
byLabel := map[string]DiskHealthRow{}
|
|
for _, r := range rows {
|
|
byLabel[r.Label] = r
|
|
}
|
|
if byLabel["sdb"].ChipLabel != "Nincs adat" {
|
|
t.Errorf("nil-smart disk chip = %q, want Nincs adat", byLabel["sdb"].ChipLabel)
|
|
}
|
|
if byLabel["sdc"].ChipLabel != "Rendben" || byLabel["sdc"].Temp != "31" {
|
|
t.Errorf("sdc row = %+v, want Rendben / 31", byLabel["sdc"])
|
|
}
|
|
}
|
|
|
|
// TTL cache: two card fetches inside 60s hit the agent once.
|
|
func TestCachedDisks_TTL(t *testing.T) {
|
|
s := testServer(t)
|
|
calls := 0
|
|
s.disksFn = func(ctx context.Context) (agentapi.DisksResponse, error) {
|
|
calls++
|
|
return agentapi.DisksResponse{}, nil
|
|
}
|
|
ctx := context.Background()
|
|
_, _ = s.cachedDisks(ctx)
|
|
_, _ = s.cachedDisks(ctx)
|
|
if calls != 1 {
|
|
t.Errorf("two fetches within the TTL should call the agent once, got %d", calls)
|
|
}
|
|
}
|
|
|
|
func sptr(s string) *string { return &s }
|
|
|
|
// v0.171.0: the card label prefers the device model (agent v0.95.0) over the raw name/UUID; it falls
|
|
// back to Name (+ speed hint) on an older agent or a modelless disk.
|
|
// Red-proof: drop the fallback (always return ModelName) → the nil-model/old-agent cases return "" and
|
|
// TestDiskDisplayLabel_PrefersModel fails (A4 — old-payload tolerance).
|
|
func TestDiskDisplayLabel_PrefersModel(t *testing.T) {
|
|
withModel := agentapi.DiskInfo{Name: "47a3361a-uuid", Class: "slow",
|
|
Smart: &agentapi.SmartSummary{Health: agentapi.SmartPassed, ModelName: sptr("TOSHIBA MQ04ABF100")}}
|
|
if got := diskDisplayLabel(withModel); got != "TOSHIBA MQ04ABF100" {
|
|
t.Errorf("label = %q, want the model name", got)
|
|
}
|
|
// A4: an old (v0.94.0) agent carries no model → Name (+ speed hint), byte-identical to before.
|
|
oldAgent := agentapi.DiskInfo{Name: "local", Class: "fast",
|
|
Smart: &agentapi.SmartSummary{Health: agentapi.SmartUnknown}}
|
|
if got := diskDisplayLabel(oldAgent); got != "local (gyors)" {
|
|
t.Errorf("old-agent label = %q, want 'local (gyors)'", got)
|
|
}
|
|
// No SMART at all → bare Name.
|
|
if got := diskDisplayLabel(agentapi.DiskInfo{Name: "sdb"}); got != "sdb" {
|
|
t.Errorf("nil-smart label = %q, want 'sdb'", got)
|
|
}
|
|
}
|