package web import ( "encoding/json" "fmt" "os" "path/filepath" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/agentapi" "gitea.dooplex.hu/admin/felhom-controller/internal/notify" ) // Persisted disk-health reporting state (v0.215.0). // // WHY IT IS PERSISTED: until v0.215.0 the baseline was in-memory only, so a controller restart while // a disk was already failing re-baselined that disk silently — and it never alerted again. A box that // reboots during exactly the fault this feature exists for was the one case it could not report. // // WHAT IT IS NOT: one small record per disk about what has been OBSERVED and REPORTED. It is NOT a // sample history and must not grow into one — SMART sample series belong in metrics.MetricsStore // under Phase 2/3, which is a schema migration and a wire change. Keeping this a flat file keeps a // one-word severity fix off that critical path. // // Crash-safety: atomic tmp+rename is sufficient. The state describes reporting history, not a // mutation in flight, so a lost write costs at most one re-baseline — and the failure direction is a // DUPLICATE alert rather than a missing one. const diskStateFileName = "disk-health-state.json" // diskStateVersion guards the on-disk shape. An unrecognised version is treated as "no prior" // (fail-safe) rather than mis-parsed into a verdict. const diskStateVersion = 1 // diskRealertCooldown is the minimum gap between two alerts for a disk already at Hiba. It is ANDed // with the doubling bar — clearing only one of them emits nothing. const diskRealertCooldown = 24 * time.Hour // diskRecord is everything the checker remembers about ONE disk between runs. type diskRecord struct { // Verdict is the last non-UNKNOWN verdict observed (agentapi.DiskVerdict as an int). Verdict int `json:"verdict"` // SawUncorrectable is THIS observation: it becomes the NEXT check's agentapi.DiskPrior, and is // what turns a one-off excursion into a sustained fault. SawUncorrectable bool `json:"saw_uncorrectable"` // PriorSawUncorrectable is the prior that PRODUCED Verdict — i.e. the previous observation. // // It exists so the CARD can reach the same verdict as the check. The two run at different times // over the same SMART: the check consumes the prior and then overwrites it with the current // observation, so a card rendering afterwards would consume its own check's write and read one // level too high — a disk at its first sighting of 8 sectors would show "Hiba" on the dashboard // while the alert (correctly) said "Figyelmeztetés". Replaying the SAME prior keeps the shared // verdict function's guarantee intact instead of merely asserting it. PriorSawUncorrectable bool `json:"prior_saw_uncorrectable,omitempty"` // Sectors is max(pending, offline_uncorrectable) at the last observation. Sectors int `json:"sectors,omitempty"` // ChangedAt is when Verdict last changed value. ChangedAt time.Time `json:"changed_at,omitempty"` // --- alert history: survives an intervening improvement, which is what makes flap damping work --- // Alerted is false until this disk has ever produced an event. Distinguishes "never alerted" // from "alerted at Nincs adat", which cannot happen but would otherwise share the zero value. Alerted bool `json:"alerted,omitempty"` AlertedVerdict int `json:"alerted_verdict,omitempty"` AlertedSectors int `json:"alerted_sectors,omitempty"` AlertedAt time.Time `json:"alerted_at,omitempty"` } // diskStateFile is the on-disk envelope. type diskStateFile struct { Version int `json:"version"` Disks map[string]*diskRecord `json:"disks"` } // diskStatePath is the state file's location, or "" when no data dir is configured (unit tests and // an unprovisioned guest). An empty path degrades to in-memory-only: correct behaviour within the // process, simply not durable. func (s *Server) diskStatePath() string { if s.cfg == nil || s.cfg.Paths.DataDir == "" { return "" } return filepath.Join(s.cfg.Paths.DataDir, diskStateFileName) } // loadDiskStateLocked populates the in-memory records from disk exactly once per process. The caller // holds s.diskHealth.mu. // // NEVER FATAL. A missing file is the normal first-boot case. A corrupt or unreadable one is LOGGED // and treated as "no prior" — the worst that costs is one silent re-baseline, whereas crashing the // check would take away disk monitoring entirely. func (s *Server) loadDiskStateLocked() { if s.diskHealth.loaded { return } s.diskHealth.loaded = true if s.diskHealth.records == nil { s.diskHealth.records = map[string]*diskRecord{} } path := s.diskStatePath() if path == "" { return } data, err := os.ReadFile(path) if err != nil { if !os.IsNotExist(err) && s.logger != nil { s.logger.Printf("[WARN] [web] disk-health state unreadable, continuing with no prior: %v", err) } return } var f diskStateFile if err := json.Unmarshal(data, &f); err != nil { if s.logger != nil { s.logger.Printf("[WARN] [web] disk-health state corrupt, continuing with no prior: %v", err) } return } if f.Version != diskStateVersion { if s.logger != nil { s.logger.Printf("[WARN] [web] disk-health state version %d unrecognised (want %d), continuing with no prior", f.Version, diskStateVersion) } return } for k, r := range f.Disks { if r != nil { s.diskHealth.records[k] = r } } if s.logger != nil { s.logger.Printf("[DEBUG] [web] disk-health state loaded: %d disk(s)", len(s.diskHealth.records)) } } // saveDiskStateLocked writes the records atomically (.tmp then rename), following the shape of // selfupdate.SaveState. Called ONCE per check run, after every disk has been evaluated — not per // disk. Caller holds s.diskHealth.mu. Errors are logged, never returned: failing to remember is not // a reason to stop monitoring. func (s *Server) saveDiskStateLocked() { path := s.diskStatePath() if path == "" { return } if err := writeDiskState(path, s.diskHealth.records); err != nil && s.logger != nil { s.logger.Printf("[WARN] [web] disk-health state not saved (a restart will re-baseline): %v", err) } } func writeDiskState(path string, records map[string]*diskRecord) error { data, err := json.MarshalIndent(diskStateFile{Version: diskStateVersion, Disks: records}, "", " ") if err != nil { return fmt.Errorf("marshaling disk-health state: %w", err) } tmp := path + ".tmp" if err := os.WriteFile(tmp, data, 0644); err != nil { return fmt.Errorf("writing temp disk-health state: %w", err) } if err := os.Rename(tmp, path); err != nil { os.Remove(tmp) return fmt.Errorf("renaming disk-health state: %w", err) } return nil } // priorFor is the agentapi.DiskPrior the CHECK consumes: what the previous check observed. A zero // DiskPrior ("nothing known") is the fail-safe — a first sighting can then only reach // Figyelmeztetés, never Hiba. Caller holds the mutex. func (s *Server) priorFor(key string) agentapi.DiskPrior { if r := s.diskHealth.records[key]; r != nil { return agentapi.DiskPrior{SawUncorrectable: r.SawUncorrectable} } return agentapi.DiskPrior{} } // cardPriorFor is the agentapi.DiskPrior the CARD replays: the prior that produced the stored // verdict, NOT the observation that check went on to record. See diskRecord.PriorSawUncorrectable — // using priorFor here would make the dashboard chip read one level more severe than the alert. // Caller holds the mutex. func (s *Server) cardPriorFor(key string) agentapi.DiskPrior { if r := s.diskHealth.records[key]; r != nil { return agentapi.DiskPrior{SawUncorrectable: r.PriorSawUncorrectable} } return agentapi.DiskPrior{} } // diskAlertDecision decides whether THIS observation produces an event, per the v0.215.0 rules. // Pure: everything, including the clock reading, arrives as an argument. // // prev == nil -> no event (first verdict ever for this disk: baseline silently) // v is Rendben (or better) -> no event (recovery is silent — unchanged behaviour) // v worse than the LAST ALERTED -> EVENT. Escalation is never damped. // v == last alerted, and v is Hiba, // and sectors >= 2x the count at // the last alert, and >= 24h since -> EVENT ("still getting worse") // anything else -> no event // // The second return value reports whether this is the "still getting worse" re-alert, which selects // a different message shape. // // WHY "worse than the last ALERTED verdict" rather than the last OBSERVED one: a disk that goes // Figyelmeztetés -> Rendben -> Figyelmeztetés has worsened against its last observation twice, and // today's transition-only logic emits two identical mild alerts on that zero-crossing. That is how a // customer learns to ignore the message. Comparing against what was last REPORTED collapses the flap // to one alert while leaving a genuine escalation (Figyelmeztetés -> Hiba) free to fire immediately. // // WHY nothing else emits at v == last alerted: the steady-state case and the post-improvement flap // case are indistinguishable without tracking intervening improvements, and both must stay silent. // Only the explicit Hiba re-alert above breaks that silence, and only when BOTH its bars are cleared. func diskAlertDecision(prev *diskRecord, v agentapi.DiskVerdict, sectors int, now time.Time) (emit bool, worsened bool) { if v <= agentapi.DiskVerdictOK { return false, false // Nincs adat never reaches here; Rendben is silent } if prev == nil { return false, false // first verdict ever for this disk } alerted := agentapi.DiskVerdictUnknown if prev.Alerted { alerted = agentapi.DiskVerdict(prev.AlertedVerdict) } if v > alerted { return true, false // escalation — damping never applies } if v == alerted && v == agentapi.DiskVerdictFail && prev.AlertedSectors > 0 && sectors >= 2*prev.AlertedSectors && now.Sub(prev.AlertedAt) >= diskRealertCooldown { return true, true // still getting worse } return false, false } // diskAlertKindFor picks the customer-facing message shape for an alert that has already been // decided. It mirrors the truth-table order so the reason quoted to the customer is the reason the // verdict actually fired on. func diskAlertKindFor(sm *agentapi.SmartSummary, v agentapi.DiskVerdict, worsened bool) notify.DiskAlertKind { if v != agentapi.DiskVerdictFail { return notify.DiskAlertWarn } switch { case sm != nil && sm.Health == agentapi.SmartFailing: return notify.DiskAlertFailSelfReported case sm != nil && sm.TemperatureC != nil && *sm.TemperatureC >= agentapi.TemperatureFailC: return notify.DiskAlertFailTemperature case agentapi.UncorrectableSectors(sm) > 0 && worsened: return notify.DiskAlertFailWorsened case agentapi.UncorrectableSectors(sm) > 0: return notify.DiskAlertFailSectors } // Hiba with no sector count and no heat: NVMe's own critical flag, or spent rated endurance. // Both are declarations BY THE DEVICE, so the self-reported wording is the honest one. return notify.DiskAlertFailSelfReported }