package monitor import ( "encoding/json" "fmt" "log" "strings" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // StorageFillChecker generalizes HostDiskChecker from "host root" to "any storage target": it alerts when // a SPECIFIC reported storage target (a vzdump/backup volume, a data drive, an lvmthin pool, a PBS // datastore, …) crosses a warning/critical fill threshold — naming the target. The felhom-pve incident's // root-backed dump volume is already covered by HostDiskChecker (root fills); this adds the case the // host-root signal MISSES: a dedicated backup/data storage on its OWN disk filling while root stays fine. // // It mirrors HostDiskChecker exactly (born/persistent, escalation-only emit, recovery re-arm, the // dispatcher's 1h cooldown) but keys state per (host, target) and emits distinct storage_fill_* events at // the NATURAL `critical` severity (hub v0.24.0 routes it). It EXCLUDES the host root-backed builtin // storage so it never double-alerts what HostDiskChecker already owns. type StorageFillChecker struct { store *store.Store warn float64 crit float64 logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex states map[string]string // key(host,target) → band; a breached key is left UNSEEDED at init (F2) customerOf map[string]string // hostID → customerID } const ( defaultStorageFillWarnPercent = 90.0 defaultStorageFillCritPercent = 95.0 ) // fillKey is the per-(host,target) state key. A NUL separator can't appear in a host id / storage name. func fillKey(hostID, target string) string { return hostID + "\x00" + target } // excludeFromStorageFill reports whether a target is the host root-backed builtin (HostDiskChecker owns // it — alerting here too would double-page for the same filesystem). The PVE builtin "local" lives on the // root fs (its used_fraction == root fill); a target mounted at "/" is the root itself. func excludeFromStorageFill(typ, mountPath string) bool { return typ == "local" || cleanFillPath(mountPath) == "/" } func cleanFillPath(p string) string { if p == "" { return "" // root-backed dirs report "" — excluded via Type=="local", not here } trimmed := strings.TrimRight(p, "/") if trimmed == "" { return "/" // p was "/" (or "///") — the literal root mount } return trimmed } // NewStorageFillChecker creates the checker (defaults 90/95 when unset/invalid) and seeds state from the // latest reports. NO events on init except that already-breached (host,target) keys are left UNSEEDED so // their first Check emits (born/persistent — the F2 lesson). func NewStorageFillChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *StorageFillChecker { warn, crit := normalizeDiskThresholds(warnPercent, critPercent) // reuse host_disk's sane-defaults guard fc := &StorageFillChecker{ store: s, warn: warn, crit: crit, logger: logger, onEvent: onEvent, states: make(map[string]string), customerOf: make(map[string]string), } rows, err := s.GetHostStorageTargets() if err != nil { logger.Printf("[WARN] Storage fill checker: failed to seed states: %v", err) return fc } var okCount, breachedCount, excluded int for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) { continue } if excludeFromStorageFill(row.Type, row.MountPath) { excluded++ continue } fc.customerOf[row.HostID] = row.CustomerID band := bandForPercent(row.Percent, fc.warn, fc.crit) if band != bandOK { breachedCount++ continue // leave UNSEEDED → first Check emits (the dispatcher's 1h cooldown dedups a restart) } fc.states[fillKey(row.HostID, row.Name)] = bandOK okCount++ } logger.Printf("[INFO] Storage fill checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded, %d root-backed excluded", warn, crit, okCount, breachedCount, excluded) return fc } // Check evaluates every (host, non-root target) and emits on each escalation (incl. the born-breach). // De-escalation/recovery re-arm silently. Same 60s sweep as the other host checkers. func (fc *StorageFillChecker) Check() { rows, err := fc.store.GetHostStorageTargets() if err != nil { fc.logger.Printf("[WARN] Storage fill check failed: %v", err) return } fc.mu.Lock() defer fc.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { if excludeFromStorageFill(row.Type, row.MountPath) { continue // root-backed builtin — HostDiskChecker owns it } key := fillKey(row.HostID, row.Name) seen[key] = true if fc.store.IsCustomerBlocked(row.CustomerID) { delete(fc.states, key) continue } fc.customerOf[row.HostID] = row.CustomerID newBand := bandForPercent(row.Percent, fc.warn, fc.crit) oldBand := fc.states[key] // "" (rank 0) for an unseen / breached-at-init key if bandRank(newBand) > bandRank(oldBand) { fc.emit(row, oldBand, newBand) } fc.states[key] = newBand } // Drop state for targets that vanished from the latest report (storage removed) so a later re-add // re-arms cleanly. for k := range fc.states { if !seen[k] { delete(fc.states, k) } } } // GetState returns the current band for a (host, target) ("unknown" if unseen). For tests. func (fc *StorageFillChecker) GetState(hostID, target string) string { fc.mu.Lock() defer fc.mu.Unlock() s := fc.states[fillKey(hostID, target)] if s == "" { return "unknown" } return s } func (fc *StorageFillChecker) emit(row store.HostStorageTargetRow, oldBand, newBand string) { var eventType, severity, message string switch newBand { case bandCritical: eventType = "storage_fill_critical" severity = "critical" // natural critical — hub v0.24.0 routes it; the operator email styles it 🔴 message = fmt.Sprintf("Host %s: storage %q CRITICALLY full at %.0f%% (threshold %.0f%%) — backups/writes to it will fail; free space immediately", row.HostID, row.Name, row.Percent, fc.crit) case bandWarning: eventType = "storage_fill_warning" severity = "warning" message = fmt.Sprintf("Host %s: storage %q high at %.0f%% (threshold %.0f%%) — free space before it fills", row.HostID, row.Name, row.Percent, fc.warn) default: return } details, _ := json.Marshal(map[string]any{ "host_id": row.HostID, "storage": row.Name, "storage_type": row.Type, "percent": row.Percent, "total_bytes": row.TotalBytes, "used_bytes": row.UsedBytes, "warn_percent": fc.warn, "crit_percent": fc.crit, }) fc.logger.Printf("[INFO] Storage fill: %s %q %.0f%% %s→%s (%s)", row.HostID, row.Name, row.Percent, bandLabel(oldBand), newBand, eventType) if _, err := fc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil { fc.logger.Printf("[WARN] Failed to save storage fill event for %s/%s: %v", row.HostID, row.Name, err) return } if fc.onEvent != nil { fc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub") } } // bandForPercent maps a fill percentage to its band (free function so both checkers share the bands). func bandForPercent(pct, warn, crit float64) string { switch { case pct >= crit: return bandCritical case pct >= warn: return bandWarning default: return bandOK } }