package monitor import ( "encoding/json" "fmt" "log" "strings" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // StorageFillChecker generalizes HostDiskChecker from "host root" to "any storage target": it alerts when // a SPECIFIC reported storage target (a vzdump/backup volume, a data drive, an lvmthin pool, a PBS // datastore, …) crosses a warning/critical fill threshold — naming the target. The felhom-pve incident's // root-backed dump volume is already covered by HostDiskChecker (root fills); this adds the case the // host-root signal MISSES: a dedicated backup/data storage on its OWN disk filling while root stays fine. // // It mirrors HostDiskChecker exactly (born/persistent, escalation-only emit, recovery re-arm, the // dispatcher's 1h cooldown) but keys state per (host, target) and emits distinct storage_fill_* events at // the NATURAL `critical` severity (hub v0.24.0 routes it). It EXCLUDES the host root-backed builtin // storage so it never double-alerts what HostDiskChecker already owns. type StorageFillChecker struct { store *store.Store warn float64 crit float64 logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex states map[string]string // key(host,target) → band; a breached key is left UNSEEDED at init (F2) customerOf map[string]string // hostID → customerID } const ( defaultStorageFillWarnPercent = 90.0 defaultStorageFillCritPercent = 95.0 // R-672 (hub v0.124.0): an lvmthin pool is CRITICAL at 90 % of data OR metadata, warned at 85 %. A full // thin pool does not merely refuse writes — every guest on it gets I/O errors and remounts read-only // (measured on demo-hp 2026-09-24: 9201's disks went read-only 5 minutes after the pool reached 100 %, // and it went from 95 % to 100 % in about a minute). The generic 90/95 bands leave no time to act. thinPoolWarnPercent = 85.0 thinPoolCritPercent = 90.0 ) // judged is what the bands are applied to: a thin pool's worse of data and metadata, with its own bands. func (fc *StorageFillChecker) judged(row store.HostStorageTargetRow) (pct, warn, crit float64) { if row.Type != "lvmthin" { return row.Percent, fc.warn, fc.crit } pct = row.Percent if row.MetaPercent > pct { pct = row.MetaPercent } return pct, minF(fc.warn, thinPoolWarnPercent), minF(fc.crit, thinPoolCritPercent) } func minF(a, b float64) float64 { if a < b { return a } return b } // fillKey is the per-(host,target) state key. A NUL separator can't appear in a host id / storage name. func fillKey(hostID, target string) string { return hostID + "\x00" + target } // excludeFromStorageFill reports whether a target is the host root-backed builtin (HostDiskChecker owns // it — alerting here too would double-page for the same filesystem). The PVE builtin "local" lives on the // root fs (its used_fraction == root fill); a target mounted at "/" is the root itself. func excludeFromStorageFill(typ, mountPath string) bool { return typ == "local" || cleanFillPath(mountPath) == "/" } func cleanFillPath(p string) string { if p == "" { return "" // root-backed dirs report "" — excluded via Type=="local", not here } trimmed := strings.TrimRight(p, "/") if trimmed == "" { return "/" // p was "/" (or "///") — the literal root mount } return trimmed } // NewStorageFillChecker creates the checker (defaults 90/95 when unset/invalid) and seeds state from the // latest reports. NO events on init except that already-breached (host,target) keys are left UNSEEDED so // their first Check emits (born/persistent — the F2 lesson). func NewStorageFillChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *StorageFillChecker { warn, crit := normalizeDiskThresholds(warnPercent, critPercent) // reuse host_disk's sane-defaults guard fc := &StorageFillChecker{ store: s, warn: warn, crit: crit, logger: logger, onEvent: onEvent, states: make(map[string]string), customerOf: make(map[string]string), } rows, err := s.GetHostStorageTargets() if err != nil { logger.Printf("[WARN] Storage fill checker: failed to seed states: %v", err) return fc } var okCount, breachedCount, excluded int for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) { continue } if excludeFromStorageFill(row.Type, row.MountPath) { excluded++ continue } fc.customerOf[row.HostID] = row.CustomerID pct, w, c := fc.judged(row) band := bandForPercent(pct, w, c) if band != bandOK { breachedCount++ continue // leave UNSEEDED → first Check emits (the dispatcher's 1h cooldown dedups a restart) } fc.states[fillKey(row.HostID, row.Name)] = bandOK okCount++ } logger.Printf("[INFO] Storage fill checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded, %d root-backed excluded", warn, crit, okCount, breachedCount, excluded) return fc } // Check evaluates every (host, non-root target) and emits on each escalation (incl. the born-breach). // De-escalation/recovery re-arm silently. Same 60s sweep as the other host checkers. func (fc *StorageFillChecker) Check() { rows, err := fc.store.GetHostStorageTargets() if err != nil { fc.logger.Printf("[WARN] Storage fill check failed: %v", err) return } fc.mu.Lock() defer fc.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { if excludeFromStorageFill(row.Type, row.MountPath) { continue // root-backed builtin — HostDiskChecker owns it } key := fillKey(row.HostID, row.Name) seen[key] = true if fc.store.IsCustomerBlocked(row.CustomerID) { delete(fc.states, key) continue } fc.customerOf[row.HostID] = row.CustomerID pct, w, c := fc.judged(row) newBand := bandForPercent(pct, w, c) oldBand := fc.states[key] // "" (rank 0) for an unseen / breached-at-init key if bandRank(newBand) > bandRank(oldBand) { fc.emit(row, oldBand, newBand) } fc.states[key] = newBand } // Drop state for targets that vanished from the latest report (storage removed) so a later re-add // re-arms cleanly. for k := range fc.states { if !seen[k] { delete(fc.states, k) } } } // GetState returns the current band for a (host, target) ("unknown" if unseen). For tests. func (fc *StorageFillChecker) GetState(hostID, target string) string { fc.mu.Lock() defer fc.mu.Unlock() s := fc.states[fillKey(hostID, target)] if s == "" { return "unknown" } return s } func (fc *StorageFillChecker) emit(row store.HostStorageTargetRow, oldBand, newBand string) { var eventType, severity, message string pct, warn, crit := fc.judged(row) thin := "" if row.Type == "lvmthin" { thin = fmt.Sprintf(" (thin pool: data %.0f%%, metadata %s) — a full thin pool turns EVERY guest on it read-only", row.Percent, metaLabel(row.MetaPercent)) } switch newBand { case bandCritical: eventType = "storage_fill_critical" severity = "critical" // natural critical — hub v0.24.0 routes it; the operator email styles it 🔴 message = fmt.Sprintf("Host %s: storage %q CRITICALLY full at %.0f%% (threshold %.0f%%)%s — backups/writes to it will fail; free space immediately", row.HostID, row.Name, pct, crit, thin) case bandWarning: eventType = "storage_fill_warning" severity = "warning" message = fmt.Sprintf("Host %s: storage %q high at %.0f%% (threshold %.0f%%)%s — free space before it fills", row.HostID, row.Name, pct, warn, thin) default: return } details, _ := json.Marshal(map[string]any{ "host_id": row.HostID, "storage": row.Name, "storage_type": row.Type, "percent": pct, "data_percent": row.Percent, "meta_percent": row.MetaPercent, "total_bytes": row.TotalBytes, "used_bytes": row.UsedBytes, "warn_percent": warn, "crit_percent": crit, }) fc.logger.Printf("[INFO] Storage fill: %s %q %.0f%% %s→%s (%s)", row.HostID, row.Name, pct, bandLabel(oldBand), newBand, eventType) if _, err := fc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil { fc.logger.Printf("[WARN] Failed to save storage fill event for %s/%s: %v", row.HostID, row.Name, err) return } if fc.onEvent != nil { fc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub") } } // bandForPercent maps a fill percentage to its band (free function so both checkers share the bands). func bandForPercent(pct, warn, crit float64) string { switch { case pct >= crit: return bandCritical case pct >= warn: return bandWarning default: return bandOK } } func metaLabel(p float64) string { if p < 0 { return "unknown" } return fmt.Sprintf("%.0f%%", p) }