package monitor import ( "encoding/json" "fmt" "log" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // HostDiskChecker raises an operator alert when a Proxmox HOST's root filesystem crosses a warning or // critical fill threshold — the silent-failure class observed on felhom-pve (vzdump piling under // /var/lib/vz/dump until PVE/sqlite/logging and even the agent's own writes start failing). It reads the // HOST root `disk_percent` the agent rides in its host-report (store.GetHostDiskUsage), so it is a // deliberate SIBLING of HostCapabilityChecker / HostLeafChecker on the same 60s sweep, with the same // SaveEvent+onEvent plumbing and the dispatcher's 1h operator cooldown. // // DISTINCT from the controller's GUEST-disk events (disk_warning/disk_critical): those are the guest's own // cgroup view; this is the HOST root fs, with its own event types (host_disk_warning/host_disk_critical) // so the two never dedup or mask each other. // // BORN/PERSISTENT (the F2 lesson, §7-B): a disk ALREADY over threshold when the hub/checker (re)starts // must alert on cycle 1 — it never observes a "crossing". This is achieved by seeding ONLY ok-band hosts // at init and leaving already-breached hosts UNSEEDED, so their first Check sees oldBand=="" (rank 0) and // emits. The dispatcher's 1h cooldown dedups the re-emit across a hub restart. A transition-only design // would stay silent forever on a persistently-full disk — exactly the wrong outcome. type HostDiskChecker struct { store *store.Store warn float64 // warning threshold (percent) crit float64 // critical threshold (percent) logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex states map[string]string // hostID → band (bandOK|bandWarning|bandCritical); breached unseeded at init customerOf map[string]string // hostID → customerID (event attribution) } const ( defaultHostDiskWarnPercent = 90.0 defaultHostDiskCritPercent = 95.0 ) // Disk fill bands, ordered by severity (bandRank). const ( bandOK = "ok" bandWarning = "warning" bandCritical = "critical" ) // NewHostDiskChecker creates the checker with the configured thresholds (defaults 90/95 when unset or // invalid) and seeds state from the latest reports. NO events on init — except that already-breached // hosts are left UNSEEDED so their first Check emits (born/persistent). func NewHostDiskChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *HostDiskChecker { warn, crit := normalizeDiskThresholds(warnPercent, critPercent) dc := &HostDiskChecker{ store: s, warn: warn, crit: crit, logger: logger, onEvent: onEvent, states: make(map[string]string), customerOf: make(map[string]string), } rows, err := s.GetHostDiskUsage() if err != nil { logger.Printf("[WARN] Host disk checker: failed to seed states: %v", err) return dc } var okCount, breachedCount int for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) { continue } dc.customerOf[row.HostID] = row.CustomerID band := dc.band(row.DiskPercent) // F2: seed only OK hosts. A host already over threshold at (re)start is left UNSEEDED so the first // Check observes oldBand=="" and emits once — otherwise a box already full when the hub restarts // would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit. if band != bandOK { breachedCount++ continue } dc.states[row.HostID] = bandOK okCount++ } logger.Printf("[INFO] Host disk checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded (first Check emits)", warn, crit, okCount, breachedCount) return dc } // Check evaluates all hosts and emits on each escalation to a more severe band (ok→warning, ok/warning→ // critical), including the first-observation born-breach. De-escalation and recovery update state // silently (re-arming a future breach). Call on the same 60s sweep as the staleness checker. func (dc *HostDiskChecker) Check() { rows, err := dc.store.GetHostDiskUsage() if err != nil { dc.logger.Printf("[WARN] Host disk check failed: %v", err) return } dc.mu.Lock() defer dc.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { seen[row.HostID] = true if dc.store.IsCustomerBlocked(row.CustomerID) { delete(dc.states, row.HostID) continue } dc.customerOf[row.HostID] = row.CustomerID newBand := dc.band(row.DiskPercent) oldBand := dc.states[row.HostID] // "" (rank 0) for an unseen / breached-at-init host // Emit only when moving to a MORE severe band — this covers the born-breach (oldBand=="" → rank 0) // and any escalation, while de-escalation/recovery just re-arm (no noise, no recovery event). if bandRank(newBand) > bandRank(oldBand) { dc.emit(row, oldBand, newBand) } dc.states[row.HostID] = newBand } for id := range dc.states { if !seen[id] { delete(dc.states, id) } } } // GetState returns the current band for a host ("unknown" if unseen). func (dc *HostDiskChecker) GetState(hostID string) string { dc.mu.Lock() defer dc.mu.Unlock() s := dc.states[hostID] if s == "" { return "unknown" } return s } // band maps a fill percentage to its severity band. func (dc *HostDiskChecker) band(pct float64) string { switch { case pct >= dc.crit: return bandCritical case pct >= dc.warn: return bandWarning default: return bandOK } } // bandRank orders the bands so an escalation is a strictly increasing rank ("" / unseen = ok = 0). func bandRank(b string) int { switch b { case bandCritical: return 2 case bandWarning: return 1 default: return 0 } } func (dc *HostDiskChecker) emit(row store.HostDiskRow, oldBand, newBand string) { var eventType, severity, message string switch newBand { case bandCritical: eventType = "host_disk_critical" // Natural "critical" severity (hub v0.24.0 routes it; the operator email styles it 🔴). Before // v0.24.0 the dispatcher silently dropped "critical", so this had to be "error" — now it is honest. severity = "critical" message = fmt.Sprintf("Host %s: root filesystem CRITICALLY full at %.0f%% (threshold %.0f%%) — PVE/logging/agent writes may start failing; free space (e.g. old vzdump backups) immediately", row.HostID, row.DiskPercent, dc.crit) case bandWarning: eventType = "host_disk_warning" severity = "warning" message = fmt.Sprintf("Host %s: root filesystem high at %.0f%% (threshold %.0f%%) — free space (e.g. old backups) before it fills", row.HostID, row.DiskPercent, dc.warn) default: return // never emit for ok } details, _ := json.Marshal(map[string]any{ "host_id": row.HostID, "disk_percent": row.DiskPercent, "disk_total_bytes": row.DiskTotalBytes, "disk_used_bytes": row.DiskUsedBytes, "warn_percent": dc.warn, "crit_percent": dc.crit, }) dc.logger.Printf("[INFO] Host disk: %s root %.0f%% %s→%s (%s)", row.HostID, row.DiskPercent, bandLabel(oldBand), newBand, eventType) if _, err := dc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil { dc.logger.Printf("[WARN] Failed to save host disk event for %s: %v", row.HostID, err) return } if dc.onEvent != nil { dc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub") } } // bandLabel renders the previous band for the log line ("unknown" for an unseen/breached-at-init host). func bandLabel(b string) string { if b == "" { return "unknown" } return b } // normalizeDiskThresholds applies the 90/95 defaults and guards against an invalid/misordered config // (warn/crit out of (0,100), or crit ≤ warn) by falling back to the defaults — a config typo can never // silence the alert or invert the bands. func normalizeDiskThresholds(warn, crit float64) (float64, float64) { if warn <= 0 || warn >= 100 { warn = defaultHostDiskWarnPercent } if crit <= 0 || crit >= 100 { crit = defaultHostDiskCritPercent } if crit <= warn { return defaultHostDiskWarnPercent, defaultHostDiskCritPercent } return warn, crit }