hub v0.25.0: per-storage worst-fill alerting (StorageFillChecker)

Generalizes host_disk to any reported storage target (dump/backup volume, data drive, thin pool, PBS). Per-(host,target) state, born/persistent, natural critical severity, distinct storage_fill_* events; excludes the root-backed builtin (host_disk owns root). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs
2026-06-30 19:01:20 +02:00
parent cac745ee04
commit 88073ac464
7 changed files with 445 additions and 0 deletions
@@ -0,0 +1,201 @@
+package monitor
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// StorageFillChecker generalizes HostDiskChecker from "host root" to "any storage target": it alerts when
+// a SPECIFIC reported storage target (a vzdump/backup volume, a data drive, an lvmthin pool, a PBS
+// datastore, …) crosses a warning/critical fill threshold — naming the target. The felhom-pve incident's
+// root-backed dump volume is already covered by HostDiskChecker (root fills); this adds the case the
+// host-root signal MISSES: a dedicated backup/data storage on its OWN disk filling while root stays fine.
+//
+// It mirrors HostDiskChecker exactly (born/persistent, escalation-only emit, recovery re-arm, the
+// dispatcher's 1h cooldown) but keys state per (host, target) and emits distinct storage_fill_* events at
+// the NATURAL `critical` severity (hub v0.24.0 routes it). It EXCLUDES the host root-backed builtin
+// storage so it never double-alerts what HostDiskChecker already owns.
+type StorageFillChecker struct {
+	store   *store.Store
+	warn    float64
+	crit    float64
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	states     map[string]string // key(host,target) → band; a breached key is left UNSEEDED at init (F2)
+	customerOf map[string]string // hostID → customerID
+}
+
+const (
+	defaultStorageFillWarnPercent = 90.0
+	defaultStorageFillCritPercent = 95.0
+)
+
+// fillKey is the per-(host,target) state key. A NUL separator can't appear in a host id / storage name.
+func fillKey(hostID, target string) string { return hostID + "\x00" + target }
+
+// excludeFromStorageFill reports whether a target is the host root-backed builtin (HostDiskChecker owns
+// it — alerting here too would double-page for the same filesystem). The PVE builtin "local" lives on the
+// root fs (its used_fraction == root fill); a target mounted at "/" is the root itself.
+func excludeFromStorageFill(typ, mountPath string) bool {
+	return typ == "local" || cleanFillPath(mountPath) == "/"
+}
+
+func cleanFillPath(p string) string {
+	if p == "" {
+		return "" // root-backed dirs report "" — excluded via Type=="local", not here
+	}
+	trimmed := strings.TrimRight(p, "/")
+	if trimmed == "" {
+		return "/" // p was "/" (or "///") — the literal root mount
+	}
+	return trimmed
+}
+
+// NewStorageFillChecker creates the checker (defaults 90/95 when unset/invalid) and seeds state from the
+// latest reports. NO events on init except that already-breached (host,target) keys are left UNSEEDED so
+// their first Check emits (born/persistent — the F2 lesson).
+func NewStorageFillChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *StorageFillChecker {
+	warn, crit := normalizeDiskThresholds(warnPercent, critPercent) // reuse host_disk's sane-defaults guard
+	fc := &StorageFillChecker{
+		store:      s,
+		warn:       warn,
+		crit:       crit,
+		logger:     logger,
+		onEvent:    onEvent,
+		states:     make(map[string]string),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostStorageTargets()
+	if err != nil {
+		logger.Printf("[WARN] Storage fill checker: failed to seed states: %v", err)
+		return fc
+	}
+	var okCount, breachedCount, excluded int
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) {
+			continue
+		}
+		if excludeFromStorageFill(row.Type, row.MountPath) {
+			excluded++
+			continue
+		}
+		fc.customerOf[row.HostID] = row.CustomerID
+		band := bandForPercent(row.Percent, fc.warn, fc.crit)
+		if band != bandOK {
+			breachedCount++
+			continue // leave UNSEEDED → first Check emits (the dispatcher's 1h cooldown dedups a restart)
+		}
+		fc.states[fillKey(row.HostID, row.Name)] = bandOK
+		okCount++
+	}
+	logger.Printf("[INFO] Storage fill checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded, %d root-backed excluded", warn, crit, okCount, breachedCount, excluded)
+	return fc
+}
+
+// Check evaluates every (host, non-root target) and emits on each escalation (incl. the born-breach).
+// De-escalation/recovery re-arm silently. Same 60s sweep as the other host checkers.
+func (fc *StorageFillChecker) Check() {
+	rows, err := fc.store.GetHostStorageTargets()
+	if err != nil {
+		fc.logger.Printf("[WARN] Storage fill check failed: %v", err)
+		return
+	}
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if excludeFromStorageFill(row.Type, row.MountPath) {
+			continue // root-backed builtin — HostDiskChecker owns it
+		}
+		key := fillKey(row.HostID, row.Name)
+		seen[key] = true
+		if fc.store.IsCustomerBlocked(row.CustomerID) {
+			delete(fc.states, key)
+			continue
+		}
+		fc.customerOf[row.HostID] = row.CustomerID
+
+		newBand := bandForPercent(row.Percent, fc.warn, fc.crit)
+		oldBand := fc.states[key] // "" (rank 0) for an unseen / breached-at-init key
+		if bandRank(newBand) > bandRank(oldBand) {
+			fc.emit(row, oldBand, newBand)
+		}
+		fc.states[key] = newBand
+	}
+
+	// Drop state for targets that vanished from the latest report (storage removed) so a later re-add
+	// re-arms cleanly.
+	for k := range fc.states {
+		if !seen[k] {
+			delete(fc.states, k)
+		}
+	}
+}
+
+// GetState returns the current band for a (host, target) ("unknown" if unseen). For tests.
+func (fc *StorageFillChecker) GetState(hostID, target string) string {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+	s := fc.states[fillKey(hostID, target)]
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func (fc *StorageFillChecker) emit(row store.HostStorageTargetRow, oldBand, newBand string) {
+	var eventType, severity, message string
+	switch newBand {
+	case bandCritical:
+		eventType = "storage_fill_critical"
+		severity = "critical" // natural critical — hub v0.24.0 routes it; the operator email styles it 🔴
+		message = fmt.Sprintf("Host %s: storage %q CRITICALLY full at %.0f%% (threshold %.0f%%) — backups/writes to it will fail; free space immediately", row.HostID, row.Name, row.Percent, fc.crit)
+	case bandWarning:
+		eventType = "storage_fill_warning"
+		severity = "warning"
+		message = fmt.Sprintf("Host %s: storage %q high at %.0f%% (threshold %.0f%%) — free space before it fills", row.HostID, row.Name, row.Percent, fc.warn)
+	default:
+		return
+	}
+
+	details, _ := json.Marshal(map[string]any{
+		"host_id":      row.HostID,
+		"storage":      row.Name,
+		"storage_type": row.Type,
+		"percent":      row.Percent,
+		"total_bytes":  row.TotalBytes,
+		"used_bytes":   row.UsedBytes,
+		"warn_percent": fc.warn,
+		"crit_percent": fc.crit,
+	})
+
+	fc.logger.Printf("[INFO] Storage fill: %s %q %.0f%% %s→%s (%s)", row.HostID, row.Name, row.Percent, bandLabel(oldBand), newBand, eventType)
+
+	if _, err := fc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil {
+		fc.logger.Printf("[WARN] Failed to save storage fill event for %s/%s: %v", row.HostID, row.Name, err)
+		return
+	}
+	if fc.onEvent != nil {
+		fc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub")
+	}
+}
+
+// bandForPercent maps a fill percentage to its band (free function so both checkers share the bands).
+func bandForPercent(pct, warn, crit float64) string {
+	switch {
+	case pct >= crit:
+		return bandCritical
+	case pct >= warn:
+		return bandWarning
+	default:
+		return bandOK
+	}
+}