felhom.eu/hub/internal/monitor/storage_fill.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"sync"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// StorageFillChecker generalizes HostDiskChecker from "host root" to "any storage target": it alerts when
// a SPECIFIC reported storage target (a vzdump/backup volume, a data drive, an lvmthin pool, a PBS
// datastore, …) crosses a warning/critical fill threshold — naming the target. The felhom-pve incident's
// root-backed dump volume is already covered by HostDiskChecker (root fills); this adds the case the
// host-root signal MISSES: a dedicated backup/data storage on its OWN disk filling while root stays fine.
//
// It mirrors HostDiskChecker exactly (born/persistent, escalation-only emit, recovery re-arm, the
// dispatcher's 1h cooldown) but keys state per (host, target) and emits distinct storage_fill_* events at
// the NATURAL `critical` severity (hub v0.24.0 routes it). It EXCLUDES the host root-backed builtin
// storage so it never double-alerts what HostDiskChecker already owns.
type StorageFillChecker struct {
	store   *store.Store
	warn    float64
	crit    float64
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu         sync.Mutex
	states     map[string]string // key(host,target) → band; a breached key is left UNSEEDED at init (F2)
	customerOf map[string]string // hostID → customerID
}

const (
	defaultStorageFillWarnPercent = 90.0
	defaultStorageFillCritPercent = 95.0

	// R-672 (hub v0.124.0): an lvmthin pool is CRITICAL at 90 % of data OR metadata, warned at 85 %. A full
	// thin pool does not merely refuse writes — every guest on it gets I/O errors and remounts read-only
	// (measured on demo-hp 2026-09-24: 9201's disks went read-only 5 minutes after the pool reached 100 %,
	// and it went from 95 % to 100 % in about a minute). The generic 90/95 bands leave no time to act.
	thinPoolWarnPercent = 85.0
	thinPoolCritPercent = 90.0
)

// judged is what the bands are applied to: a thin pool's worse of data and metadata, with its own bands.
func (fc *StorageFillChecker) judged(row store.HostStorageTargetRow) (pct, warn, crit float64) {
	if row.Type != "lvmthin" {
		return row.Percent, fc.warn, fc.crit
	}
	pct = row.Percent
	if row.MetaPercent > pct {
		pct = row.MetaPercent
	}
	return pct, minF(fc.warn, thinPoolWarnPercent), minF(fc.crit, thinPoolCritPercent)
}

func minF(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}

// fillKey is the per-(host,target) state key. A NUL separator can't appear in a host id / storage name.
func fillKey(hostID, target string) string { return hostID + "\x00" + target }

// excludeFromStorageFill reports whether a target is the host root-backed builtin (HostDiskChecker owns
// it — alerting here too would double-page for the same filesystem). The PVE builtin "local" lives on the
// root fs (its used_fraction == root fill); a target mounted at "/" is the root itself.
func excludeFromStorageFill(typ, mountPath string) bool {
	return typ == "local" || cleanFillPath(mountPath) == "/"
}

func cleanFillPath(p string) string {
	if p == "" {
		return "" // root-backed dirs report "" — excluded via Type=="local", not here
	}
	trimmed := strings.TrimRight(p, "/")
	if trimmed == "" {
		return "/" // p was "/" (or "///") — the literal root mount
	}
	return trimmed
}

// NewStorageFillChecker creates the checker (defaults 90/95 when unset/invalid) and seeds state from the
// latest reports. NO events on init except that already-breached (host,target) keys are left UNSEEDED so
// their first Check emits (born/persistent — the F2 lesson).
func NewStorageFillChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *StorageFillChecker {
	warn, crit := normalizeDiskThresholds(warnPercent, critPercent) // reuse host_disk's sane-defaults guard
	fc := &StorageFillChecker{
		store:      s,
		warn:       warn,
		crit:       crit,
		logger:     logger,
		onEvent:    onEvent,
		states:     make(map[string]string),
		customerOf: make(map[string]string),
	}
	rows, err := s.GetHostStorageTargets()
	if err != nil {
		logger.Printf("[WARN] Storage fill checker: failed to seed states: %v", err)
		return fc
	}
	var okCount, breachedCount, excluded int
	for _, row := range rows {
		if s.IsCustomerBlocked(row.CustomerID) {
			continue
		}
		if excludeFromStorageFill(row.Type, row.MountPath) {
			excluded++
			continue
		}
		fc.customerOf[row.HostID] = row.CustomerID
		pct, w, c := fc.judged(row)
		band := bandForPercent(pct, w, c)
		if band != bandOK {
			breachedCount++
			continue // leave UNSEEDED → first Check emits (the dispatcher's 1h cooldown dedups a restart)
		}
		fc.states[fillKey(row.HostID, row.Name)] = bandOK
		okCount++
	}
	logger.Printf("[INFO] Storage fill checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded, %d root-backed excluded", warn, crit, okCount, breachedCount, excluded)
	return fc
}

// Check evaluates every (host, non-root target) and emits on each escalation (incl. the born-breach).
// De-escalation/recovery re-arm silently. Same 60s sweep as the other host checkers.
func (fc *StorageFillChecker) Check() {
	rows, err := fc.store.GetHostStorageTargets()
	if err != nil {
		fc.logger.Printf("[WARN] Storage fill check failed: %v", err)
		return
	}
	fc.mu.Lock()
	defer fc.mu.Unlock()

	seen := make(map[string]bool, len(rows))
	for _, row := range rows {
		if excludeFromStorageFill(row.Type, row.MountPath) {
			continue // root-backed builtin — HostDiskChecker owns it
		}
		key := fillKey(row.HostID, row.Name)
		seen[key] = true
		if fc.store.IsCustomerBlocked(row.CustomerID) {
			delete(fc.states, key)
			continue
		}
		fc.customerOf[row.HostID] = row.CustomerID

		pct, w, c := fc.judged(row)
		newBand := bandForPercent(pct, w, c)
		oldBand := fc.states[key] // "" (rank 0) for an unseen / breached-at-init key
		if bandRank(newBand) > bandRank(oldBand) {
			fc.emit(row, oldBand, newBand)
		}
		fc.states[key] = newBand
	}

	// Drop state for targets that vanished from the latest report (storage removed) so a later re-add
	// re-arms cleanly.
	for k := range fc.states {
		if !seen[k] {
			delete(fc.states, k)
		}
	}
}

// GetState returns the current band for a (host, target) ("unknown" if unseen). For tests.
func (fc *StorageFillChecker) GetState(hostID, target string) string {
	fc.mu.Lock()
	defer fc.mu.Unlock()
	s := fc.states[fillKey(hostID, target)]
	if s == "" {
		return "unknown"
	}
	return s
}

func (fc *StorageFillChecker) emit(row store.HostStorageTargetRow, oldBand, newBand string) {
	var eventType, severity, message string
	pct, warn, crit := fc.judged(row)
	thin := ""
	if row.Type == "lvmthin" {
		thin = fmt.Sprintf(" (thin pool: data %.0f%%, metadata %s) — a full thin pool turns EVERY guest on it read-only", row.Percent, metaLabel(row.MetaPercent))
	}
	switch newBand {
	case bandCritical:
		eventType = "storage_fill_critical"
		severity = "critical" // natural critical — hub v0.24.0 routes it; the operator email styles it 🔴
		message = fmt.Sprintf("Host %s: storage %q CRITICALLY full at %.0f%% (threshold %.0f%%)%s — backups/writes to it will fail; free space immediately", row.HostID, row.Name, pct, crit, thin)
	case bandWarning:
		eventType = "storage_fill_warning"
		severity = "warning"
		message = fmt.Sprintf("Host %s: storage %q high at %.0f%% (threshold %.0f%%)%s — free space before it fills", row.HostID, row.Name, pct, warn, thin)
	default:
		return
	}

	details, _ := json.Marshal(map[string]any{
		"host_id":      row.HostID,
		"storage":      row.Name,
		"storage_type": row.Type,
		"percent":      pct,
		"data_percent": row.Percent,
		"meta_percent": row.MetaPercent,
		"total_bytes":  row.TotalBytes,
		"used_bytes":   row.UsedBytes,
		"warn_percent": warn,
		"crit_percent": crit,
	})

	fc.logger.Printf("[INFO] Storage fill: %s %q %.0f%% %s→%s (%s)", row.HostID, row.Name, pct, bandLabel(oldBand), newBand, eventType)

	if _, err := fc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil {
		fc.logger.Printf("[WARN] Failed to save storage fill event for %s/%s: %v", row.HostID, row.Name, err)
		return
	}
	if fc.onEvent != nil {
		fc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub")
	}
}

// bandForPercent maps a fill percentage to its band (free function so both checkers share the bands).
func bandForPercent(pct, warn, crit float64) string {
	switch {
	case pct >= crit:
		return bandCritical
	case pct >= warn:
		return bandWarning
	default:
		return bandOK
	}
}

func metaLabel(p float64) string {
	if p < 0 {
		return "unknown"
	}
	return fmt.Sprintf("%.0f%%", p)
}