felhom.eu/hub/internal/monitor/host_disk.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// HostDiskChecker raises an operator alert when a Proxmox HOST's root filesystem crosses a warning or
// critical fill threshold — the silent-failure class observed on felhom-pve (vzdump piling under
// /var/lib/vz/dump until PVE/sqlite/logging and even the agent's own writes start failing). It reads the
// HOST root `disk_percent` the agent rides in its host-report (store.GetHostDiskUsage), so it is a
// deliberate SIBLING of HostCapabilityChecker / HostLeafChecker on the same 60s sweep, with the same
// SaveEvent+onEvent plumbing and the dispatcher's 1h operator cooldown.
//
// DISTINCT from the controller's GUEST-disk events (disk_warning/disk_critical): those are the guest's own
// cgroup view; this is the HOST root fs, with its own event types (host_disk_warning/host_disk_critical)
// so the two never dedup or mask each other.
//
// BORN/PERSISTENT (the F2 lesson, §7-B): a disk ALREADY over threshold when the hub/checker (re)starts
// must alert on cycle 1 — it never observes a "crossing". This is achieved by seeding ONLY ok-band hosts
// at init and leaving already-breached hosts UNSEEDED, so their first Check sees oldBand=="" (rank 0) and
// emits. The dispatcher's 1h cooldown dedups the re-emit across a hub restart. A transition-only design
// would stay silent forever on a persistently-full disk — exactly the wrong outcome.
type HostDiskChecker struct {
	store   *store.Store
	warn    float64 // warning threshold (percent)
	crit    float64 // critical threshold (percent)
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu         sync.Mutex
	states     map[string]string // hostID → band (bandOK|bandWarning|bandCritical); breached unseeded at init
	customerOf map[string]string // hostID → customerID (event attribution)
}

const (
	defaultHostDiskWarnPercent = 90.0
	defaultHostDiskCritPercent = 95.0
)

// Disk fill bands, ordered by severity (bandRank).
const (
	bandOK       = "ok"
	bandWarning  = "warning"
	bandCritical = "critical"
)

// NewHostDiskChecker creates the checker with the configured thresholds (defaults 90/95 when unset or
// invalid) and seeds state from the latest reports. NO events on init — except that already-breached
// hosts are left UNSEEDED so their first Check emits (born/persistent).
func NewHostDiskChecker(s *store.Store, warnPercent, critPercent float64, onEvent EventNotifyFunc, logger *log.Logger) *HostDiskChecker {
	warn, crit := normalizeDiskThresholds(warnPercent, critPercent)
	dc := &HostDiskChecker{
		store:      s,
		warn:       warn,
		crit:       crit,
		logger:     logger,
		onEvent:    onEvent,
		states:     make(map[string]string),
		customerOf: make(map[string]string),
	}
	rows, err := s.GetHostDiskUsage()
	if err != nil {
		logger.Printf("[WARN] Host disk checker: failed to seed states: %v", err)
		return dc
	}
	var okCount, breachedCount int
	for _, row := range rows {
		if s.IsCustomerBlocked(row.CustomerID) {
			continue
		}
		dc.customerOf[row.HostID] = row.CustomerID
		band := dc.band(row.DiskPercent)
		// F2: seed only OK hosts. A host already over threshold at (re)start is left UNSEEDED so the first
		// Check observes oldBand=="" and emits once — otherwise a box already full when the hub restarts
		// would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit.
		if band != bandOK {
			breachedCount++
			continue
		}
		dc.states[row.HostID] = bandOK
		okCount++
	}
	logger.Printf("[INFO] Host disk checker initialized: warn=%.0f%% crit=%.0f%%, %d ok seeded, %d already-breached left unseeded (first Check emits)", warn, crit, okCount, breachedCount)
	return dc
}

// Check evaluates all hosts and emits on each escalation to a more severe band (ok→warning, ok/warning→
// critical), including the first-observation born-breach. De-escalation and recovery update state
// silently (re-arming a future breach). Call on the same 60s sweep as the staleness checker.
func (dc *HostDiskChecker) Check() {
	rows, err := dc.store.GetHostDiskUsage()
	if err != nil {
		dc.logger.Printf("[WARN] Host disk check failed: %v", err)
		return
	}
	dc.mu.Lock()
	defer dc.mu.Unlock()

	seen := make(map[string]bool, len(rows))
	for _, row := range rows {
		seen[row.HostID] = true
		if dc.store.IsCustomerBlocked(row.CustomerID) {
			delete(dc.states, row.HostID)
			continue
		}
		dc.customerOf[row.HostID] = row.CustomerID

		newBand := dc.band(row.DiskPercent)
		oldBand := dc.states[row.HostID] // "" (rank 0) for an unseen / breached-at-init host
		// Emit only when moving to a MORE severe band — this covers the born-breach (oldBand=="" → rank 0)
		// and any escalation, while de-escalation/recovery just re-arm (no noise, no recovery event).
		if bandRank(newBand) > bandRank(oldBand) {
			dc.emit(row, oldBand, newBand)
		}
		dc.states[row.HostID] = newBand
	}

	for id := range dc.states {
		if !seen[id] {
			delete(dc.states, id)
		}
	}
}

// GetState returns the current band for a host ("unknown" if unseen).
func (dc *HostDiskChecker) GetState(hostID string) string {
	dc.mu.Lock()
	defer dc.mu.Unlock()
	s := dc.states[hostID]
	if s == "" {
		return "unknown"
	}
	return s
}

// band maps a fill percentage to its severity band.
func (dc *HostDiskChecker) band(pct float64) string {
	switch {
	case pct >= dc.crit:
		return bandCritical
	case pct >= dc.warn:
		return bandWarning
	default:
		return bandOK
	}
}

// bandRank orders the bands so an escalation is a strictly increasing rank ("" / unseen = ok = 0).
func bandRank(b string) int {
	switch b {
	case bandCritical:
		return 2
	case bandWarning:
		return 1
	default:
		return 0
	}
}

func (dc *HostDiskChecker) emit(row store.HostDiskRow, oldBand, newBand string) {
	var eventType, severity, message string
	switch newBand {
	case bandCritical:
		eventType = "host_disk_critical"
		// Natural "critical" severity (hub v0.24.0 routes it; the operator email styles it 🔴). Before
		// v0.24.0 the dispatcher silently dropped "critical", so this had to be "error" — now it is honest.
		severity = "critical"
		message = fmt.Sprintf("Host %s: root filesystem CRITICALLY full at %.0f%% (threshold %.0f%%) — PVE/logging/agent writes may start failing; free space (e.g. old vzdump backups) immediately", row.HostID, row.DiskPercent, dc.crit)
	case bandWarning:
		eventType = "host_disk_warning"
		severity = "warning"
		message = fmt.Sprintf("Host %s: root filesystem high at %.0f%% (threshold %.0f%%) — free space (e.g. old backups) before it fills", row.HostID, row.DiskPercent, dc.warn)
	default:
		return // never emit for ok
	}

	details, _ := json.Marshal(map[string]any{
		"host_id":          row.HostID,
		"disk_percent":     row.DiskPercent,
		"disk_total_bytes": row.DiskTotalBytes,
		"disk_used_bytes":  row.DiskUsedBytes,
		"warn_percent":     dc.warn,
		"crit_percent":     dc.crit,
	})

	dc.logger.Printf("[INFO] Host disk: %s root %.0f%% %s→%s (%s)", row.HostID, row.DiskPercent, bandLabel(oldBand), newBand, eventType)

	if _, err := dc.store.SaveEvent(row.CustomerID, eventType, severity, message, string(details), "hub"); err != nil {
		dc.logger.Printf("[WARN] Failed to save host disk event for %s: %v", row.HostID, err)
		return
	}
	if dc.onEvent != nil {
		dc.onEvent(row.CustomerID, eventType, severity, message, string(details), "hub")
	}
}

// bandLabel renders the previous band for the log line ("unknown" for an unseen/breached-at-init host).
func bandLabel(b string) string {
	if b == "" {
		return "unknown"
	}
	return b
}

// normalizeDiskThresholds applies the 90/95 defaults and guards against an invalid/misordered config
// (warn/crit out of (0,100), or crit ≤ warn) by falling back to the defaults — a config typo can never
// silence the alert or invert the bands.
func normalizeDiskThresholds(warn, crit float64) (float64, float64) {
	if warn <= 0 || warn >= 100 {
		warn = defaultHostDiskWarnPercent
	}
	if crit <= 0 || crit >= 100 {
		crit = defaultHostDiskCritPercent
	}
	if crit <= warn {
		return defaultHostDiskWarnPercent, defaultHostDiskCritPercent
	}
	return warn, crit
}