felhom-controller/controller/internal/fillwatch/fillwatch.go

// Package fillwatch warns the CUSTOMER that a filesystem is filling, BEFORE anything fails.
//
// R-167, operator decision D-c (2026-08-02, felhom.eu CONTEXT.md S-5). Nothing warned before this:
// the first sign of a full filesystem was a backup that did not happen. The healthcheck folded disk
// pressure into a generic `health_degraded` at 90%, for REGISTERED STORAGE PATHS ONLY — it never
// looked at the docker area or the system-data area, never reported free bytes, and never named the
// drive.
//
// WHY IT SHIPS BEFORE THE mp1→mp0 MERGE (D-a / R-165), not with it. Today an app that outgrows the
// 20 G backup area is refused per app with its last good recovery unit preserved byte-identical —
// a wall, but one that fails safely, one app at a time. The merge removes that wall so backups share
// the large area. D-a's own condition (2) says the monitoring lands in the same step and never after;
// landing it FIRST is strictly better and costs nothing, so the warnings go in and get proven on real
// hardware while the wall is still standing.
//
// WHY THE FILESYSTEM IS THE UNIT AND NOT THE APP. One full disk holding ten apps would produce ten
// identical warnings, nine of them noise. The customer's action — free space, delete files, add a
// drive — is per filesystem too.
package fillwatch

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
)

// ── Thresholds ───────────────────────────────────────────────────────────────────────────────────
//
// TWO TERMS, AND WHICHEVER TRIPS FIRST WINS, because a percentage alone lies at both ends of the
// size range this fleet actually has: 85% of a 20 G backup area leaves 3 G — not enough for one
// DB-backed app's recovery unit, which is up to ~2× its data (measured: 21.1 GB → 40.2 GB,
// architecture/07-backup-architecture.md §7.5) — while 85% of a 4 TB media drive leaves 600 G and is
// nothing to write home about. A free-bytes floor catches the first; a percentage catches the second.
const (
	// WarnUsedPercent / WarnFreeGiB — enter the warning band.
	WarnUsedPercent = 85.0
	WarnFreeGiB     = 5.0

	// CritUsedPercent / CritFreeGiB — enter the critical band. Below 2 GiB a volume tar of almost any
	// real app fails, so this is "the next backup will not complete", not "it is getting tight".
	CritUsedPercent = 95.0
	CritFreeGiB     = 2.0

	// ClearUsedPercent / ClearFreeGiB — the RETURN threshold, deliberately well below the warn pair.
	// Both must hold. Between clear and warn is a dead zone in which the previous band is HELD, so a
	// filesystem hovering on the line does not flap between warned and cleared. The gap is pinned by
	// TestThresholdsKeepTheirHysteresisGap — a warn and clear threshold that can be edited into
	// equality is a flapping bug waiting to be introduced.
	ClearUsedPercent = 75.0
	ClearFreeGiB     = 7.0
)

// Band is a filesystem's fill state. Ordered: escalation is an increase.
type Band int

const (
	BandOK Band = iota
	BandWarning
	BandCritical
)

func (b Band) String() string {
	switch b {
	case BandWarning:
		return "warning"
	case BandCritical:
		return "critical"
	default:
		return "ok"
	}
}

// EventType maps a band to the hub event type it emits.
//
// These two types already existed in the hub's allowedEventTypes, already carried Hungarian copy,
// already sat in the controller's DefaultEnabledEvents and already had a UI checkbox — and NOTHING
// IN ANY REPO EMITTED THEM. A complete customer pipeline with no producer, the sixth "built but
// never wired" instance in this project. This package is that producer; a new near-duplicate event
// type would have left the pair inert forever.
func (b Band) EventType() string {
	switch b {
	case BandCritical:
		return "disk_critical"
	case BandWarning:
		return "disk_warning"
	default:
		return ""
	}
}

// Severity is the hub severity for a band. `disk_warning` must be "warning" and `disk_critical`
// "critical" — the dispatcher drops "info" entirely (severityNotifies), so getting this wrong stores
// the event and mails nobody.
func (b Band) Severity() string {
	switch b {
	case BandCritical:
		return "critical"
	case BandWarning:
		return "warning"
	default:
		return ""
	}
}

// Usage is one filesystem's occupancy. Deliberately not `system.DiskUsageInfo` — this package must be
// testable without a real filesystem, and the seam is the reason it is.
type Usage struct {
	UsedPercent float64
	AvailGB     float64
	UsedGB      float64
	TotalGB     float64
}

// Target is a watched filesystem: the path to stat, and the name a CUSTOMER will recognise.
type Target struct {
	Path  string
	Label string
}

// Event is one crossing, handed to the notify seam.
type Event struct {
	Target  Target
	Band    Band
	Usage   Usage
	Message string // the Hungarian customer text, already rendered
}

// Watcher holds the persisted per-filesystem band and emits on escalation only.
type Watcher struct {
	statePath string
	logger    *log.Logger

	// targets returns the filesystems to watch, resolved at CHECK time — a drive added or removed
	// between checks must be picked up without a restart.
	targets func() []Target
	// usage reads one filesystem. NIL RESULT MEANS UNREADABLE, NEVER FULL (§8.4).
	usage func(path string) *Usage
	// notify pushes the customer event. Nil-safe.
	notify func(Event)

	mu    sync.Mutex
	bands map[string]Band
}

// New builds a Watcher over a state file. Nothing is read until Check runs.
func New(statePath string, logger *log.Logger, targets func() []Target, usage func(string) *Usage) *Watcher {
	if logger == nil {
		logger = log.Default()
	}
	return &Watcher{
		statePath: statePath,
		logger:    logger,
		targets:   targets,
		usage:     usage,
		bands:     make(map[string]Band),
	}
}

// SetNotify wires the customer event push. INIT-ONLY — call once at startup.
func (w *Watcher) SetNotify(fn func(Event)) { w.notify = fn }

// classify decides the band from a reading AND the previous band, which is what makes the hysteresis
// work: between the clear and warn thresholds the previous band is HELD rather than recomputed.
//
// Pure, so it is unit-testable without a filesystem, a clock or a notifier.
func classify(u Usage, prev Band) Band {
	switch {
	case u.UsedPercent >= CritUsedPercent || u.AvailGB < CritFreeGiB:
		return BandCritical
	case u.UsedPercent >= WarnUsedPercent || u.AvailGB < WarnFreeGiB:
		return BandWarning
	case u.UsedPercent <= ClearUsedPercent && u.AvailGB >= ClearFreeGiB:
		return BandOK
	default:
		// The dead zone. Holding `prev` is the whole hysteresis: a filesystem sitting at 80% neither
		// warns (it is below the warn line) nor clears (it is above the clear line).
		return prev
	}
}

// Check reads every target once and emits for each ESCALATION. Safe to call from the scheduler.
func (w *Watcher) Check() error {
	if w == nil {
		return nil
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	w.loadLocked()

	targets := w.targets()
	seen := make(map[string]bool, len(targets))
	changed := false
	var checked, skipped, emitted int

	for _, t := range targets {
		if t.Path == "" {
			continue
		}
		seen[t.Path] = true
		u := w.usage(t.Path)
		if u == nil {
			// §8.4 — NOT-YET-MOUNTED IS NOT FULL. An absent, unmounted or unreadable filesystem is
			// the drive gate's business and already has its own alert (storage_disconnected /
			// backup_target_absent). Reporting it as "full" would be a false alarm with a misleading
			// cause, and would tell the customer to delete files that are not the problem.
			w.logger.Printf("[DEBUG] [fillwatch] %s: usage unreadable — skipped (an unreadable filesystem is not a full one)", t.Path)
			skipped++
			continue
		}
		checked++
		prev := w.bands[t.Path]
		next := classify(*u, prev)
		if next == prev {
			continue
		}
		w.bands[t.Path] = next
		changed = true

		if next <= prev {
			// De-escalation, including the return to OK, is SILENT. The customer already acted, or
			// the app deleted its own temp files; telling them a resolved problem is resolved is
			// noise. The state change re-arms the warning, which is what Scenario F asks for.
			w.logger.Printf("[INFO] [fillwatch] %s: %s → %s (%.0f%% used, %.1f GB free) — cleared silently, re-armed",
				t.Path, prev, next, u.UsedPercent, u.AvailGB)
			continue
		}

		msg := Message(t, *u, next)
		w.logger.Printf("[WARN] [fillwatch] %s (%q): %s → %s — %.0f%% used, %.1f GB free of %.1f GB; notifying the customer",
			t.Path, t.Label, prev, next, u.UsedPercent, u.AvailGB, u.TotalGB)
		emitted++
		if w.notify != nil {
			w.notify(Event{Target: t, Band: next, Usage: *u, Message: msg})
		}
	}

	// Forget filesystems that no longer exist (a decommissioned drive), so the state file cannot grow
	// without bound and a re-added drive starts from OK rather than from a stale band.
	for path := range w.bands {
		if !seen[path] {
			delete(w.bands, path)
			changed = true
		}
	}

	// A POSITIVE OBSERVABLE, every run, healthy or not (standing rule 3). Without it a quiet run is
	// indistinguishable from a checker that never ran — and because this check is edge-triggered, the
	// HEALTHY steady state IS a quiet run, so the ambiguity is permanent rather than rare. It was hit
	// for real while live-validating v0.191.1 on 9201: an unchanged band produced zero log lines, and
	// proving the checker was alive needed a deliberate threshold crossing.
	w.logger.Printf("[INFO] [fillwatch] checked %d filesystem(s), %d unreadable/skipped, %d notification(s); bands: %s",
		checked, skipped, emitted, w.bandSummaryLocked())

	if !changed {
		return nil
	}
	return w.saveLocked()
}

// bandSummaryLocked renders the current per-path bands for the run summary, deterministically. Caller
// holds w.mu.
func (w *Watcher) bandSummaryLocked() string {
	if len(w.bands) == 0 {
		return "all ok"
	}
	paths := make([]string, 0, len(w.bands))
	for p := range w.bands {
		paths = append(paths, p)
	}
	sort.Strings(paths)
	parts := make([]string, 0, len(paths))
	for _, p := range paths {
		parts = append(parts, p+"="+w.bands[p].String())
	}
	return strings.Join(parts, " ")
}

// Bands returns a copy of the current per-path state (diagnostics + tests).
func (w *Watcher) Bands() map[string]Band {
	w.mu.Lock()
	defer w.mu.Unlock()
	out := make(map[string]Band, len(w.bands))
	for k, v := range w.bands {
		out[k] = v
	}
	return out
}

// ── State persistence ────────────────────────────────────────────────────────────────────────────
//
// Persisted so a controller restart does not re-warn about a filesystem the customer has already
// been told about — the offboxEnlargeBlockedNotify discipline. LOSING IT IS THE ACCEPTABLE
// DIRECTION: at most one extra warning, never a missed one. THE HUB OWNS COOLDOWN; this package must
// never add a timer of its own.

type persisted struct {
	Bands map[string]string `json:"bands"`
}

func (w *Watcher) loadLocked() {
	if w.statePath == "" {
		return
	}
	data, err := os.ReadFile(w.statePath)
	if err != nil {
		if !os.IsNotExist(err) {
			w.logger.Printf("[WARN] [fillwatch] could not read %s: %v — starting from OK (at most one extra warning)", w.statePath, err)
		}
		return
	}
	var p persisted
	if err := json.Unmarshal(data, &p); err != nil {
		w.logger.Printf("[WARN] [fillwatch] %s is corrupt: %v — starting from OK (at most one extra warning)", w.statePath, err)
		return
	}
	for path, band := range p.Bands {
		switch band {
		case "critical":
			w.bands[path] = BandCritical
		case "warning":
			w.bands[path] = BandWarning
		}
	}
}

func (w *Watcher) saveLocked() error {
	if w.statePath == "" {
		return nil
	}
	p := persisted{Bands: make(map[string]string, len(w.bands))}
	for path, band := range w.bands {
		if band == BandOK {
			continue // OK is the default — do not persist it
		}
		p.Bands[path] = band.String()
	}
	data, err := json.MarshalIndent(p, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(w.statePath), 0o755); err != nil {
		return err
	}
	tmp := w.statePath + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, w.statePath)
}

// ── Customer copy (Hungarian) ────────────────────────────────────────────────────────────────────

// Message renders the customer-facing Hungarian warning.
//
// IT MUST SAY WHAT TO DO, not merely that something is happening — a warning a customer cannot act
// on is an operator alert wearing the wrong clothes. It names the storage by its LABEL (the customer
// named it; a path means nothing to them) and gives the free space in GB.
//
// It is sent as the event's MESSAGE, and the hub deliberately has no `customerMessages` entry for
// these two types: `FormatCustomerEmail` PREFERS the entry over the message, so a static template
// would discard the label and the figures — exactly why `offbox_enlarge_blocked` and
// `disk_health_degraded` have none either.
func Message(t Target, u Usage, band Band) string {
	name := t.Label
	if name == "" {
		name = t.Path
	}
	if band == BandCritical {
		return fmt.Sprintf(
			"A(z) „%s” tároló kritikusan megtelt: %s szabad hely maradt (%s foglalt). "+
				"A biztonsági mentések és az alkalmazások írásai bármikor meghiúsulhatnak. "+
				"Kérjük, mielőbb szabadíts fel helyet: törölj felesleges fájlokat, vagy csatlakoztass új meghajtót.",
			name, hunGB(u.AvailGB), hunPercent(u.UsedPercent))
	}
	return fmt.Sprintf(
		"A(z) „%s” tároló %s foglalt — %s szabad hely maradt. "+
			"Kérjük, szabadíts fel helyet, mielőtt megtelik: törölj felesleges fájlokat, vagy csatlakoztass új meghajtót. "+
			"Ha megtelik, a biztonsági mentések meghiúsulnak.",
		name, hunPercent(u.UsedPercent), hunGB(u.AvailGB))
}

// hunGB formats a GB figure the Hungarian way — decimal COMMA, one decimal place. A "4.2 GB" in a
// Hungarian sentence reads as a typo to the customer.
func hunGB(gb float64) string {
	if gb >= 1024 {
		return strings.Replace(fmt.Sprintf("%.1f TB", gb/1024), ".", ",", 1)
	}
	return strings.Replace(fmt.Sprintf("%.1f GB", gb), ".", ",", 1)
}

func hunPercent(p float64) string { return fmt.Sprintf("%.0f%%", p) }

// SortTargets gives Check a deterministic order, so a multi-filesystem crossing produces its events
// in a stable sequence (tests, and a readable log).
func SortTargets(ts []Target) []Target {
	sort.Slice(ts, func(i, j int) bool { return ts[i].Path < ts[j].Path })
	return ts
}