felhom-controller/controller/internal/fillwatch/fillwatch.go

// Package fillwatch warns the CUSTOMER that a filesystem is filling, BEFORE anything fails.
//
// R-167, operator decision D-c (2026-08-02, felhom.eu CONTEXT.md S-5). Nothing warned before this:
// the first sign of a full filesystem was a backup that did not happen. The healthcheck folded disk
// pressure into a generic `health_degraded` at 90%, for REGISTERED STORAGE PATHS ONLY — it never
// looked at the docker area or the system-data area, never reported free bytes, and never named the
// drive.
//
// WHY IT SHIPS BEFORE THE mp1→mp0 MERGE (D-a / R-165), not with it. Today an app that outgrows the
// 20 G backup area is refused per app with its last good recovery unit preserved byte-identical —
// a wall, but one that fails safely, one app at a time. The merge removes that wall so backups share
// the large area. D-a's own condition (2) says the monitoring lands in the same step and never after;
// landing it FIRST is strictly better and costs nothing, so the warnings go in and get proven on real
// hardware while the wall is still standing.
//
// WHY THE FILESYSTEM IS THE UNIT AND NOT THE APP. One full disk holding ten apps would produce ten
// identical warnings, nine of them noise. The customer's action — free space, delete files, add a
// drive — is per filesystem too.
package fillwatch

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"gitea.dooplex.hu/admin/felhom-controller/internal/i18n"
)

// ── Thresholds ───────────────────────────────────────────────────────────────────────────────────
//
// TWO TERMS, AND WHICHEVER TRIPS FIRST WINS, because a percentage alone lies at both ends of the
// size range this fleet actually has: 85% of a 20 G backup area leaves 3 G — not enough for one
// DB-backed app's recovery unit, which is up to ~2× its data (measured: 21.1 GB → 40.2 GB,
// architecture/07-backup-architecture.md §7.5) — while 85% of a 4 TB media drive leaves 600 G and is
// nothing to write home about. A free-bytes floor catches the first; a percentage catches the second.
const (
	// WarnUsedPercent / WarnFreeGiB — enter the warning band.
	WarnUsedPercent = 85.0
	WarnFreeGiB     = 5.0

	// CritUsedPercent / CritFreeGiB — enter the critical band. Below 2 GiB a volume tar of almost any
	// real app fails, so this is "the next backup will not complete", not "it is getting tight".
	CritUsedPercent = 95.0
	CritFreeGiB     = 2.0

	// ClearUsedPercent / ClearFreeGiB — the RETURN threshold, deliberately well below the warn pair.
	// Both must hold. Between clear and warn is a dead zone in which the previous band is HELD, so a
	// filesystem hovering on the line does not flap between warned and cleared. The gap is pinned by
	// TestThresholdsKeepTheirHysteresisGap — a warn and clear threshold that can be edited into
	// equality is a flapping bug waiting to be introduced.
	ClearUsedPercent = 75.0
	ClearFreeGiB     = 7.0
)

// Band is a filesystem's fill state. Ordered: escalation is an increase.
type Band int

const (
	BandOK Band = iota
	BandWarning
	BandCritical
)

func (b Band) String() string {
	switch b {
	case BandWarning:
		return "warning"
	case BandCritical:
		return "critical"
	default:
		return "ok"
	}
}

// EventType maps a band to the hub event type it emits.
//
// These two types already existed in the hub's allowedEventTypes, already carried Hungarian copy,
// already sat in the controller's DefaultEnabledEvents and already had a UI checkbox — and NOTHING
// IN ANY REPO EMITTED THEM. A complete customer pipeline with no producer, the sixth "built but
// never wired" instance in this project. This package is that producer; a new near-duplicate event
// type would have left the pair inert forever.
func (b Band) EventType() string {
	switch b {
	case BandCritical:
		return "disk_critical"
	case BandWarning:
		return "disk_warning"
	default:
		return ""
	}
}

// Severity is the hub severity for a band. `disk_warning` must be "warning" and `disk_critical`
// "critical" — the dispatcher drops "info" entirely (severityNotifies), so getting this wrong stores
// the event and mails nobody.
func (b Band) Severity() string {
	switch b {
	case BandCritical:
		return "critical"
	case BandWarning:
		return "warning"
	default:
		return ""
	}
}

// Usage is one filesystem's occupancy. Deliberately not `system.DiskUsageInfo` — this package must be
// testable without a real filesystem, and the seam is the reason it is.
type Usage struct {
	UsedPercent float64
	AvailGB     float64
	UsedGB      float64
	TotalGB     float64
}

// Target is a watched filesystem: the path to stat, and the name a CUSTOMER will recognise.
type Target struct {
	Path  string
	Label string
	// LabelKey, when set, names the target in the household's language from the bundle (the two
	// built-in areas: „Alkalmazások területe", „Rendszer- és mentési terület"). Label stays the
	// Hungarian name for the log and the event details. A household's own drive label has no key —
	// the household named it, in whatever language it wished.
	LabelKey string
}

// Event is one crossing, handed to the notify seam.
type Event struct {
	Target  Target
	Band    Band
	Usage   Usage
	Message string // the Hungarian customer text, already rendered (the wire's `message`)
	// MessageHousehold is the same sentence in the household's language (the wire's
	// `message_customer`, R-558); "" for a Hungarian household, which sends nothing extra.
	MessageHousehold string
}

// Watcher holds the persisted per-filesystem band and emits on escalation only.
type Watcher struct {
	statePath string
	logger    *log.Logger

	// targets returns the filesystems to watch, resolved at CHECK time — a drive added or removed
	// between checks must be picked up without a restart.
	targets func() []Target
	// usage reads one filesystem. NIL RESULT MEANS UNREADABLE, NEVER FULL (§8.4).
	usage func(path string) *Usage
	// notify pushes the customer event. Nil-safe.
	notify func(Event)
	// extra appends a sentence to the warning for one target, in a language — the kept folders on it
	// the household can delete (`09` §3 decision 36). Nil-safe; "" appends nothing.
	extra func(t Target, lang string) string
	// lang returns the household's language (R-521 F7: the warning reaches an English household in
	// English). Nil-safe: nil means Hungarian only.
	lang func() string

	mu    sync.Mutex
	bands map[string]Band
}

// New builds a Watcher over a state file. Nothing is read until Check runs.
func New(statePath string, logger *log.Logger, targets func() []Target, usage func(string) *Usage) *Watcher {
	if logger == nil {
		logger = log.Default()
	}
	return &Watcher{
		statePath: statePath,
		logger:    logger,
		targets:   targets,
		usage:     usage,
		bands:     make(map[string]Band),
	}
}

// SetNotify wires the customer event push. INIT-ONLY — call once at startup.
func (w *Watcher) SetNotify(fn func(Event)) { w.notify = fn }

// SetExtra wires the kept-data sentence (main.go). INIT-ONLY.
func (w *Watcher) SetExtra(fn func(t Target, lang string) string) { w.extra = fn }

// SetLang wires the household's language (main.go: settings.GetLanguage). INIT-ONLY.
func (w *Watcher) SetLang(fn func() string) { w.lang = fn }

// classify decides the band from a reading AND the previous band, which is what makes the hysteresis
// work: between the clear and warn thresholds the previous band is HELD rather than recomputed.
//
// Pure, so it is unit-testable without a filesystem, a clock or a notifier.
func classify(u Usage, prev Band) Band {
	switch {
	case u.UsedPercent >= CritUsedPercent || u.AvailGB < CritFreeGiB:
		return BandCritical
	case u.UsedPercent >= WarnUsedPercent || u.AvailGB < WarnFreeGiB:
		return BandWarning
	case u.UsedPercent <= ClearUsedPercent && u.AvailGB >= ClearFreeGiB:
		return BandOK
	default:
		// The dead zone. Holding `prev` is the whole hysteresis: a filesystem sitting at 80% neither
		// warns (it is below the warn line) nor clears (it is above the clear line).
		return prev
	}
}

// Check reads every target once and emits for each ESCALATION. Safe to call from the scheduler.
func (w *Watcher) Check() error {
	if w == nil {
		return nil
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	w.loadLocked()

	targets := w.targets()
	seen := make(map[string]bool, len(targets))
	changed := false
	var checked, skipped, emitted int

	for _, t := range targets {
		if t.Path == "" {
			continue
		}
		seen[t.Path] = true
		u := w.usage(t.Path)
		if u == nil {
			// §8.4 — NOT-YET-MOUNTED IS NOT FULL. An absent, unmounted or unreadable filesystem is
			// the drive gate's business and already has its own alert (storage_disconnected /
			// backup_target_absent). Reporting it as "full" would be a false alarm with a misleading
			// cause, and would tell the customer to delete files that are not the problem.
			w.logger.Printf("[DEBUG] [fillwatch] %s: usage unreadable — skipped (an unreadable filesystem is not a full one)", t.Path)
			skipped++
			continue
		}
		checked++
		prev := w.bands[t.Path]
		next := classify(*u, prev)
		if next == prev {
			continue
		}
		w.bands[t.Path] = next
		changed = true

		if next <= prev {
			// De-escalation, including the return to OK, is SILENT. The customer already acted, or
			// the app deleted its own temp files; telling them a resolved problem is resolved is
			// noise. The state change re-arms the warning, which is what Scenario F asks for.
			w.logger.Printf("[INFO] [fillwatch] %s: %s → %s (%.0f%% used, %.1f GB free) — cleared silently, re-armed",
				t.Path, prev, next, u.UsedPercent, u.AvailGB)
			continue
		}

		msg := w.messageWithExtra(i18n.Default, t, *u, next)
		household := ""
		if w.lang != nil {
			if lang := i18n.Normalize(w.lang()); lang != i18n.Default {
				household = w.messageWithExtra(lang, t, *u, next)
			}
		}
		w.logger.Printf("[WARN] [fillwatch] %s (%q): %s → %s — %.0f%% used, %.1f GB free of %.1f GB; notifying the customer",
			t.Path, t.Label, prev, next, u.UsedPercent, u.AvailGB, u.TotalGB)
		emitted++
		if w.notify != nil {
			w.notify(Event{Target: t, Band: next, Usage: *u, Message: msg, MessageHousehold: household})
		}
	}

	// Forget filesystems that no longer exist (a decommissioned drive), so the state file cannot grow
	// without bound and a re-added drive starts from OK rather than from a stale band.
	for path := range w.bands {
		if !seen[path] {
			delete(w.bands, path)
			changed = true
		}
	}

	// A POSITIVE OBSERVABLE, every run, healthy or not (standing rule 3). Without it a quiet run is
	// indistinguishable from a checker that never ran — and because this check is edge-triggered, the
	// HEALTHY steady state IS a quiet run, so the ambiguity is permanent rather than rare. It was hit
	// for real while live-validating v0.191.1 on 9201: an unchanged band produced zero log lines, and
	// proving the checker was alive needed a deliberate threshold crossing.
	w.logger.Printf("[INFO] [fillwatch] checked %d filesystem(s), %d unreadable/skipped, %d notification(s); bands: %s",
		checked, skipped, emitted, w.bandSummaryLocked())

	if !changed {
		return nil
	}
	return w.saveLocked()
}

// messageWithExtra is the warning in one language, with the kept-data sentence when there is one.
func (w *Watcher) messageWithExtra(lang string, t Target, u Usage, band Band) string {
	msg := MessageLang(lang, t, u, band)
	if w.extra != nil {
		if x := w.extra(t, lang); x != "" {
			msg += " " + x
		}
	}
	return msg
}

// bandSummaryLocked renders the current per-path bands for the run summary, deterministically. Caller
// holds w.mu.
func (w *Watcher) bandSummaryLocked() string {
	if len(w.bands) == 0 {
		return "all ok"
	}
	paths := make([]string, 0, len(w.bands))
	for p := range w.bands {
		paths = append(paths, p)
	}
	sort.Strings(paths)
	parts := make([]string, 0, len(paths))
	for _, p := range paths {
		parts = append(parts, p+"="+w.bands[p].String())
	}
	return strings.Join(parts, " ")
}

// Bands returns a copy of the current per-path state (diagnostics + tests).
func (w *Watcher) Bands() map[string]Band {
	w.mu.Lock()
	defer w.mu.Unlock()
	out := make(map[string]Band, len(w.bands))
	for k, v := range w.bands {
		out[k] = v
	}
	return out
}

// ── State persistence ────────────────────────────────────────────────────────────────────────────
//
// Persisted so a controller restart does not re-warn about a filesystem the customer has already
// been told about — the offboxEnlargeBlockedNotify discipline. LOSING IT IS THE ACCEPTABLE
// DIRECTION: at most one extra warning, never a missed one. THE HUB OWNS COOLDOWN; this package must
// never add a timer of its own.

type persisted struct {
	Bands map[string]string `json:"bands"`
}

func (w *Watcher) loadLocked() {
	if w.statePath == "" {
		return
	}
	data, err := os.ReadFile(w.statePath)
	if err != nil {
		if !os.IsNotExist(err) {
			w.logger.Printf("[WARN] [fillwatch] could not read %s: %v — starting from OK (at most one extra warning)", w.statePath, err)
		}
		return
	}
	var p persisted
	if err := json.Unmarshal(data, &p); err != nil {
		w.logger.Printf("[WARN] [fillwatch] %s is corrupt: %v — starting from OK (at most one extra warning)", w.statePath, err)
		return
	}
	for path, band := range p.Bands {
		switch band {
		case "critical":
			w.bands[path] = BandCritical
		case "warning":
			w.bands[path] = BandWarning
		}
	}
}

func (w *Watcher) saveLocked() error {
	if w.statePath == "" {
		return nil
	}
	p := persisted{Bands: make(map[string]string, len(w.bands))}
	for path, band := range w.bands {
		if band == BandOK {
			continue // OK is the default — do not persist it
		}
		p.Bands[path] = band.String()
	}
	data, err := json.MarshalIndent(p, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(w.statePath), 0o755); err != nil {
		return err
	}
	tmp := w.statePath + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, w.statePath)
}

// ── Customer copy (Hungarian) ────────────────────────────────────────────────────────────────────

// Message renders the customer-facing Hungarian warning.
//
// IT MUST SAY WHAT TO DO, not merely that something is happening — a warning a customer cannot act
// on is an operator alert wearing the wrong clothes. It names the storage by its LABEL (the customer
// named it; a path means nothing to them) and gives the free space in GB.
//
// It is sent as the event's MESSAGE, and the hub deliberately has no `customerMessages` entry for
// these two types: `FormatCustomerEmail` PREFERS the entry over the message, so a static template
// would discard the label and the figures — exactly why `offbox_enlarge_blocked` and
// `disk_health_degraded` have none either.
func Message(t Target, u Usage, band Band) string { return MessageLang(i18n.Default, t, u, band) }

// MessageLang is Message in a language, from the bundle keys `event.fill.critical` /
// `event.fill.warning` (R-521 F7). The Hungarian is the literal these keys replaced minus its formal
// „Kérjük," opening (the R-516 te-form gate); TestMessageHungarianWireText pins the exact bytes.
func MessageLang(lang string, t Target, u Usage, band Band) string {
	b, err := i18n.Shared()
	if err != nil {
		// The bundle is embedded: this is a broken build. Say it in the log and send the figures bare
		// rather than nothing — an event with a visible fault reaches the operator and gets fixed.
		return fmt.Sprintf("%s: %s, %s (%v)", t.Path, gbIn(lang, u.AvailGB), percent(u.UsedPercent), err)
	}
	name := t.Label
	if t.LabelKey != "" && b.Has(i18n.Normalize(lang), t.LabelKey) {
		name = b.Msg(lang, t.LabelKey)
	}
	if name == "" {
		name = t.Path
	}
	if band == BandCritical {
		return b.Msgf(lang, "event.fill.critical", name, gbIn(lang, u.AvailGB), percent(u.UsedPercent))
	}
	return b.Msgf(lang, "event.fill.warning", name, percent(u.UsedPercent), gbIn(lang, u.AvailGB))
}

// gbIn formats a GB figure with lang's decimal separator, one decimal place — a "4.2 GB" in a
// Hungarian sentence reads as a typo to the customer, and "4,2 GB" in an English one.
func gbIn(lang string, gb float64) string {
	if gb >= 1024 {
		return i18n.FormatDecimal(lang, gb/1024, 1) + " TB"
	}
	return i18n.FormatDecimal(lang, gb, 1) + " GB"
}

func percent(p float64) string { return fmt.Sprintf("%.0f%%", p) }

// SortTargets gives Check a deterministic order, so a multi-filesystem crossing produces its events
// in a stable sequence (tests, and a readable log).
func SortTargets(ts []Target) []Target {
	sort.Slice(ts, func(i, j int) bool { return ts[i].Path < ts[j].Path })
	return ts
}