Files
felhom.eu/hub/internal/monitor/storage_fill_test.go
T
admin 88073ac464 hub v0.25.0: per-storage worst-fill alerting (StorageFillChecker)
Generalizes host_disk to any reported storage target (dump/backup volume, data drive,
thin pool, PBS). Per-(host,target) state, born/persistent, natural critical severity,
distinct storage_fill_* events; excludes the root-backed builtin (host_disk owns root).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs
2026-06-30 19:01:20 +02:00

148 lines
6.5 KiB
Go

package monitor
import (
"fmt"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// stTarget is a storage-target fixture for saveStorageReport.
type stTarget struct {
name, typ, mount string
pct float64
}
// saveStorageReport records a host-report whose report_json carries the given storage_targets[].
func saveStorageReport(t *testing.T, st *store.Store, targets ...stTarget) {
t.Helper()
var parts []string
for _, tg := range targets {
parts = append(parts, fmt.Sprintf(
`{"name":%q,"type":%q,"mount_path":%q,"used_fraction":%g,"total_bytes":1000000000,"used_bytes":%d}`,
tg.name, tg.typ, tg.mount, tg.pct/100, int64(tg.pct*1e7)))
}
body := `{"host_id":"h1","storage_targets":[` + strings.Join(parts, ",") + `]}`
if err := st.SaveHostReport("h1", "c1", []byte(body), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
}
// TestStorageFill_PerTargetBands: state is keyed per (host, target). A breaches → only A alerts; B silent.
// A escalates warning→critical; A recovers → clears + re-arms; B never fires (§7-A/C).
func TestStorageFill_PerTargetBands(t *testing.T) {
st := newDiskStore(t)
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 50}, stTarget{"datadrive", "usb", "/mnt/data", 50})
var events, types []string
onEvent := func(_, et, sev, _, _, _ string) { events = append(events, sev); types = append(types, et) }
fc := NewStorageFillChecker(st, 90, 95, onEvent, quietLog())
// A → 92 warning; B stays 50.
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 92}, stTarget{"datadrive", "usb", "/mnt/data", 50})
fc.Check()
if fc.GetState("h1", "dumpvol") != bandWarning || fc.GetState("h1", "datadrive") != bandOK {
t.Fatalf("states: dumpvol=%s datadrive=%s", fc.GetState("h1", "dumpvol"), fc.GetState("h1", "datadrive"))
}
if len(types) != 1 || types[0] != "storage_fill_warning" {
t.Fatalf("want one storage_fill_warning, got %v", types)
}
// A → 96 critical (escalation); B still 50 (silent).
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 96}, stTarget{"datadrive", "usb", "/mnt/data", 50})
fc.Check()
if len(types) != 2 || types[1] != "storage_fill_critical" || events[1] != "critical" {
t.Fatalf("want storage_fill_critical (severity critical), got types=%v sev=%v", types, events)
}
// A recovers → clears + re-arms; no event.
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 40}, stTarget{"datadrive", "usb", "/mnt/data", 50})
fc.Check()
if fc.GetState("h1", "dumpvol") != bandOK || len(types) != 2 {
t.Fatalf("recovery should clear without event; state=%s events=%v", fc.GetState("h1", "dumpvol"), types)
}
// Re-armed: a fresh breach alerts again.
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 93}, stTarget{"datadrive", "usb", "/mnt/data", 50})
fc.Check()
if len(types) != 3 || types[2] != "storage_fill_warning" {
t.Fatalf("re-arm: a new breach must alert again, got %v", types)
}
}
// TestStorageFill_BornPersistent (F2): a target already over critical at init emits on the FIRST Check.
// Companion: a seed-all (transition-only) model — emulated by pre-seeding the key to critical — stays silent.
func TestStorageFill_BornPersistent(t *testing.T) {
st := newDiskStore(t)
saveStorageReport(t, st, stTarget{"dumpvol", "local-dir", "/mnt/backup", 97})
var types, sev []string
fc := NewStorageFillChecker(st, 90, 95, func(_, et, s, _, _, _ string) { types = append(types, et); sev = append(sev, s) }, quietLog())
if fc.GetState("h1", "dumpvol") != "unknown" {
t.Fatalf("an already-breached target must be left unseeded, got %s", fc.GetState("h1", "dumpvol"))
}
fc.Check()
if len(types) != 1 || types[0] != "storage_fill_critical" || sev[0] != "critical" {
t.Fatalf("born-persistent: first Check must emit storage_fill_critical/critical, got %v %v", types, sev)
}
// COMPANION: a seed-all impl seeds the breached key at init → first Check sees no transition → SILENT.
st2 := newDiskStore(t)
saveStorageReport(t, st2, stTarget{"dumpvol", "local-dir", "/mnt/backup", 97})
var silent []string
fc2 := NewStorageFillChecker(st2, 90, 95, func(_, et, _, _, _, _ string) { silent = append(silent, et) }, quietLog())
fc2.mu.Lock()
fc2.states[fillKey("h1", "dumpvol")] = bandCritical // the WRONG seed-all design's seed
fc2.mu.Unlock()
fc2.Check()
if len(silent) != 0 {
t.Fatalf("control: a seed-all model should be silent on the born-breach, got %v", silent)
}
}
// TestStorageFill_RootExcluded (§7-D): a root-backed target at 96% must NOT alert here (HostDiskChecker
// owns root). Companion: prove the exclusion is what suppresses it (without it the band would be critical).
func TestStorageFill_RootExcluded(t *testing.T) {
st := newDiskStore(t)
saveStorageReport(t, st,
stTarget{"local", "local", "", 96}, // PVE builtin local = root-backed → excluded by Type
stTarget{"rootmnt", "local-dir", "/", 96}, // a target literally at "/" → excluded by mount path
)
var types []string
fc := NewStorageFillChecker(st, 90, 95, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog())
fc.Check()
if len(types) != 0 {
t.Fatalf("root-backed targets must NOT alert here (host_disk owns root), got %v", types)
}
// COMPANION: the exclusion is load-bearing — without it both targets are in the critical band.
if !excludeFromStorageFill("local", "") || !excludeFromStorageFill("local-dir", "/") {
t.Fatal("root-backed targets must be excluded")
}
if excludeFromStorageFill("local-dir", "/mnt/backup") {
t.Fatal("a real off-root storage must NOT be excluded")
}
if bandForPercent(96, 90, 95) != bandCritical {
t.Fatal("control: 96%% IS the critical band — only the exclusion suppresses the root alert")
}
}
// TestStorageFill_ParseAndExcludeFromStore checks GetHostStorageTargets parses names/percent + the checker
// keys per target end-to-end (one report → two off-root targets tracked).
func TestStorageFill_ParseFromStore(t *testing.T) {
st := newDiskStore(t)
saveStorageReport(t, st, stTarget{"backupdrive", "usb", "/mnt/bk", 80}, stTarget{"local", "local", "", 99})
rows, err := st.GetHostStorageTargets()
if err != nil {
t.Fatal(err)
}
got := map[string]float64{}
for _, r := range rows {
got[r.Name] = r.Percent
}
if v := got["backupdrive"]; v < 79.9 || v > 80.1 {
t.Fatalf("backupdrive percent = %v, want ~80", v)
}
if _, ok := got["local"]; !ok {
t.Fatal("parse should include all targets (the checker excludes root, not the store)")
}
}