admin
9a3c4855d7
gates / gates (push) Successful in 9s
Earned during v0.191.1's own live validation. After the customer had been warned, a restart produced ZERO fillwatch lines — equally consistent with 'ran and chose silence' and 'never ran'. Proving the checker was alive needed a deliberate crossing into the critical band. For an edge-triggered check the quiet run IS the healthy steady state, so that ambiguity is permanent rather than rare. Check now logs a per-RUN summary on every run, counting unreadable separately from healthy so a drive that has quietly gone unreadable cannot read as 'all fine'.
374 lines
14 KiB
Go
374 lines
14 KiB
Go
package fillwatch
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// R-167 / decision D-c, customer half — the customer is warned BEFORE a fill, once.
|
|
|
|
type harness struct {
|
|
w *Watcher
|
|
events []Event
|
|
usage map[string]*Usage
|
|
target []Target
|
|
}
|
|
|
|
func newHarness(t *testing.T, targets ...Target) *harness {
|
|
t.Helper()
|
|
h := &harness{usage: map[string]*Usage{}, target: targets}
|
|
h.w = New(filepath.Join(t.TempDir(), "fillwatch.json"), log.New(io.Discard, "", 0),
|
|
func() []Target { return h.target },
|
|
func(p string) *Usage { return h.usage[p] })
|
|
h.w.SetNotify(func(e Event) { h.events = append(h.events, e) })
|
|
return h
|
|
}
|
|
|
|
func (h *harness) set(path string, usedPct, availGB float64) {
|
|
h.usage[path] = &Usage{UsedPercent: usedPct, AvailGB: availGB, TotalGB: 100, UsedGB: usedPct}
|
|
}
|
|
|
|
func (h *harness) check(t *testing.T) {
|
|
t.Helper()
|
|
if err := h.w.Check(); err != nil {
|
|
t.Fatalf("Check: %v", err)
|
|
}
|
|
}
|
|
|
|
var photos = Target{Path: "/mnt/felhom-drives/hdd_1", Label: "Fotók"}
|
|
|
|
// --- Scenario E — warned once, and the second pass is silent -------------------------------------
|
|
|
|
func TestWarnsOnceThenIsSilent(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
h.set(photos.Path, 87, 4.2)
|
|
|
|
h.check(t)
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d events on the first crossing, want 1 — the customer was not warned before "+
|
|
"the fill, which is the whole customer half of D-c", len(h.events))
|
|
}
|
|
e := h.events[0]
|
|
if e.Band != BandWarning {
|
|
t.Fatalf("band = %v, want warning", e.Band)
|
|
}
|
|
if e.Band.EventType() != "disk_warning" {
|
|
t.Fatalf("event type = %q, want disk_warning", e.Band.EventType())
|
|
}
|
|
if e.Band.Severity() != "warning" {
|
|
t.Fatalf("severity = %q, want warning — the hub's severityNotifies DROPS \"info\", so a "+
|
|
"wrong severity stores the event and mails nobody", e.Band.Severity())
|
|
}
|
|
|
|
// SECOND PASS, nothing changed. Edge-triggered means exactly nothing fires.
|
|
h.check(t)
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d events after an unchanged second pass, want still 1 — the warning is not "+
|
|
"edge-triggered, so a daily schedule would re-warn the customer every single night",
|
|
len(h.events))
|
|
}
|
|
}
|
|
|
|
// The state must survive a restart — a fresh Watcher over the SAME file must not re-warn.
|
|
func TestEdgeStateSurvivesARestart(t *testing.T) {
|
|
dir := t.TempDir()
|
|
state := filepath.Join(dir, "fillwatch.json")
|
|
usage := map[string]*Usage{photos.Path: {UsedPercent: 87, AvailGB: 4.2, TotalGB: 100}}
|
|
targets := func() []Target { return []Target{photos} }
|
|
read := func(p string) *Usage { return usage[p] }
|
|
|
|
var first []Event
|
|
w1 := New(state, log.New(io.Discard, "", 0), targets, read)
|
|
w1.SetNotify(func(e Event) { first = append(first, e) })
|
|
if err := w1.Check(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(first) != 1 {
|
|
t.Fatalf("first controller: %d events, want 1", len(first))
|
|
}
|
|
|
|
// <restart> — a brand-new Watcher, same file.
|
|
var second []Event
|
|
w2 := New(state, log.New(io.Discard, "", 0), targets, read)
|
|
w2.SetNotify(func(e Event) { second = append(second, e) })
|
|
if err := w2.Check(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(second) != 0 {
|
|
t.Fatalf("a restarted controller re-warned about an already-warned filesystem (%d events) — "+
|
|
"the state is not persisted, so every restart nags the customer", len(second))
|
|
}
|
|
}
|
|
|
|
// --- Scenario F — it clears, and it can fire again ------------------------------------------------
|
|
|
|
func TestClearsSilentlyThenReArms(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("warn: got %d events, want 1", len(h.events))
|
|
}
|
|
|
|
// Drop below the CLEAR thresholds (both must hold): 70% used, 12 GB free.
|
|
h.set(photos.Path, 70, 12)
|
|
h.check(t)
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("clearing fired an event (%d total) — a resolved problem must clear SILENTLY", len(h.events))
|
|
}
|
|
if b := h.w.Bands()[photos.Path]; b != BandOK {
|
|
t.Fatalf("band after clearing = %v, want ok — it is latched, and the customer can never be "+
|
|
"warned about this filesystem again", b)
|
|
}
|
|
|
|
// Cross again — a NEW crossing must warn again.
|
|
h.set(photos.Path, 88, 3.9)
|
|
h.check(t)
|
|
if len(h.events) != 2 {
|
|
t.Fatalf("a NEW crossing after a clear produced %d events total, want 2 — the warning is "+
|
|
"latched forever after one firing", len(h.events))
|
|
}
|
|
}
|
|
|
|
// The dead zone is the hysteresis. A filesystem that falls back to 80% — below warn, above clear —
|
|
// must NOT clear, or it flaps warned/cleared/warned as it wobbles across one line.
|
|
func TestDeadZoneHoldsThePreviousBand(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
|
|
h.set(photos.Path, 80, 6) // between clear (75 / 7 GB) and warn (85 / 5 GB)
|
|
h.check(t)
|
|
if b := h.w.Bands()[photos.Path]; b != BandWarning {
|
|
t.Fatalf("band in the dead zone = %v, want warning held — without hysteresis a filesystem "+
|
|
"hovering on the line flaps between warned and cleared", b)
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("the dead zone produced an extra event (%d total)", len(h.events))
|
|
}
|
|
}
|
|
|
|
// Escalation warning → critical MUST fire: it is a different message and a different urgency.
|
|
func TestEscalationToCriticalFires(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
|
|
h.set(photos.Path, 96, 1.4)
|
|
h.check(t)
|
|
if len(h.events) != 2 {
|
|
t.Fatalf("got %d events, want 2 — an escalation from warning to critical went unreported", len(h.events))
|
|
}
|
|
e := h.events[1]
|
|
if e.Band != BandCritical || e.Band.EventType() != "disk_critical" {
|
|
t.Fatalf("second event = %v/%s, want critical/disk_critical", e.Band, e.Band.EventType())
|
|
}
|
|
// De-escalating critical → warning must be silent (it is still bad; do not celebrate).
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
if len(h.events) != 2 {
|
|
t.Fatalf("a critical→warning de-escalation fired (%d total) — only escalation notifies", len(h.events))
|
|
}
|
|
}
|
|
|
|
// --- Scenario I / §8.4 — a nil usage read is never a warning -------------------------------------
|
|
|
|
func TestUnreadableFilesystemNeverWarns(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
// No entry in h.usage → the seam returns nil, which is what system.GetDiskUsage does on error.
|
|
h.check(t)
|
|
|
|
if len(h.events) != 0 {
|
|
t.Fatalf("an UNREADABLE filesystem produced %d warning(s) — an absent, unmounted or "+
|
|
"unreadable drive is the drive gate's business and has its own alert; reporting it as "+
|
|
"\"full\" is a false alarm with a misleading cause, and tells the customer to delete "+
|
|
"files that are not the problem (§8.4)", len(h.events))
|
|
}
|
|
if b, ok := h.w.Bands()[photos.Path]; ok && b != BandOK {
|
|
t.Fatalf("an unreadable filesystem was recorded as %v", b)
|
|
}
|
|
}
|
|
|
|
// An unreadable filesystem must not CLEAR an existing warning either — that would silently retract a
|
|
// true alarm the moment a drive blipped.
|
|
func TestUnreadableDoesNotClearAnExistingWarning(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
|
|
delete(h.usage, photos.Path) // now unreadable
|
|
h.check(t)
|
|
if b := h.w.Bands()[photos.Path]; b != BandWarning {
|
|
t.Fatalf("band after an unreadable read = %v, want the warning HELD — a blipping drive "+
|
|
"would otherwise silently retract a true alarm", b)
|
|
}
|
|
}
|
|
|
|
// --- Group H — the thresholds keep their gap ------------------------------------------------------
|
|
|
|
// A warn and a clear threshold that can be edited into equality is a flapping bug waiting to be
|
|
// introduced. This pins the ORDERING and a real margin, not the literal numbers.
|
|
func TestThresholdsKeepTheirHysteresisGap(t *testing.T) {
|
|
if ClearUsedPercent >= WarnUsedPercent {
|
|
t.Fatalf("ClearUsedPercent (%.1f) must be strictly BELOW WarnUsedPercent (%.1f) — equal "+
|
|
"thresholds make a filesystem sitting on the line warn, clear, warn, clear every check",
|
|
ClearUsedPercent, WarnUsedPercent)
|
|
}
|
|
if ClearFreeGiB <= WarnFreeGiB {
|
|
t.Fatalf("ClearFreeGiB (%.1f) must be strictly ABOVE WarnFreeGiB (%.1f) — the free-byte term "+
|
|
"needs the same hysteresis as the percentage term, or it flaps on its own",
|
|
ClearFreeGiB, WarnFreeGiB)
|
|
}
|
|
// A margin, not merely an inequality: a 0.1-point gap is arithmetically a gap and practically none.
|
|
if WarnUsedPercent-ClearUsedPercent < 5 {
|
|
t.Fatalf("the used-percent hysteresis gap is %.1f points — too narrow to damp real wobble",
|
|
WarnUsedPercent-ClearUsedPercent)
|
|
}
|
|
if ClearFreeGiB-WarnFreeGiB < 1 {
|
|
t.Fatalf("the free-space hysteresis gap is %.1f GiB — too narrow", ClearFreeGiB-WarnFreeGiB)
|
|
}
|
|
if CritUsedPercent <= WarnUsedPercent || CritFreeGiB >= WarnFreeGiB {
|
|
t.Fatal("the critical band must be strictly tighter than the warning band on BOTH terms, " +
|
|
"or a filesystem can be critical without ever having been warned")
|
|
}
|
|
}
|
|
|
|
// Both terms must be able to trip INDEPENDENTLY — that is the entire reason there are two.
|
|
func TestEitherTermCanTripTheWarning(t *testing.T) {
|
|
// A big drive: only 60% used, but under the free-space floor. 85% of a 4 TB drive leaves 600 GB,
|
|
// so the percentage alone would never fire here.
|
|
if got := classify(Usage{UsedPercent: 60, AvailGB: 3}, BandOK); got != BandWarning {
|
|
t.Fatalf("60%% used with 3 GB free classified as %v — the free-byte term did not trip, so a "+
|
|
"large drive can run out of space without ever warning", got)
|
|
}
|
|
// A small volume: plenty of GB free in absolute terms is impossible here, so the percentage is
|
|
// what must fire. 88% of a 100 GB volume leaves 12 GB — above the 5 GiB floor.
|
|
if got := classify(Usage{UsedPercent: 88, AvailGB: 12}, BandOK); got != BandWarning {
|
|
t.Fatalf("88%% used with 12 GB free classified as %v — the percentage term did not trip", got)
|
|
}
|
|
if got := classify(Usage{UsedPercent: 50, AvailGB: 50}, BandOK); got != BandOK {
|
|
t.Fatalf("a healthy filesystem classified as %v", got)
|
|
}
|
|
}
|
|
|
|
// --- The copy ------------------------------------------------------------------------------------
|
|
|
|
// The customer message must be ACTIONABLE and specific. A warning that says only "something is
|
|
// filling" is an operator alert wearing the wrong clothes.
|
|
func TestCustomerMessageNamesTheDriveTheSpaceAndTheAction(t *testing.T) {
|
|
msg := Message(photos, Usage{UsedPercent: 87, AvailGB: 4.2, TotalGB: 100}, BandWarning)
|
|
|
|
if !strings.Contains(msg, "Fotók") {
|
|
t.Fatalf("the message does not name the storage by its LABEL — a path means nothing to a "+
|
|
"customer. Got: %s", msg)
|
|
}
|
|
if !strings.Contains(msg, "4,2 GB") {
|
|
t.Fatalf("the message does not give the free space in Hungarian number format (decimal "+
|
|
"COMMA). Got: %s", msg)
|
|
}
|
|
if !strings.Contains(msg, "87%") {
|
|
t.Fatalf("the message does not give the fill percentage. Got: %s", msg)
|
|
}
|
|
if !strings.Contains(msg, "szabadíts fel helyet") && !strings.Contains(msg, "szabadíts fel helyet:") {
|
|
t.Fatalf("the message does not tell the customer WHAT TO DO. Got: %s", msg)
|
|
}
|
|
// Design-system rule: no emoji in customer copy.
|
|
for _, r := range msg {
|
|
if r > 0x2100 {
|
|
t.Fatalf("the message contains an emoji/symbol %q — the design system forbids it in "+
|
|
"customer copy. Got: %s", string(r), msg)
|
|
}
|
|
}
|
|
|
|
crit := Message(photos, Usage{UsedPercent: 96, AvailGB: 1.4, TotalGB: 100}, BandCritical)
|
|
if crit == msg {
|
|
t.Fatal("the critical message is identical to the warning — the urgency must differ")
|
|
}
|
|
if !strings.Contains(crit, "1,4 GB") || !strings.Contains(crit, "Fotók") {
|
|
t.Fatalf("the critical message lost its specifics. Got: %s", crit)
|
|
}
|
|
}
|
|
|
|
// A label-less target must still produce a usable message rather than an empty quote.
|
|
func TestMessageFallsBackToThePathWhenUnlabelled(t *testing.T) {
|
|
msg := Message(Target{Path: "/mnt/sys_drive"}, Usage{UsedPercent: 90, AvailGB: 1.5}, BandWarning)
|
|
if !strings.Contains(msg, "/mnt/sys_drive") {
|
|
t.Fatalf("an unlabelled target rendered without any identifier: %s", msg)
|
|
}
|
|
}
|
|
|
|
// A decommissioned drive must fall out of the state file, so it cannot grow without bound and a
|
|
// re-added drive starts fresh.
|
|
func TestVanishedTargetIsForgotten(t *testing.T) {
|
|
h := newHarness(t, photos)
|
|
h.set(photos.Path, 87, 4.2)
|
|
h.check(t)
|
|
if _, ok := h.w.Bands()[photos.Path]; !ok {
|
|
t.Fatal("the warned filesystem was not recorded")
|
|
}
|
|
|
|
h.target = nil // the drive is decommissioned
|
|
h.check(t)
|
|
if _, ok := h.w.Bands()[photos.Path]; ok {
|
|
t.Fatal("a removed filesystem kept its band — the state file grows without bound and a " +
|
|
"re-added drive would resume from a stale band instead of warning afresh")
|
|
}
|
|
}
|
|
|
|
// --- The run summary is the POSITIVE OBSERVABLE ---------------------------------------------------
|
|
|
|
// Standing rule 3, aimed at the one place it bites hardest here: this check is edge-triggered, so the
|
|
// HEALTHY STEADY STATE IS A QUIET RUN. Without a per-run summary line, "the checker ran and correctly
|
|
// said nothing" and "the checker is dead" produce byte-identical logs, permanently.
|
|
//
|
|
// This was hit for real while live-validating v0.191.1 on guest 9201: an unchanged band produced zero
|
|
// log lines, and proving the checker was alive required a deliberate threshold crossing.
|
|
func TestEveryRunLogsAPositiveObservable(t *testing.T) {
|
|
var buf strings.Builder
|
|
usage := map[string]*Usage{photos.Path: {UsedPercent: 40, AvailGB: 60, TotalGB: 100}}
|
|
w := New(filepath.Join(t.TempDir(), "s.json"), log.New(&buf, "", 0),
|
|
func() []Target { return []Target{photos} },
|
|
func(p string) *Usage { return usage[p] })
|
|
|
|
if err := w.Check(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out := buf.String()
|
|
if !strings.Contains(out, "[fillwatch] checked 1 filesystem(s)") {
|
|
t.Fatalf("a HEALTHY run logged no summary — a quiet run is then indistinguishable from a "+
|
|
"checker that never ran, and for an edge-triggered check that is the normal state. Got:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "all ok") {
|
|
t.Fatalf("the summary does not report the bands. Got:\n%s", out)
|
|
}
|
|
|
|
// And a second, equally quiet run must log again — the observable is per RUN, not per change.
|
|
buf.Reset()
|
|
if err := w.Check(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(buf.String(), "checked 1 filesystem(s)") {
|
|
t.Fatalf("the second unchanged run logged nothing. Got:\n%s", buf.String())
|
|
}
|
|
}
|
|
|
|
// The summary must distinguish an unreadable filesystem from a healthy one — otherwise a drive that
|
|
// has silently gone unreadable for weeks reads as "all fine".
|
|
func TestSummaryCountsUnreadableSeparately(t *testing.T) {
|
|
var buf strings.Builder
|
|
w := New(filepath.Join(t.TempDir(), "s.json"), log.New(&buf, "", 0),
|
|
func() []Target { return []Target{photos} },
|
|
func(string) *Usage { return nil })
|
|
if err := w.Check(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(buf.String(), "checked 0 filesystem(s), 1 unreadable/skipped") {
|
|
t.Fatalf("an unreadable filesystem is not distinguished in the summary — it would read as a "+
|
|
"healthy check. Got:\n%s", buf.String())
|
|
}
|
|
}
|