219 lines
7.7 KiB
Go
219 lines
7.7 KiB
Go
package monitor
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
|
|
)
|
|
|
|
// fakeUsage is a usageReader for tests: it returns a scripted usage/err and counts calls.
|
|
type fakeUsage struct {
|
|
mu sync.Mutex
|
|
usage tenantsync.BoxUsage
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeUsage) Usage(_ context.Context) (tenantsync.BoxUsage, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls++
|
|
return f.usage, f.err
|
|
}
|
|
func (f *fakeUsage) set(u tenantsync.BoxUsage, err error) {
|
|
f.mu.Lock()
|
|
f.usage, f.err = u, err
|
|
f.mu.Unlock()
|
|
}
|
|
func (f *fakeUsage) nCalls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.calls }
|
|
|
|
func usageBytes(totalGiB, usedGiB int64) tenantsync.BoxUsage {
|
|
return tenantsync.BoxUsage{Total: totalGiB * gib, Used: usedGiB * gib, Avail: (totalGiB - usedGiB) * gib}
|
|
}
|
|
|
|
// C1 — throttle: ≤1 ep0 usage call per 15-min window across an hour of 60 s sweeps (≈4, not ≈60).
|
|
func TestPBSDRBox_Throttle(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(40, 8)}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, noEvent, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
base := time.Now().UTC()
|
|
for i := 0; i < 60; i++ {
|
|
cur = base.Add(time.Duration(i) * 60 * time.Second)
|
|
c.Check()
|
|
}
|
|
if f.nCalls() < 3 || f.nCalls() > 5 {
|
|
t.Fatalf("usage op called %d times over an hour of 60 s sweeps, want ≈4 (throttled to 15 min)", f.nCalls())
|
|
}
|
|
snap, ok := c.Snapshot()
|
|
if !ok || snap.State != PBSStateOK || snap.CapacityBytes != 40*gib {
|
|
t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
|
|
}
|
|
}
|
|
|
|
// C2/C3 — fill bands: escalation-only, in-band no re-emit, recovery re-arm, pbsdr-box scope.
|
|
// Red-proof (esc): remove the escalation-only guard → the C3 same-band case re-emits → fails.
|
|
func TestPBSDRBox_FillBands(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(1000, 750)} // 75%
|
|
ev := &capturedBox{}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
base := time.Now().UTC()
|
|
cur = base
|
|
step := func(usedGiB int64) {
|
|
cur = cur.Add(16 * time.Minute)
|
|
f.set(usageBytes(1000, usedGiB), nil)
|
|
c.Check()
|
|
}
|
|
|
|
c.Check() // 75% → no emit
|
|
if len(ev.typ) != 0 {
|
|
t.Fatalf("75%% must not emit, got %v", ev.typ)
|
|
}
|
|
step(820) // 82% → one warning
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 || ev.sev[0] != "warning" {
|
|
t.Fatalf("82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
|
|
}
|
|
step(850) // 85% same band → no re-emit
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("same-band must NOT re-emit, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
step(920) // 92% → critical
|
|
if count(ev.typ, "pbsdr_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
|
|
t.Fatalf("92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
|
|
}
|
|
step(700) // 70% → recovery re-arm (silent)
|
|
if c.FillState() != bandOK {
|
|
t.Fatalf("recovery must re-arm to ok, got %s", c.FillState())
|
|
}
|
|
if count(ev.typ, "pbsdr_box_fill") != 2 {
|
|
t.Fatalf("recovery must be silent, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
step(920) // re-breach → emits again
|
|
if count(ev.typ, "pbsdr_box_fill") != 3 {
|
|
t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
for _, cust := range ev.cust {
|
|
if cust != "pbsdr-box" {
|
|
t.Fatalf("every emit must carry the pbsdr-box scope, got %q", cust)
|
|
}
|
|
}
|
|
}
|
|
|
|
// C4 — usage-unsupported (ErrUsageUnsupported) → the distinct "unavailable" state, NOT degraded, NO
|
|
// alert, NO band transition. Red-proof (unavail-band): let unavailable drive a band → alert fires → fails.
|
|
func TestPBSDRBox_Unavailable(t *testing.T) {
|
|
f := &fakeUsage{err: tenantsync.ErrUsageUnsupported}
|
|
ev := &capturedBox{}
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.Check()
|
|
snap, ok := c.Snapshot()
|
|
if !ok || snap.State != PBSStateUnavailable {
|
|
t.Fatalf("ErrUsageUnsupported must yield the 'unavailable' state, got %+v", snap)
|
|
}
|
|
if len(ev.typ) != 0 {
|
|
t.Fatalf("unavailable must NOT alert, got %v", ev.typ)
|
|
}
|
|
if c.FillState() != "unknown" {
|
|
t.Fatalf("unavailable must not set a fill band, got %s", c.FillState())
|
|
}
|
|
}
|
|
|
|
// C5 — exec error/timeout → last snapshot kept, marked degraded, NO band transition (missing ≠ 0%).
|
|
func TestPBSDRBox_DegradedKeepsLast(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(1000, 920)} // 92% critical
|
|
ev := &capturedBox{}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
cur = time.Now().UTC()
|
|
|
|
c.Check() // establish 92% critical + emit
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("setup: want one critical emit, got %v", ev.typ)
|
|
}
|
|
before, _ := c.Snapshot()
|
|
|
|
f.set(tenantsync.BoxUsage{}, fmt.Errorf("ssh timeout"))
|
|
cur = cur.Add(16 * time.Minute)
|
|
c.Check()
|
|
|
|
after, ok := c.Snapshot()
|
|
if !ok || after.State != PBSStateDegraded {
|
|
t.Fatalf("a failed poll must mark degraded (keeping last), got %+v", after)
|
|
}
|
|
if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
|
|
t.Fatalf("a failed poll must keep the last-known values (before %d/%d, after %d/%d)",
|
|
before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
|
|
}
|
|
if c.FillState() != bandCritical {
|
|
t.Fatalf("a failed poll must NOT transition the band, got %s", c.FillState())
|
|
}
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("a failed poll must not emit, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
}
|
|
|
|
// R-1165: household namespaces with no prune job. A non-empty set alarms once (operator scope); the same set again is
|
|
// silent; a changed set alarms again; an empty set re-arms silently; an UNREAD list (old endpoint, unreadable jobs)
|
|
// changes nothing — it never reads as „all covered".
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with checkPruneCoverage's call removed from Check this fails with
|
|
//
|
|
// pbsdr_box_test.go: two unpruned namespaces → want 1 pbsdr_namespace_without_prune_job, got 0
|
|
//
|
|
// Restored.
|
|
func TestPBSDRBox_R1165_NamespaceWithoutPruneJob(t *testing.T) {
|
|
f := &fakeUsage{}
|
|
ev := &capturedBox{}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
cur = time.Now().UTC()
|
|
step := func(checked bool, ns ...string) {
|
|
cur = cur.Add(16 * time.Minute)
|
|
u := usageBytes(1000, 100)
|
|
u.UnprunedChecked, u.Unpruned = checked, ns
|
|
f.set(u, nil)
|
|
c.Check()
|
|
}
|
|
const typ = "pbsdr_namespace_without_prune_job"
|
|
|
|
step(false) // an endpoint before v1.3.0: nothing to say
|
|
if count(ev.typ, typ) != 0 {
|
|
t.Fatalf("an unread list must not alarm, got %v", ev.typ)
|
|
}
|
|
step(true, "tester-1", "Tester-2")
|
|
if count(ev.typ, typ) != 1 {
|
|
t.Fatalf("two unpruned namespaces → want 1 %s, got %d", typ, count(ev.typ, typ))
|
|
}
|
|
i := len(ev.typ) - 1
|
|
if ev.sev[i] != "warning" || ev.cust[i] != "pbsdr-box" || !strings.Contains(ev.msg[i], "Tester-2,tester-1") {
|
|
t.Errorf("the alarm must be an operator-scope warning naming both: sev=%s scope=%s msg=%q", ev.sev[i], ev.cust[i], ev.msg[i])
|
|
}
|
|
step(true, "Tester-2", "tester-1") // the same set, another order
|
|
step(false) // an unreadable list in between: no change, no re-arm
|
|
step(true, "tester-1", "Tester-2")
|
|
if count(ev.typ, typ) != 1 {
|
|
t.Fatalf("the same set must not re-alarm, got %d", count(ev.typ, typ))
|
|
}
|
|
step(true, "tester-1", "Tester-2", "house-3") // a new household without a job
|
|
if count(ev.typ, typ) != 2 {
|
|
t.Fatalf("a changed set must alarm again, got %d", count(ev.typ, typ))
|
|
}
|
|
step(true) // every namespace covered: silent re-arm
|
|
if count(ev.typ, typ) != 2 {
|
|
t.Fatalf("all covered must be silent, got %d", count(ev.typ, typ))
|
|
}
|
|
step(true, "house-3")
|
|
if count(ev.typ, typ) != 3 {
|
|
t.Fatalf("after a re-arm a gap must alarm again, got %d", count(ev.typ, typ))
|
|
}
|
|
}
|