230 lines
8.5 KiB
Go
230 lines
8.5 KiB
Go
package monitor
|
||
|
||
import (
|
||
"fmt"
|
||
"math"
|
||
"testing"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
)
|
||
|
||
const gib = int64(1) << 30
|
||
|
||
// boxWith builds a fake StorageBox with the given capacity + used bytes (data-only, no snapshots).
|
||
func boxWith(capacityBytes, usedBytes int64) hetznerapi.StorageBox {
|
||
return hetznerapi.StorageBox{
|
||
ID: 1, Status: "active",
|
||
StorageBoxType: hetznerapi.StorageBoxType{Name: "bx11", Size: capacityBytes},
|
||
Stats: hetznerapi.StorageBoxStats{Size: usedBytes, SizeData: usedBytes},
|
||
}
|
||
}
|
||
|
||
// seedOffsiteCfg writes a customer config whose ConfigJSON carries an offsite descriptor.
|
||
func seedOffsiteCfg(t *testing.T, st *store.Store, id string, enabled bool, typ string, quotaGB int) {
|
||
t.Helper()
|
||
cfgJSON := fmt.Sprintf(`{"offsite":{"enabled":%v,"type":%q,"quota_gb":%d}}`, enabled, typ, quotaGB)
|
||
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: id, APIKey: "k-" + id, RetrievalPassword: "p", ConfigJSON: cfgJSON}); err != nil {
|
||
t.Fatalf("seed config %s: %v", id, err)
|
||
}
|
||
}
|
||
|
||
func noEvent(_, _, _, _, _, _ string) {}
|
||
|
||
// Scenario A — fetch is throttled: ~4 API reads over an hour of 60 s sweeps, not ~60.
|
||
// Red-proof (i): drop the throttle guard in Check() → GetBoxCalls ≈ 60 → this fails.
|
||
func TestOffsiteBox_FetchThrottle(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
fake := hetznerapi.NewFake()
|
||
fake.Boxes[1] = boxWith(1<<40, 300*gib) // 1 TiB, 300 GiB used
|
||
var cur time.Time
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, noEvent, quietLog())
|
||
c.now = func() time.Time { return cur }
|
||
|
||
base := time.Now().UTC()
|
||
for i := 0; i < 60; i++ { // 60 sweeps at 60 s spacing = one simulated hour
|
||
cur = base.Add(time.Duration(i) * 60 * time.Second)
|
||
c.Check()
|
||
}
|
||
if fake.GetBoxCalls < 3 || fake.GetBoxCalls > 5 {
|
||
t.Fatalf("GetStorageBox called %d times over an hour of 60 s sweeps, want ≈4 (fetch-throttled to 15 min)", fake.GetBoxCalls)
|
||
}
|
||
snap, ok := c.Snapshot()
|
||
if !ok || snap.CapacityBytes != 1<<40 || snap.UsedBytes != 300*gib {
|
||
t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
|
||
}
|
||
}
|
||
|
||
// Scenario B — oversubscription math: Σ = shared+enabled only (A+B); dedicated (C) and disabled (D)
|
||
// excluded; ratio = Σquota / CAPACITY (not used).
|
||
// Red-proof (ii): include dedicated/disabled in the sum → this fails.
|
||
func TestOffsiteBox_OversubMath(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
seedOffsiteCfg(t, st, "a", true, "shared", 500)
|
||
seedOffsiteCfg(t, st, "b", true, "shared", 700)
|
||
seedOffsiteCfg(t, st, "c", true, "dedicated", 9999) // excluded: dedicated
|
||
seedOffsiteCfg(t, st, "d", false, "shared", 300) // excluded: disabled
|
||
|
||
fake := hetznerapi.NewFake()
|
||
fake.Boxes[1] = boxWith(1024*gib, 100*gib) // capacity 1024 GB
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, noEvent, quietLog())
|
||
c.Check()
|
||
|
||
snap, _ := c.Snapshot()
|
||
if snap.SumQuotaGB != 1200 {
|
||
t.Fatalf("Σquota = %d GB, want 1200 (A 500 + B 700 only)", snap.SumQuotaGB)
|
||
}
|
||
want := 1200.0 / 1024.0
|
||
if math.Abs(snap.Ratio-want) > 0.001 {
|
||
t.Fatalf("ratio = %.4f, want %.4f (Σquota/capacity)", snap.Ratio, want)
|
||
}
|
||
}
|
||
|
||
// captureEvents records (eventType, severity, customerID) per emit.
|
||
type capturedBox struct{ typ, sev, cust, msg []string }
|
||
|
||
func (c *capturedBox) fn(cust, et, sev, msg, _, _ string) {
|
||
c.msg = append(c.msg, msg)
|
||
c.cust = append(c.cust, cust)
|
||
c.typ = append(c.typ, et)
|
||
c.sev = append(c.sev, sev)
|
||
}
|
||
|
||
// Scenario C (fill) — escalation-only, in-band no re-emit, recovery re-arms. Every emit carries the
|
||
// "pool-box" scope. Red-proof (iii): remove the escalation-only guard → C3 fails.
|
||
func TestOffsiteBox_FillBands(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
fake := hetznerapi.NewFake()
|
||
cap := int64(1000) * gib
|
||
fake.Boxes[1] = boxWith(cap, 750*gib) // 75%
|
||
ev := &capturedBox{}
|
||
var cur time.Time
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, ev.fn, quietLog())
|
||
c.now = func() time.Time { return cur }
|
||
base := time.Now().UTC()
|
||
step := func(usedGiB int64) {
|
||
cur = cur.Add(16 * time.Minute) // bust the 15-min throttle each step
|
||
fake.Boxes[1] = boxWith(cap, usedGiB*gib)
|
||
c.Check()
|
||
}
|
||
cur = base
|
||
|
||
c.Check() // C1: 75% → no emit
|
||
if len(ev.typ) != 0 {
|
||
t.Fatalf("C1 75%% must not emit, got %v", ev.typ)
|
||
}
|
||
step(820) // C2: 82% → one warning
|
||
if count(ev.typ, "offsite_box_fill") != 1 || ev.sev[0] != "warning" {
|
||
t.Fatalf("C2 82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
|
||
}
|
||
step(850) // C3: 85% same band → no second emit
|
||
if count(ev.typ, "offsite_box_fill") != 1 {
|
||
t.Fatalf("C3 same-band must NOT re-emit, got %d", count(ev.typ, "offsite_box_fill"))
|
||
}
|
||
step(920) // C4: 92% → escalate to critical
|
||
if count(ev.typ, "offsite_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
|
||
t.Fatalf("C4 92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
|
||
}
|
||
step(700) // C5: 70% → recovery re-arm (no emit)
|
||
if c.FillState() != bandOK {
|
||
t.Fatalf("C5 recovery must re-arm to ok, got %s", c.FillState())
|
||
}
|
||
if count(ev.typ, "offsite_box_fill") != 2 {
|
||
t.Fatalf("C5 recovery must be silent, got %d emits", count(ev.typ, "offsite_box_fill"))
|
||
}
|
||
step(920) // re-breach after recovery → emits again
|
||
if count(ev.typ, "offsite_box_fill") != 3 {
|
||
t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "offsite_box_fill"))
|
||
}
|
||
for _, cust := range ev.cust {
|
||
if cust != "pool-box" {
|
||
t.Fatalf("every emit must carry the pool-box scope, got %q", cust)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Scenario C6 — oversubscription is an INDEPENDENT signal: it fires on ratio alone even when fill is
|
||
// nominal, with its own event type.
|
||
func TestOffsiteBox_OversubIndependent(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
seedOffsiteCfg(t, st, "a", true, "shared", 2300) // Σ 2300 GB
|
||
fake := hetznerapi.NewFake()
|
||
fake.Boxes[1] = boxWith(1000*gib, 500*gib) // 50% fill (nominal), ratio 2.3×
|
||
ev := &capturedBox{}
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, ev.fn, quietLog())
|
||
c.Check()
|
||
if count(ev.typ, "offsite_box_oversub") != 1 {
|
||
t.Fatalf("2.3× oversub must warn once, got %v", ev.typ)
|
||
}
|
||
if count(ev.typ, "offsite_box_fill") != 0 {
|
||
t.Fatalf("fill is nominal (50%%) — no fill emit, got %v", ev.typ)
|
||
}
|
||
if ev.cust[0] != "pool-box" {
|
||
t.Fatalf("oversub emit scope = %q, want pool-box", ev.cust[0])
|
||
}
|
||
}
|
||
|
||
// Scenario D — API failure honesty: the last snapshot is KEPT (marked degraded), no band transition,
|
||
// and a failed fetch NEVER zeroes the snapshot (the recovery re-arm must not trigger off missing data).
|
||
// Red-proof (iv): let a failed fetch zero the snapshot → this fails.
|
||
func TestOffsiteBox_FailedFetchHonesty(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
fake := hetznerapi.NewFake()
|
||
cap := int64(1000) * gib
|
||
fake.Boxes[1] = boxWith(cap, 920*gib) // 92% → critical
|
||
ev := &capturedBox{}
|
||
var cur time.Time
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, ev.fn, quietLog())
|
||
c.now = func() time.Time { return cur }
|
||
base := time.Now().UTC()
|
||
cur = base
|
||
|
||
c.Check() // establishes a 92% critical snapshot + emit
|
||
if count(ev.typ, "offsite_box_fill") != 1 {
|
||
t.Fatalf("setup: want one critical emit, got %v", ev.typ)
|
||
}
|
||
before, _ := c.Snapshot()
|
||
|
||
// Now the API fails.
|
||
fake.FailGetBox = fmt.Errorf("hetzner timeout")
|
||
cur = cur.Add(16 * time.Minute)
|
||
c.Check()
|
||
|
||
after, ok := c.Snapshot()
|
||
if !ok {
|
||
t.Fatal("a failed fetch must KEEP the last snapshot, not drop it")
|
||
}
|
||
if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
|
||
t.Fatalf("a failed fetch must not alter the last-known values (before %d/%d, after %d/%d)",
|
||
before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
|
||
}
|
||
if !after.Degraded {
|
||
t.Fatal("a failed fetch must mark the served snapshot Degraded (visible staleness)")
|
||
}
|
||
if c.FillState() != bandCritical {
|
||
t.Fatalf("a failed fetch must NOT transition the band (missing data ≠ 0%%), got %s", c.FillState())
|
||
}
|
||
// No new emit on the failure.
|
||
if count(ev.typ, "offsite_box_fill") != 1 {
|
||
t.Fatalf("a failed fetch must not emit, got %d total", count(ev.typ, "offsite_box_fill"))
|
||
}
|
||
}
|
||
|
||
// Not-configured / zero-capacity guard: a zero-capacity box marks degraded, never divides, never alerts.
|
||
func TestOffsiteBox_ZeroCapacityGuard(t *testing.T) {
|
||
st := newDiskStore(t)
|
||
fake := hetznerapi.NewFake()
|
||
fake.Boxes[1] = boxWith(0, 0) // initializing box — no capacity yet
|
||
ev := &capturedBox{}
|
||
c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, 0, ev.fn, quietLog())
|
||
c.Check()
|
||
snap, ok := c.Snapshot()
|
||
if !ok || !snap.Degraded {
|
||
t.Fatalf("zero capacity must yield a degraded snapshot, got %+v", snap)
|
||
}
|
||
if len(ev.typ) != 0 {
|
||
t.Fatalf("zero capacity must not alert, got %v", ev.typ)
|
||
}
|
||
}
|