hub v0.64.0 — offsite pool-box aggregate: fill, oversubscription, per-customer bars, operator alert (R-5)

The operator sees the shared pool box's real state on the hub: total box fill vs capacity, Σ(shared soft quotas) vs capacity (the oversubscription ratio), per-customer usage/quota bars, and a box-level operator alert (fill % + oversub ratio) on the existing dispatcher's operator channel. Per-customer fill alerts already existed; the box-level aggregate was the gap. READ-ONLY against Hetzner (GET only). Phase-0 probe (gate PASSED): the live pool box 611714 returns capacity via storage_box_type.size (1 TiB / bx11) and usage via a stats object (size/size_data/ size_snapshots), all bytes; our token reads it (200). - hetznerapi: additive StorageBoxType + StorageBoxStats on StorageBox (no existing field/ method changed); fake carries them + a GetBoxCalls counter; golden decode test. - monitor.OffsiteBoxChecker: OffsiteChecker-sibling for the box; fetch-throttled (1 GET/ 15min), cached BoxSnapshot, escalation-only + recovery re-arm. FILL (used/capacity 80/90) + OVERSUB (Σ shared+enabled quotas / capacity, 2.0x) — independent. Σ from the ConfigJSON Descriptor (offsite.ReadDescriptor, new), never the report echo; dedicated+disabled excluded. Scope "pool-box" -> operator channel only, no SaveEvent. Failed fetch keeps the last snapshot degraded; missing data never becomes 0% and never transitions a band. - config: Alerting.OffsiteBoxFill{Warn,Crit}Percent + OffsiteOversubWarnRatio (80/90/2.0 defaults; thresholds pending Viktor's ruling). Constructed in the HETZNER_TOKEN branch, 60s sweep, snapshot handed to the web server. - web: Offsite-tab panel (fill bar, Σ+ratio, per-customer usage/quota rows) + a compact dashboard tile; reads the cached snapshot only, never fetches; nil -> "not configured". Tests: 10 new + 4 red-proofs (throttle, Σ filter, escalation-only, failed-fetch honesty), all confirmed red then restored. go build/vet/test all pass; hub confirm gate OK.
2026-07-17 20:15:34 +02:00
parent 85a14192e7
commit 4bb2df0dc4
17 changed files with 1034 additions and 17 deletions
@@ -0,0 +1,228 @@
+package monitor
+
+import (
+	"fmt"
+	"math"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+const gib = int64(1) << 30
+
+// boxWith builds a fake StorageBox with the given capacity + used bytes (data-only, no snapshots).
+func boxWith(capacityBytes, usedBytes int64) hetznerapi.StorageBox {
+	return hetznerapi.StorageBox{
+		ID: 1, Status: "active",
+		StorageBoxType: hetznerapi.StorageBoxType{Name: "bx11", Size: capacityBytes},
+		Stats:          hetznerapi.StorageBoxStats{Size: usedBytes, SizeData: usedBytes},
+	}
+}
+
+// seedOffsiteCfg writes a customer config whose ConfigJSON carries an offsite descriptor.
+func seedOffsiteCfg(t *testing.T, st *store.Store, id string, enabled bool, typ string, quotaGB int) {
+	t.Helper()
+	cfgJSON := fmt.Sprintf(`{"offsite":{"enabled":%v,"type":%q,"quota_gb":%d}}`, enabled, typ, quotaGB)
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: id, APIKey: "k-" + id, RetrievalPassword: "p", ConfigJSON: cfgJSON}); err != nil {
+		t.Fatalf("seed config %s: %v", id, err)
+	}
+}
+
+func noEvent(_, _, _, _, _, _ string) {}
+
+// Scenario A — fetch is throttled: ~4 API reads over an hour of 60 s sweeps, not ~60.
+// Red-proof (i): drop the throttle guard in Check() → GetBoxCalls ≈ 60 → this fails.
+func TestOffsiteBox_FetchThrottle(t *testing.T) {
+	st := newDiskStore(t)
+	fake := hetznerapi.NewFake()
+	fake.Boxes[1] = boxWith(1<<40, 300*gib) // 1 TiB, 300 GiB used
+	var cur time.Time
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, noEvent, quietLog())
+	c.now = func() time.Time { return cur }
+
+	base := time.Now().UTC()
+	for i := 0; i < 60; i++ { // 60 sweeps at 60 s spacing = one simulated hour
+		cur = base.Add(time.Duration(i) * 60 * time.Second)
+		c.Check()
+	}
+	if fake.GetBoxCalls < 3 || fake.GetBoxCalls > 5 {
+		t.Fatalf("GetStorageBox called %d times over an hour of 60 s sweeps, want ≈4 (fetch-throttled to 15 min)", fake.GetBoxCalls)
+	}
+	snap, ok := c.Snapshot()
+	if !ok || snap.CapacityBytes != 1<<40 || snap.UsedBytes != 300*gib {
+		t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
+	}
+}
+
+// Scenario B — oversubscription math: Σ = shared+enabled only (A+B); dedicated (C) and disabled (D)
+// excluded; ratio = Σquota / CAPACITY (not used).
+// Red-proof (ii): include dedicated/disabled in the sum → this fails.
+func TestOffsiteBox_OversubMath(t *testing.T) {
+	st := newDiskStore(t)
+	seedOffsiteCfg(t, st, "a", true, "shared", 500)
+	seedOffsiteCfg(t, st, "b", true, "shared", 700)
+	seedOffsiteCfg(t, st, "c", true, "dedicated", 9999) // excluded: dedicated
+	seedOffsiteCfg(t, st, "d", false, "shared", 300)    // excluded: disabled
+
+	fake := hetznerapi.NewFake()
+	fake.Boxes[1] = boxWith(1024*gib, 100*gib) // capacity 1024 GB
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, noEvent, quietLog())
+	c.Check()
+
+	snap, _ := c.Snapshot()
+	if snap.SumQuotaGB != 1200 {
+		t.Fatalf("Σquota = %d GB, want 1200 (A 500 + B 700 only)", snap.SumQuotaGB)
+	}
+	want := 1200.0 / 1024.0
+	if math.Abs(snap.Ratio-want) > 0.001 {
+		t.Fatalf("ratio = %.4f, want %.4f (Σquota/capacity)", snap.Ratio, want)
+	}
+}
+
+// captureEvents records (eventType, severity, customerID) per emit.
+type capturedBox struct{ typ, sev, cust []string }
+
+func (c *capturedBox) fn(cust, et, sev, _, _, _ string) {
+	c.cust = append(c.cust, cust)
+	c.typ = append(c.typ, et)
+	c.sev = append(c.sev, sev)
+}
+
+// Scenario C (fill) — escalation-only, in-band no re-emit, recovery re-arms. Every emit carries the
+// "pool-box" scope. Red-proof (iii): remove the escalation-only guard → C3 fails.
+func TestOffsiteBox_FillBands(t *testing.T) {
+	st := newDiskStore(t)
+	fake := hetznerapi.NewFake()
+	cap := int64(1000) * gib
+	fake.Boxes[1] = boxWith(cap, 750*gib) // 75%
+	ev := &capturedBox{}
+	var cur time.Time
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog())
+	c.now = func() time.Time { return cur }
+	base := time.Now().UTC()
+	step := func(usedGiB int64) {
+		cur = cur.Add(16 * time.Minute) // bust the 15-min throttle each step
+		fake.Boxes[1] = boxWith(cap, usedGiB*gib)
+		c.Check()
+	}
+	cur = base
+
+	c.Check() // C1: 75% → no emit
+	if len(ev.typ) != 0 {
+		t.Fatalf("C1 75%% must not emit, got %v", ev.typ)
+	}
+	step(820) // C2: 82% → one warning
+	if count(ev.typ, "offsite_box_fill") != 1 || ev.sev[0] != "warning" {
+		t.Fatalf("C2 82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
+	}
+	step(850) // C3: 85% same band → no second emit
+	if count(ev.typ, "offsite_box_fill") != 1 {
+		t.Fatalf("C3 same-band must NOT re-emit, got %d", count(ev.typ, "offsite_box_fill"))
+	}
+	step(920) // C4: 92% → escalate to critical
+	if count(ev.typ, "offsite_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
+		t.Fatalf("C4 92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
+	}
+	step(700) // C5: 70% → recovery re-arm (no emit)
+	if c.FillState() != bandOK {
+		t.Fatalf("C5 recovery must re-arm to ok, got %s", c.FillState())
+	}
+	if count(ev.typ, "offsite_box_fill") != 2 {
+		t.Fatalf("C5 recovery must be silent, got %d emits", count(ev.typ, "offsite_box_fill"))
+	}
+	step(920) // re-breach after recovery → emits again
+	if count(ev.typ, "offsite_box_fill") != 3 {
+		t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "offsite_box_fill"))
+	}
+	for _, cust := range ev.cust {
+		if cust != "pool-box" {
+			t.Fatalf("every emit must carry the pool-box scope, got %q", cust)
+		}
+	}
+}
+
+// Scenario C6 — oversubscription is an INDEPENDENT signal: it fires on ratio alone even when fill is
+// nominal, with its own event type.
+func TestOffsiteBox_OversubIndependent(t *testing.T) {
+	st := newDiskStore(t)
+	seedOffsiteCfg(t, st, "a", true, "shared", 2300) // Σ 2300 GB
+	fake := hetznerapi.NewFake()
+	fake.Boxes[1] = boxWith(1000*gib, 500*gib) // 50% fill (nominal), ratio 2.3×
+	ev := &capturedBox{}
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog())
+	c.Check()
+	if count(ev.typ, "offsite_box_oversub") != 1 {
+		t.Fatalf("2.3× oversub must warn once, got %v", ev.typ)
+	}
+	if count(ev.typ, "offsite_box_fill") != 0 {
+		t.Fatalf("fill is nominal (50%%) — no fill emit, got %v", ev.typ)
+	}
+	if ev.cust[0] != "pool-box" {
+		t.Fatalf("oversub emit scope = %q, want pool-box", ev.cust[0])
+	}
+}
+
+// Scenario D — API failure honesty: the last snapshot is KEPT (marked degraded), no band transition,
+// and a failed fetch NEVER zeroes the snapshot (the recovery re-arm must not trigger off missing data).
+// Red-proof (iv): let a failed fetch zero the snapshot → this fails.
+func TestOffsiteBox_FailedFetchHonesty(t *testing.T) {
+	st := newDiskStore(t)
+	fake := hetznerapi.NewFake()
+	cap := int64(1000) * gib
+	fake.Boxes[1] = boxWith(cap, 920*gib) // 92% → critical
+	ev := &capturedBox{}
+	var cur time.Time
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog())
+	c.now = func() time.Time { return cur }
+	base := time.Now().UTC()
+	cur = base
+
+	c.Check() // establishes a 92% critical snapshot + emit
+	if count(ev.typ, "offsite_box_fill") != 1 {
+		t.Fatalf("setup: want one critical emit, got %v", ev.typ)
+	}
+	before, _ := c.Snapshot()
+
+	// Now the API fails.
+	fake.FailGetBox = fmt.Errorf("hetzner timeout")
+	cur = cur.Add(16 * time.Minute)
+	c.Check()
+
+	after, ok := c.Snapshot()
+	if !ok {
+		t.Fatal("a failed fetch must KEEP the last snapshot, not drop it")
+	}
+	if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
+		t.Fatalf("a failed fetch must not alter the last-known values (before %d/%d, after %d/%d)",
+			before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
+	}
+	if !after.Degraded {
+		t.Fatal("a failed fetch must mark the served snapshot Degraded (visible staleness)")
+	}
+	if c.FillState() != bandCritical {
+		t.Fatalf("a failed fetch must NOT transition the band (missing data ≠ 0%%), got %s", c.FillState())
+	}
+	// No new emit on the failure.
+	if count(ev.typ, "offsite_box_fill") != 1 {
+		t.Fatalf("a failed fetch must not emit, got %d total", count(ev.typ, "offsite_box_fill"))
+	}
+}
+
+// Not-configured / zero-capacity guard: a zero-capacity box marks degraded, never divides, never alerts.
+func TestOffsiteBox_ZeroCapacityGuard(t *testing.T) {
+	st := newDiskStore(t)
+	fake := hetznerapi.NewFake()
+	fake.Boxes[1] = boxWith(0, 0) // initializing box — no capacity yet
+	ev := &capturedBox{}
+	c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog())
+	c.Check()
+	snap, ok := c.Snapshot()
+	if !ok || !snap.Degraded {
+		t.Fatalf("zero capacity must yield a degraded snapshot, got %+v", snap)
+	}
+	if len(ev.typ) != 0 {
+		t.Fatalf("zero capacity must not alert, got %v", ev.typ)
+	}
+}