package monitor import ( "fmt" "math" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) const gib = int64(1) << 30 // boxWith builds a fake StorageBox with the given capacity + used bytes (data-only, no snapshots). func boxWith(capacityBytes, usedBytes int64) hetznerapi.StorageBox { return hetznerapi.StorageBox{ ID: 1, Status: "active", StorageBoxType: hetznerapi.StorageBoxType{Name: "bx11", Size: capacityBytes}, Stats: hetznerapi.StorageBoxStats{Size: usedBytes, SizeData: usedBytes}, } } // seedOffsiteCfg writes a customer config whose ConfigJSON carries an offsite descriptor. func seedOffsiteCfg(t *testing.T, st *store.Store, id string, enabled bool, typ string, quotaGB int) { t.Helper() cfgJSON := fmt.Sprintf(`{"offsite":{"enabled":%v,"type":%q,"quota_gb":%d}}`, enabled, typ, quotaGB) if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: id, APIKey: "k-" + id, RetrievalPassword: "p", ConfigJSON: cfgJSON}); err != nil { t.Fatalf("seed config %s: %v", id, err) } } func noEvent(_, _, _, _, _, _ string) {} // Scenario A — fetch is throttled: ~4 API reads over an hour of 60 s sweeps, not ~60. // Red-proof (i): drop the throttle guard in Check() → GetBoxCalls ≈ 60 → this fails. func TestOffsiteBox_FetchThrottle(t *testing.T) { st := newDiskStore(t) fake := hetznerapi.NewFake() fake.Boxes[1] = boxWith(1<<40, 300*gib) // 1 TiB, 300 GiB used var cur time.Time c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, noEvent, quietLog()) c.now = func() time.Time { return cur } base := time.Now().UTC() for i := 0; i < 60; i++ { // 60 sweeps at 60 s spacing = one simulated hour cur = base.Add(time.Duration(i) * 60 * time.Second) c.Check() } if fake.GetBoxCalls < 3 || fake.GetBoxCalls > 5 { t.Fatalf("GetStorageBox called %d times over an hour of 60 s sweeps, want ≈4 (fetch-throttled to 15 min)", fake.GetBoxCalls) } snap, ok := c.Snapshot() if !ok || snap.CapacityBytes != 1<<40 || snap.UsedBytes != 300*gib { t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok) } } // Scenario B — oversubscription math: Σ = shared+enabled only (A+B); dedicated (C) and disabled (D) // excluded; ratio = Σquota / CAPACITY (not used). // Red-proof (ii): include dedicated/disabled in the sum → this fails. func TestOffsiteBox_OversubMath(t *testing.T) { st := newDiskStore(t) seedOffsiteCfg(t, st, "a", true, "shared", 500) seedOffsiteCfg(t, st, "b", true, "shared", 700) seedOffsiteCfg(t, st, "c", true, "dedicated", 9999) // excluded: dedicated seedOffsiteCfg(t, st, "d", false, "shared", 300) // excluded: disabled fake := hetznerapi.NewFake() fake.Boxes[1] = boxWith(1024*gib, 100*gib) // capacity 1024 GB c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, noEvent, quietLog()) c.Check() snap, _ := c.Snapshot() if snap.SumQuotaGB != 1200 { t.Fatalf("Σquota = %d GB, want 1200 (A 500 + B 700 only)", snap.SumQuotaGB) } want := 1200.0 / 1024.0 if math.Abs(snap.Ratio-want) > 0.001 { t.Fatalf("ratio = %.4f, want %.4f (Σquota/capacity)", snap.Ratio, want) } } // captureEvents records (eventType, severity, customerID) per emit. type capturedBox struct{ typ, sev, cust []string } func (c *capturedBox) fn(cust, et, sev, _, _, _ string) { c.cust = append(c.cust, cust) c.typ = append(c.typ, et) c.sev = append(c.sev, sev) } // Scenario C (fill) — escalation-only, in-band no re-emit, recovery re-arms. Every emit carries the // "pool-box" scope. Red-proof (iii): remove the escalation-only guard → C3 fails. func TestOffsiteBox_FillBands(t *testing.T) { st := newDiskStore(t) fake := hetznerapi.NewFake() cap := int64(1000) * gib fake.Boxes[1] = boxWith(cap, 750*gib) // 75% ev := &capturedBox{} var cur time.Time c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog()) c.now = func() time.Time { return cur } base := time.Now().UTC() step := func(usedGiB int64) { cur = cur.Add(16 * time.Minute) // bust the 15-min throttle each step fake.Boxes[1] = boxWith(cap, usedGiB*gib) c.Check() } cur = base c.Check() // C1: 75% → no emit if len(ev.typ) != 0 { t.Fatalf("C1 75%% must not emit, got %v", ev.typ) } step(820) // C2: 82% → one warning if count(ev.typ, "offsite_box_fill") != 1 || ev.sev[0] != "warning" { t.Fatalf("C2 82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev) } step(850) // C3: 85% same band → no second emit if count(ev.typ, "offsite_box_fill") != 1 { t.Fatalf("C3 same-band must NOT re-emit, got %d", count(ev.typ, "offsite_box_fill")) } step(920) // C4: 92% → escalate to critical if count(ev.typ, "offsite_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" { t.Fatalf("C4 92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev) } step(700) // C5: 70% → recovery re-arm (no emit) if c.FillState() != bandOK { t.Fatalf("C5 recovery must re-arm to ok, got %s", c.FillState()) } if count(ev.typ, "offsite_box_fill") != 2 { t.Fatalf("C5 recovery must be silent, got %d emits", count(ev.typ, "offsite_box_fill")) } step(920) // re-breach after recovery → emits again if count(ev.typ, "offsite_box_fill") != 3 { t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "offsite_box_fill")) } for _, cust := range ev.cust { if cust != "pool-box" { t.Fatalf("every emit must carry the pool-box scope, got %q", cust) } } } // Scenario C6 — oversubscription is an INDEPENDENT signal: it fires on ratio alone even when fill is // nominal, with its own event type. func TestOffsiteBox_OversubIndependent(t *testing.T) { st := newDiskStore(t) seedOffsiteCfg(t, st, "a", true, "shared", 2300) // Σ 2300 GB fake := hetznerapi.NewFake() fake.Boxes[1] = boxWith(1000*gib, 500*gib) // 50% fill (nominal), ratio 2.3× ev := &capturedBox{} c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog()) c.Check() if count(ev.typ, "offsite_box_oversub") != 1 { t.Fatalf("2.3× oversub must warn once, got %v", ev.typ) } if count(ev.typ, "offsite_box_fill") != 0 { t.Fatalf("fill is nominal (50%%) — no fill emit, got %v", ev.typ) } if ev.cust[0] != "pool-box" { t.Fatalf("oversub emit scope = %q, want pool-box", ev.cust[0]) } } // Scenario D — API failure honesty: the last snapshot is KEPT (marked degraded), no band transition, // and a failed fetch NEVER zeroes the snapshot (the recovery re-arm must not trigger off missing data). // Red-proof (iv): let a failed fetch zero the snapshot → this fails. func TestOffsiteBox_FailedFetchHonesty(t *testing.T) { st := newDiskStore(t) fake := hetznerapi.NewFake() cap := int64(1000) * gib fake.Boxes[1] = boxWith(cap, 920*gib) // 92% → critical ev := &capturedBox{} var cur time.Time c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog()) c.now = func() time.Time { return cur } base := time.Now().UTC() cur = base c.Check() // establishes a 92% critical snapshot + emit if count(ev.typ, "offsite_box_fill") != 1 { t.Fatalf("setup: want one critical emit, got %v", ev.typ) } before, _ := c.Snapshot() // Now the API fails. fake.FailGetBox = fmt.Errorf("hetzner timeout") cur = cur.Add(16 * time.Minute) c.Check() after, ok := c.Snapshot() if !ok { t.Fatal("a failed fetch must KEEP the last snapshot, not drop it") } if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes { t.Fatalf("a failed fetch must not alter the last-known values (before %d/%d, after %d/%d)", before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes) } if !after.Degraded { t.Fatal("a failed fetch must mark the served snapshot Degraded (visible staleness)") } if c.FillState() != bandCritical { t.Fatalf("a failed fetch must NOT transition the band (missing data ≠ 0%%), got %s", c.FillState()) } // No new emit on the failure. if count(ev.typ, "offsite_box_fill") != 1 { t.Fatalf("a failed fetch must not emit, got %d total", count(ev.typ, "offsite_box_fill")) } } // Not-configured / zero-capacity guard: a zero-capacity box marks degraded, never divides, never alerts. func TestOffsiteBox_ZeroCapacityGuard(t *testing.T) { st := newDiskStore(t) fake := hetznerapi.NewFake() fake.Boxes[1] = boxWith(0, 0) // initializing box — no capacity yet ev := &capturedBox{} c := NewOffsiteBoxChecker(fake, 1, st, 80, 90, 2.0, ev.fn, quietLog()) c.Check() snap, ok := c.Snapshot() if !ok || !snap.Degraded { t.Fatalf("zero capacity must yield a degraded snapshot, got %+v", snap) } if len(ev.typ) != 0 { t.Fatalf("zero capacity must not alert, got %v", ev.typ) } }