Files
felhom.eu/hub/internal/monitor/pbsdr_box_test.go
T
admin 7f11cfb36c hub v0.65.0 — PBS DR storage visibility (ep0 usage op) + Offsite tab split + dual dashboard gauges (R-5)
Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.

Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.

- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
  path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
  cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
  no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
  neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
  60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
  the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).

Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
2026-07-17 21:13:30 +02:00

161 lines
5.4 KiB
Go

package monitor
import (
"context"
"fmt"
"sync"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
)
// fakeUsage is a usageReader for tests: it returns a scripted usage/err and counts calls.
type fakeUsage struct {
mu sync.Mutex
usage tenantsync.BoxUsage
err error
calls int
}
func (f *fakeUsage) Usage(_ context.Context) (tenantsync.BoxUsage, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls++
return f.usage, f.err
}
func (f *fakeUsage) set(u tenantsync.BoxUsage, err error) {
f.mu.Lock()
f.usage, f.err = u, err
f.mu.Unlock()
}
func (f *fakeUsage) nCalls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.calls }
func usageBytes(totalGiB, usedGiB int64) tenantsync.BoxUsage {
return tenantsync.BoxUsage{Total: totalGiB * gib, Used: usedGiB * gib, Avail: (totalGiB - usedGiB) * gib}
}
// C1 — throttle: ≤1 ep0 usage call per 15-min window across an hour of 60 s sweeps (≈4, not ≈60).
func TestPBSDRBox_Throttle(t *testing.T) {
f := &fakeUsage{usage: usageBytes(40, 8)}
var cur time.Time
c := NewPBSDRBoxChecker(f, 80, 90, noEvent, quietLog())
c.now = func() time.Time { return cur }
base := time.Now().UTC()
for i := 0; i < 60; i++ {
cur = base.Add(time.Duration(i) * 60 * time.Second)
c.Check()
}
if f.nCalls() < 3 || f.nCalls() > 5 {
t.Fatalf("usage op called %d times over an hour of 60 s sweeps, want ≈4 (throttled to 15 min)", f.nCalls())
}
snap, ok := c.Snapshot()
if !ok || snap.State != PBSStateOK || snap.CapacityBytes != 40*gib {
t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
}
}
// C2/C3 — fill bands: escalation-only, in-band no re-emit, recovery re-arm, pbsdr-box scope.
// Red-proof (esc): remove the escalation-only guard → the C3 same-band case re-emits → fails.
func TestPBSDRBox_FillBands(t *testing.T) {
f := &fakeUsage{usage: usageBytes(1000, 750)} // 75%
ev := &capturedBox{}
var cur time.Time
c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
c.now = func() time.Time { return cur }
base := time.Now().UTC()
cur = base
step := func(usedGiB int64) {
cur = cur.Add(16 * time.Minute)
f.set(usageBytes(1000, usedGiB), nil)
c.Check()
}
c.Check() // 75% → no emit
if len(ev.typ) != 0 {
t.Fatalf("75%% must not emit, got %v", ev.typ)
}
step(820) // 82% → one warning
if count(ev.typ, "pbsdr_box_fill") != 1 || ev.sev[0] != "warning" {
t.Fatalf("82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
}
step(850) // 85% same band → no re-emit
if count(ev.typ, "pbsdr_box_fill") != 1 {
t.Fatalf("same-band must NOT re-emit, got %d", count(ev.typ, "pbsdr_box_fill"))
}
step(920) // 92% → critical
if count(ev.typ, "pbsdr_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
t.Fatalf("92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
}
step(700) // 70% → recovery re-arm (silent)
if c.FillState() != bandOK {
t.Fatalf("recovery must re-arm to ok, got %s", c.FillState())
}
if count(ev.typ, "pbsdr_box_fill") != 2 {
t.Fatalf("recovery must be silent, got %d", count(ev.typ, "pbsdr_box_fill"))
}
step(920) // re-breach → emits again
if count(ev.typ, "pbsdr_box_fill") != 3 {
t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "pbsdr_box_fill"))
}
for _, cust := range ev.cust {
if cust != "pbsdr-box" {
t.Fatalf("every emit must carry the pbsdr-box scope, got %q", cust)
}
}
}
// C4 — usage-unsupported (ErrUsageUnsupported) → the distinct "unavailable" state, NOT degraded, NO
// alert, NO band transition. Red-proof (unavail-band): let unavailable drive a band → alert fires → fails.
func TestPBSDRBox_Unavailable(t *testing.T) {
f := &fakeUsage{err: tenantsync.ErrUsageUnsupported}
ev := &capturedBox{}
c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
c.Check()
snap, ok := c.Snapshot()
if !ok || snap.State != PBSStateUnavailable {
t.Fatalf("ErrUsageUnsupported must yield the 'unavailable' state, got %+v", snap)
}
if len(ev.typ) != 0 {
t.Fatalf("unavailable must NOT alert, got %v", ev.typ)
}
if c.FillState() != "unknown" {
t.Fatalf("unavailable must not set a fill band, got %s", c.FillState())
}
}
// C5 — exec error/timeout → last snapshot kept, marked degraded, NO band transition (missing ≠ 0%).
func TestPBSDRBox_DegradedKeepsLast(t *testing.T) {
f := &fakeUsage{usage: usageBytes(1000, 920)} // 92% critical
ev := &capturedBox{}
var cur time.Time
c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
c.now = func() time.Time { return cur }
cur = time.Now().UTC()
c.Check() // establish 92% critical + emit
if count(ev.typ, "pbsdr_box_fill") != 1 {
t.Fatalf("setup: want one critical emit, got %v", ev.typ)
}
before, _ := c.Snapshot()
f.set(tenantsync.BoxUsage{}, fmt.Errorf("ssh timeout"))
cur = cur.Add(16 * time.Minute)
c.Check()
after, ok := c.Snapshot()
if !ok || after.State != PBSStateDegraded {
t.Fatalf("a failed poll must mark degraded (keeping last), got %+v", after)
}
if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
t.Fatalf("a failed poll must keep the last-known values (before %d/%d, after %d/%d)",
before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
}
if c.FillState() != bandCritical {
t.Fatalf("a failed poll must NOT transition the band, got %s", c.FillState())
}
if count(ev.typ, "pbsdr_box_fill") != 1 {
t.Fatalf("a failed poll must not emit, got %d", count(ev.typ, "pbsdr_box_fill"))
}
}