hub v0.65.0 — PBS DR storage visibility (ep0 usage op) + Offsite tab split + dual dashboard gauges (R-5)

Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR = the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A (Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout. Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path> yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token. - scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation. - tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful). - monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle, cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only, no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0, neutral no-alert) / degraded (exec failed, keep last). - config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client, 60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation). - web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel; the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a). - runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change). Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band) confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
2026-07-17 21:13:30 +02:00
parent 3588a31b78
commit 7f11cfb36c
19 changed files with 856 additions and 22 deletions
@@ -0,0 +1,184 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
+)
+
+// usageReader is the ep0 datastore-usage seam (satisfied by *tenantsync.Client). Narrow so tests inject
+// a fake and the checker never depends on the full tenantsync surface.
+type usageReader interface {
+	Usage(ctx context.Context) (tenantsync.BoxUsage, error)
+}
+
+// PBSDRBoxChecker (v0.65.0) watches the PBS DR datastore (felhom-offsite on ep0) fill — the sibling of
+// OffsiteBoxChecker over a DIFFERENT source: the ep0 read-only `usage` op (df on the datastore path), NOT
+// the Hetzner API. FILL ONLY (PBS DR uses namespaces, not soft quotas — no oversubscription concept). Same
+// shape: 15-min throttle, cached snapshot, escalation-only emit + silent recovery re-arm, degraded-on-
+// failure honesty. THREE snapshot states:
+//   - "ok"          → bands drive alerts.
+//   - "unavailable" → the endpoint script predates the usage op (≤ v1.1.0) → ErrUsageUnsupported. An
+//                     EXPECTED pre-update condition: neutral, NO alert, logged once. The gauge shows n/a.
+//   - "degraded"    → exec failed/timed out → keep the last snapshot, NO band transition (missing ≠ 0%).
+//
+// Events carry the customer-less scope "pbsdr-box" → operator channel ONLY (processCustomer no-ops on it);
+// NO SaveEvent (no customer row to key it to). Deliberately NOT bolted onto OffsiteBoxChecker.
+type PBSDRBoxChecker struct {
+	reader   usageReader
+	fillWarn float64
+	fillCrit float64
+	onEvent  EventNotifyFunc
+	logger   *log.Logger
+	now      func() time.Time // injectable clock (tests)
+
+	mu            sync.Mutex
+	snap          PBSBoxSnapshot
+	haveSnap      bool
+	lastFetch     time.Time
+	fillBand      string // "" until first ok eval → born/persistent; only an "ok" fetch updates it
+	unavailLogged bool   // log the "unavailable" state ONCE, not per sweep
+}
+
+// pbsdrBoxScope is the customer-less cooldown/display key for PBS-DR box events.
+const pbsdrBoxScope = "pbsdr-box"
+
+// PBS-DR snapshot states.
+const (
+	PBSStateOK          = "ok"
+	PBSStateUnavailable = "unavailable"
+	PBSStateDegraded    = "degraded"
+)
+
+const (
+	defaultPBSDRBoxFillWarnPercent = 80.0
+	defaultPBSDRBoxFillCritPercent = 90.0
+)
+
+// PBSBoxSnapshot is the cached PBS-DR datastore aggregate the web layer renders. Sizes in BYTES.
+type PBSBoxSnapshot struct {
+	CapacityBytes int64
+	UsedBytes     int64
+	FillPercent   float64
+	FillBand      string // "" | ok | warning | critical
+	State         string // ok | unavailable | degraded
+	FetchedAt     time.Time
+}
+
+// NewPBSDRBoxChecker builds the checker (defaults 80/90 on invalid thresholds). Nothing seeded → the
+// first ok Check that finds a breach emits (born/persistent; the dispatcher's 1 h cooldown dedups a restart).
+func NewPBSDRBoxChecker(reader usageReader, fillWarn, fillCrit float64, onEvent EventNotifyFunc, logger *log.Logger) *PBSDRBoxChecker {
+	if fillWarn <= 0 || fillWarn >= 100 {
+		fillWarn = defaultPBSDRBoxFillWarnPercent
+	}
+	if fillCrit <= 0 || fillCrit > 100 || fillCrit <= fillWarn {
+		fillCrit = defaultPBSDRBoxFillCritPercent
+	}
+	if fillCrit <= fillWarn {
+		fillWarn, fillCrit = defaultPBSDRBoxFillWarnPercent, defaultPBSDRBoxFillCritPercent
+	}
+	c := &PBSDRBoxChecker{
+		reader: reader, fillWarn: fillWarn, fillCrit: fillCrit,
+		onEvent: onEvent, logger: logger, now: time.Now,
+	}
+	logger.Printf("[INFO] PBS-DR box checker initialized: fill warn=%.0f%% crit=%.0f%%, refresh %s", fillWarn, fillCrit, boxFetchInterval)
+	return c
+}
+
+// Check runs on the 60 s sweep; fetches (throttled) and emits on each fill escalation.
+func (c *PBSDRBoxChecker) Check() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.haveSnap && c.now().Sub(c.lastFetch) < boxFetchInterval {
+		return // throttle — serve the cache
+	}
+	c.lastFetch = c.now() // updated even on failure → retried once per window, not per sweep
+
+	usage, err := c.reader.Usage(context.Background())
+	if err != nil {
+		if errors.Is(err, tenantsync.ErrUsageUnsupported) {
+			// EXPECTED pre-update condition — a distinct "unavailable" state, NOT degraded, NO alert.
+			c.snap = PBSBoxSnapshot{State: PBSStateUnavailable, FetchedAt: c.now()}
+			c.haveSnap = true
+			if !c.unavailLogged {
+				c.logger.Printf("[INFO] PBS-DR box: usage op unavailable — endpoint tenantsync update (v1.2.0) pending; gauge shows n/a until then")
+				c.unavailLogged = true
+			}
+			return // fillBand untouched — an expected data gap never re-arms/transitions
+		}
+		c.logger.Printf("[WARN] PBS-DR box: usage read failed (keeping last snapshot): %v", err)
+		if c.haveSnap {
+			c.snap.State = PBSStateDegraded // serve last-known, visibly stale; NO band transition
+		}
+		return
+	}
+	c.unavailLogged = false // recovered from unavailable
+
+	snap := PBSBoxSnapshot{CapacityBytes: usage.Total, UsedBytes: usage.Used, State: PBSStateOK, FetchedAt: c.now()}
+	if usage.Total > 0 {
+		snap.FillPercent = float64(usage.Used) * 100 / float64(usage.Total)
+		snap.FillBand = bandForPercent(snap.FillPercent, c.fillWarn, c.fillCrit)
+	} else {
+		snap.State = PBSStateDegraded // zero total — guard the division
+	}
+	c.snap = snap
+	c.haveSnap = true
+
+	if snap.State != PBSStateOK {
+		return
+	}
+	c.logger.Printf("[INFO] PBS-DR box refreshed: %.1f%% full (%s of %s)", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
+
+	// FILL band (escalation-only; recovery re-arms because bandOK has rank 0).
+	if bandRank(snap.FillBand) > bandRank(c.fillBand) {
+		c.emitFill(snap, snap.FillBand)
+	}
+	c.fillBand = snap.FillBand
+}
+
+// Snapshot returns the cached aggregate + whether one exists (mutex copy-out). The web layer's only
+// source — it NEVER polls ep0 in the request path.
+func (c *PBSDRBoxChecker) Snapshot() (PBSBoxSnapshot, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.snap, c.haveSnap
+}
+
+// FillState exposes the current band (tests).
+func (c *PBSDRBoxChecker) FillState() string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return orUnknown(c.fillBand)
+}
+
+func (c *PBSDRBoxChecker) emitFill(snap PBSBoxSnapshot, band string) {
+	var severity, message string
+	switch band {
+	case bandCritical:
+		severity = "critical"
+		message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — approaching capacity; the offsite DR tier's retention is at risk; free space or grow the datastore",
+			snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
+	case bandWarning:
+		severity = "warning"
+		message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — the offsite DR datastore is filling",
+			snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
+	default:
+		return
+	}
+	details, _ := json.Marshal(map[string]any{
+		"scope": pbsdrBoxScope, "capacity_bytes": snap.CapacityBytes, "used_bytes": snap.UsedBytes,
+		"fill_percent": snap.FillPercent, "warn_percent": c.fillWarn, "crit_percent": c.fillCrit,
+	})
+	c.logger.Printf("[INFO] PBS-DR box fill: %.0f%% (%s)", snap.FillPercent, band)
+	// NO SaveEvent — customer-less scope; straight to the operator dispatcher.
+	if c.onEvent != nil {
+		c.onEvent(pbsdrBoxScope, "pbsdr_box_fill", severity, message, string(details), "hub")
+	}
+}
@@ -0,0 +1,160 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
+)
+
+// fakeUsage is a usageReader for tests: it returns a scripted usage/err and counts calls.
+type fakeUsage struct {
+	mu    sync.Mutex
+	usage tenantsync.BoxUsage
+	err   error
+	calls int
+}
+
+func (f *fakeUsage) Usage(_ context.Context) (tenantsync.BoxUsage, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls++
+	return f.usage, f.err
+}
+func (f *fakeUsage) set(u tenantsync.BoxUsage, err error) {
+	f.mu.Lock()
+	f.usage, f.err = u, err
+	f.mu.Unlock()
+}
+func (f *fakeUsage) nCalls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.calls }
+
+func usageBytes(totalGiB, usedGiB int64) tenantsync.BoxUsage {
+	return tenantsync.BoxUsage{Total: totalGiB * gib, Used: usedGiB * gib, Avail: (totalGiB - usedGiB) * gib}
+}
+
+// C1 — throttle: ≤1 ep0 usage call per 15-min window across an hour of 60 s sweeps (≈4, not ≈60).
+func TestPBSDRBox_Throttle(t *testing.T) {
+	f := &fakeUsage{usage: usageBytes(40, 8)}
+	var cur time.Time
+	c := NewPBSDRBoxChecker(f, 80, 90, noEvent, quietLog())
+	c.now = func() time.Time { return cur }
+	base := time.Now().UTC()
+	for i := 0; i < 60; i++ {
+		cur = base.Add(time.Duration(i) * 60 * time.Second)
+		c.Check()
+	}
+	if f.nCalls() < 3 || f.nCalls() > 5 {
+		t.Fatalf("usage op called %d times over an hour of 60 s sweeps, want ≈4 (throttled to 15 min)", f.nCalls())
+	}
+	snap, ok := c.Snapshot()
+	if !ok || snap.State != PBSStateOK || snap.CapacityBytes != 40*gib {
+		t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
+	}
+}
+
+// C2/C3 — fill bands: escalation-only, in-band no re-emit, recovery re-arm, pbsdr-box scope.
+// Red-proof (esc): remove the escalation-only guard → the C3 same-band case re-emits → fails.
+func TestPBSDRBox_FillBands(t *testing.T) {
+	f := &fakeUsage{usage: usageBytes(1000, 750)} // 75%
+	ev := &capturedBox{}
+	var cur time.Time
+	c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
+	c.now = func() time.Time { return cur }
+	base := time.Now().UTC()
+	cur = base
+	step := func(usedGiB int64) {
+		cur = cur.Add(16 * time.Minute)
+		f.set(usageBytes(1000, usedGiB), nil)
+		c.Check()
+	}
+
+	c.Check() // 75% → no emit
+	if len(ev.typ) != 0 {
+		t.Fatalf("75%% must not emit, got %v", ev.typ)
+	}
+	step(820) // 82% → one warning
+	if count(ev.typ, "pbsdr_box_fill") != 1 || ev.sev[0] != "warning" {
+		t.Fatalf("82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
+	}
+	step(850) // 85% same band → no re-emit
+	if count(ev.typ, "pbsdr_box_fill") != 1 {
+		t.Fatalf("same-band must NOT re-emit, got %d", count(ev.typ, "pbsdr_box_fill"))
+	}
+	step(920) // 92% → critical
+	if count(ev.typ, "pbsdr_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
+		t.Fatalf("92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
+	}
+	step(700) // 70% → recovery re-arm (silent)
+	if c.FillState() != bandOK {
+		t.Fatalf("recovery must re-arm to ok, got %s", c.FillState())
+	}
+	if count(ev.typ, "pbsdr_box_fill") != 2 {
+		t.Fatalf("recovery must be silent, got %d", count(ev.typ, "pbsdr_box_fill"))
+	}
+	step(920) // re-breach → emits again
+	if count(ev.typ, "pbsdr_box_fill") != 3 {
+		t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "pbsdr_box_fill"))
+	}
+	for _, cust := range ev.cust {
+		if cust != "pbsdr-box" {
+			t.Fatalf("every emit must carry the pbsdr-box scope, got %q", cust)
+		}
+	}
+}
+
+// C4 — usage-unsupported (ErrUsageUnsupported) → the distinct "unavailable" state, NOT degraded, NO
+// alert, NO band transition. Red-proof (unavail-band): let unavailable drive a band → alert fires → fails.
+func TestPBSDRBox_Unavailable(t *testing.T) {
+	f := &fakeUsage{err: tenantsync.ErrUsageUnsupported}
+	ev := &capturedBox{}
+	c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
+	c.Check()
+	snap, ok := c.Snapshot()
+	if !ok || snap.State != PBSStateUnavailable {
+		t.Fatalf("ErrUsageUnsupported must yield the 'unavailable' state, got %+v", snap)
+	}
+	if len(ev.typ) != 0 {
+		t.Fatalf("unavailable must NOT alert, got %v", ev.typ)
+	}
+	if c.FillState() != "unknown" {
+		t.Fatalf("unavailable must not set a fill band, got %s", c.FillState())
+	}
+}
+
+// C5 — exec error/timeout → last snapshot kept, marked degraded, NO band transition (missing ≠ 0%).
+func TestPBSDRBox_DegradedKeepsLast(t *testing.T) {
+	f := &fakeUsage{usage: usageBytes(1000, 920)} // 92% critical
+	ev := &capturedBox{}
+	var cur time.Time
+	c := NewPBSDRBoxChecker(f, 80, 90, ev.fn, quietLog())
+	c.now = func() time.Time { return cur }
+	cur = time.Now().UTC()
+
+	c.Check() // establish 92% critical + emit
+	if count(ev.typ, "pbsdr_box_fill") != 1 {
+		t.Fatalf("setup: want one critical emit, got %v", ev.typ)
+	}
+	before, _ := c.Snapshot()
+
+	f.set(tenantsync.BoxUsage{}, fmt.Errorf("ssh timeout"))
+	cur = cur.Add(16 * time.Minute)
+	c.Check()
+
+	after, ok := c.Snapshot()
+	if !ok || after.State != PBSStateDegraded {
+		t.Fatalf("a failed poll must mark degraded (keeping last), got %+v", after)
+	}
+	if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
+		t.Fatalf("a failed poll must keep the last-known values (before %d/%d, after %d/%d)",
+			before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
+	}
+	if c.FillState() != bandCritical {
+		t.Fatalf("a failed poll must NOT transition the band, got %s", c.FillState())
+	}
+	if count(ev.typ, "pbsdr_box_fill") != 1 {
+		t.Fatalf("a failed poll must not emit, got %d", count(ev.typ, "pbsdr_box_fill"))
+	}
+}