hub v0.64.0 — offsite pool-box aggregate: fill, oversubscription, per-customer bars, operator alert (R-5)

The operator sees the shared pool box's real state on the hub: total box fill vs capacity, Σ(shared soft quotas) vs capacity (the oversubscription ratio), per-customer usage/quota bars, and a box-level operator alert (fill % + oversub ratio) on the existing dispatcher's operator channel. Per-customer fill alerts already existed; the box-level aggregate was the gap. READ-ONLY against Hetzner (GET only). Phase-0 probe (gate PASSED): the live pool box 611714 returns capacity via storage_box_type.size (1 TiB / bx11) and usage via a stats object (size/size_data/ size_snapshots), all bytes; our token reads it (200). - hetznerapi: additive StorageBoxType + StorageBoxStats on StorageBox (no existing field/ method changed); fake carries them + a GetBoxCalls counter; golden decode test. - monitor.OffsiteBoxChecker: OffsiteChecker-sibling for the box; fetch-throttled (1 GET/ 15min), cached BoxSnapshot, escalation-only + recovery re-arm. FILL (used/capacity 80/90) + OVERSUB (Σ shared+enabled quotas / capacity, 2.0x) — independent. Σ from the ConfigJSON Descriptor (offsite.ReadDescriptor, new), never the report echo; dedicated+disabled excluded. Scope "pool-box" -> operator channel only, no SaveEvent. Failed fetch keeps the last snapshot degraded; missing data never becomes 0% and never transitions a band. - config: Alerting.OffsiteBoxFill{Warn,Crit}Percent + OffsiteOversubWarnRatio (80/90/2.0 defaults; thresholds pending Viktor's ruling). Constructed in the HETZNER_TOKEN branch, 60s sweep, snapshot handed to the web server. - web: Offsite-tab panel (fill bar, Σ+ratio, per-customer usage/quota rows) + a compact dashboard tile; reads the cached snapshot only, never fetches; nil -> "not configured". Tests: 10 new + 4 red-proofs (throttle, Σ filter, escalation-only, failed-fetch honesty), all confirmed red then restored. go build/vet/test all pass; hub confirm gate OK.
2026-07-17 20:15:34 +02:00
parent 85a14192e7
commit 4bb2df0dc4
17 changed files with 1034 additions and 17 deletions
@@ -23,10 +23,12 @@ type Fake struct {
 	BoxResetCalls      int
 	DeletedSubaccounts int
 	DeletedBoxes       int
+	GetBoxCalls        int // v0.64.0: the fetch-throttle assertion (Scenario A) counts these

 	// Failure injection.
 	FailCreate error // if set, CreateSubaccount/CreateStorageBox return this error
 	FailAction bool  // if set, created actions come back status "error" (WaitAction fails)
+	FailGetBox error // v0.64.0: if set, GetStorageBox returns this (the box-poll failure path, Scenario D)
 }

 // NewFake returns an empty Fake.
@@ -145,6 +147,10 @@ func (f *Fake) ListStorageBoxes(_ context.Context, sel string) ([]StorageBox, er
 func (f *Fake) GetStorageBox(_ context.Context, boxID int64) (StorageBox, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	f.GetBoxCalls++
+	if f.FailGetBox != nil {
+		return StorageBox{}, f.FailGetBox
+	}
 	b, ok := f.Boxes[boxID]
 	if !ok {
 		return StorageBox{}, fmt.Errorf("hetznerapi(fake): box %d not found", boxID)
@@ -62,6 +62,29 @@ type StorageBox struct {
 	Status   string `json:"status"` // initializing | active | …
 	Server   string `json:"server"` // uXXXXXX.your-storagebox.de
 	Labels   map[string]string `json:"labels"`
+
+	// StorageBoxType + Stats added v0.64.0 (R-5 pool-box aggregate). Shape pinned by the Phase-0 probe
+	// of the live pool box 611714 (2026-07-17): GET /storage_boxes/{id} returns a `storage_box_type`
+	// object whose `size` is the CAPACITY in bytes (1099511627776 = 1 TiB for bx11), and a `stats` object
+	// carrying total/data/snapshot usage in bytes. Additive — no existing field or method changed.
+	StorageBoxType StorageBoxType  `json:"storage_box_type"`
+	Stats          StorageBoxStats `json:"stats"`
+}
+
+// StorageBoxType is the box's plan; its Size is the CAPACITY in bytes (never use stats.size for capacity).
+type StorageBoxType struct {
+	ID          int64  `json:"id"`
+	Name        string `json:"name"`        // e.g. "bx11"
+	Description string `json:"description"` // e.g. "BX11"
+	Size        int64  `json:"size"`        // capacity in BYTES
+}
+
+// StorageBoxStats is the box's live usage, all in BYTES (probe-confirmed): total, the data portion, and
+// the snapshot portion (Size == SizeData + SizeSnapshots on the live box).
+type StorageBoxStats struct {
+	Size          int64 `json:"size"`           // total used bytes
+	SizeData      int64 `json:"size_data"`      // data bytes
+	SizeSnapshots int64 `json:"size_snapshots"` // snapshot bytes
 }

 // CreateSubaccountRequest — POST /storage_boxes/{box}/subaccounts. Password satisfies the 4-class policy.
@@ -0,0 +1,45 @@
+package hetznerapi
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// goldenPoolBox is the (redacted, non-secret) response captured from the Phase-0 probe of the live pool
+// box 611714 (2026-07-17): GET /storage_boxes/{id}. The stats/capacity extension must decode it EXACTLY.
+const goldenPoolBox = `{
+  "storage_box": {
+    "id": 611714,
+    "username": "u629488",
+    "name": "storage-box-pool-1",
+    "status": "active",
+    "server": "u629488.your-storagebox.de",
+    "storage_box_type": { "id": 1333, "name": "bx11", "description": "BX11", "size": 1099511627776 },
+    "stats": { "size": 2746220544, "size_data": 2746220544, "size_snapshots": 0 }
+  }
+}`
+
+// TestStorageBox_DecodesProbeShape pins the type extension to the live API shape: capacity is the box
+// TYPE's size (1 TiB), and stats carries the total/data/snapshot split, all in bytes.
+func TestStorageBox_DecodesProbeShape(t *testing.T) {
+	var out struct {
+		StorageBox StorageBox `json:"storage_box"`
+	}
+	if err := json.Unmarshal([]byte(goldenPoolBox), &out); err != nil {
+		t.Fatalf("decode golden pool-box: %v", err)
+	}
+	b := out.StorageBox
+	if b.ID != 611714 || b.Name != "storage-box-pool-1" || b.Status != "active" {
+		t.Fatalf("base fields wrong: %+v", b)
+	}
+	if b.StorageBoxType.Name != "bx11" || b.StorageBoxType.Size != 1099511627776 {
+		t.Fatalf("capacity (storage_box_type.size) wrong: %+v", b.StorageBoxType)
+	}
+	if b.Stats.Size != 2746220544 || b.Stats.SizeData != 2746220544 || b.Stats.SizeSnapshots != 0 {
+		t.Fatalf("stats wrong: %+v", b.Stats)
+	}
+	// Sanity: the split sums to the total (the invariant the live box holds).
+	if b.Stats.SizeData+b.Stats.SizeSnapshots != b.Stats.Size {
+		t.Fatalf("data+snapshots (%d) != total (%d)", b.Stats.SizeData+b.Stats.SizeSnapshots, b.Stats.Size)
+	}
+}