hub v0.64.0 — offsite pool-box aggregate: fill, oversubscription, per-customer bars, operator alert (R-5)

The operator sees the shared pool box's real state on the hub: total box fill vs
capacity, Σ(shared soft quotas) vs capacity (the oversubscription ratio), per-customer
usage/quota bars, and a box-level operator alert (fill % + oversub ratio) on the existing
dispatcher's operator channel. Per-customer fill alerts already existed; the box-level
aggregate was the gap. READ-ONLY against Hetzner (GET only).

Phase-0 probe (gate PASSED): the live pool box 611714 returns capacity via
storage_box_type.size (1 TiB / bx11) and usage via a stats object (size/size_data/
size_snapshots), all bytes; our token reads it (200).

- hetznerapi: additive StorageBoxType + StorageBoxStats on StorageBox (no existing field/
  method changed); fake carries them + a GetBoxCalls counter; golden decode test.
- monitor.OffsiteBoxChecker: OffsiteChecker-sibling for the box; fetch-throttled (1 GET/
  15min), cached BoxSnapshot, escalation-only + recovery re-arm. FILL (used/capacity 80/90)
  + OVERSUB (Σ shared+enabled quotas / capacity, 2.0x) — independent. Σ from the ConfigJSON
  Descriptor (offsite.ReadDescriptor, new), never the report echo; dedicated+disabled
  excluded. Scope "pool-box" -> operator channel only, no SaveEvent. Failed fetch keeps the
  last snapshot degraded; missing data never becomes 0% and never transitions a band.
- config: Alerting.OffsiteBoxFill{Warn,Crit}Percent + OffsiteOversubWarnRatio (80/90/2.0
  defaults; thresholds pending Viktor's ruling). Constructed in the HETZNER_TOKEN branch,
  60s sweep, snapshot handed to the web server.
- web: Offsite-tab panel (fill bar, Σ+ratio, per-customer usage/quota rows) + a compact
  dashboard tile; reads the cached snapshot only, never fetches; nil -> "not configured".

Tests: 10 new + 4 red-proofs (throttle, Σ filter, escalation-only, failed-fetch honesty),
all confirmed red then restored. go build/vet/test all pass; hub confirm gate OK.
This commit is contained in:
2026-07-17 20:15:34 +02:00
parent 85a14192e7
commit 4bb2df0dc4
17 changed files with 1034 additions and 17 deletions
+9 -5
View File
@@ -139,12 +139,16 @@ func (s *Server) handleOffsite(w http.ResponseWriter, r *http.Request) {
})
}
boxView, custRows := s.offsiteBoxData() // R-5 pool-box aggregate panel
data := map[string]interface{}{
"Endpoints": cards,
"HasEndpoints": len(cards) > 0,
"Peers": rows,
"CSRFToken": s.getCSRFToken(r),
"Flash": r.URL.Query().Get("flash"),
"Endpoints": cards,
"HasEndpoints": len(cards) > 0,
"Peers": rows,
"OffsiteBox": boxView,
"OffsiteCusts": custRows,
"CSRFToken": s.getCSRFToken(r),
"Flash": r.URL.Query().Get("flash"),
}
if err := s.templates.ExecuteTemplate(w, "offsite.html", data); err != nil {
s.logger.Printf("[ERROR] offsite.html template: %v", err)
+193
View File
@@ -0,0 +1,193 @@
package web
// Offsite pool-box aggregate surfaces (v0.64.0, R-5): the Offsite-tab panel + the Dashboard tile.
// The web layer NEVER fetches from Hetzner — it reads the checker's cached snapshot (s.offsiteBox) and
// composes per-customer rows from state the hub already holds (quota from the ConfigJSON Descriptor,
// usage from the controller report echo).
import (
"encoding/json"
"fmt"
"sort"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
)
// offsiteBoxView is the pool-box panel model. Every display value is composed handler-side; band
// strings ("ok"/"warning"/"critical") drive the bar color via templates.
type offsiteBoxView struct {
Configured bool // s.offsiteBox wired (HETZNER_TOKEN + box id present)
Pending bool // configured but the first refresh hasn't landed yet
Degraded bool // last fetch failed / zero capacity — values shown stale
CapacityStr string
UsedStr string
DataStr string
SnapshotStr string
FillPercent float64
FillBand string
SumQuotaStr string
RatioStr string // "1.17×"
OversubBand string
BoxType string
FetchedAt time.Time
}
// offsiteCustomerRow is one per-customer usage/quota row under the panel.
type offsiteCustomerRow struct {
CustomerName string
Dedicated bool
QuotaStr string // "" for dedicated
UsageStr string // "" when NoUsage
UsageBytes int64 // sort key
UsagePercent float64 // usage/quota (shared only)
UsageBand string // ok|warning|critical for the bar
HasBar bool // shared + quota>0 + has usage → render a bar
NoUsage bool // no offsite object in the report yet
}
// fmtBytesGB renders bytes as GB (one decimal) below 1 TB, TB (two decimals) above.
func fmtBytesGB(b int64) string {
if b >= 1<<40 {
return fmt.Sprintf("%.2f TB", float64(b)/float64(int64(1)<<40))
}
return fmt.Sprintf("%.1f GB", float64(b)/float64(int64(1)<<30))
}
// pctBand maps a percent to a band with the given warn/crit (web-side twin of monitor.bandForPercent —
// unexported there; a 3-line copy keeps web from depending on monitor internals).
func pctBand(pct, warn, crit float64) string {
switch {
case pct >= crit:
return "critical"
case pct >= warn:
return "warning"
default:
return "ok"
}
}
// offsiteUsageBytes reads repo_size_bytes from a customer's report `offsite` object — the ONLY place the
// live repo size exists. ok=false when the report carries no offsite object (never reported). Quota is
// NEVER taken from here (the report echo would undercount promises) — only usage. Twin of monitor's
// parseOffsite, scoped to the one field the rows need.
func offsiteUsageBytes(reportJSON string) (int64, bool) {
var r struct {
Offsite *struct {
RepoSizeBytes int64 `json:"repo_size_bytes"`
} `json:"offsite"`
}
if json.Unmarshal([]byte(reportJSON), &r) != nil || r.Offsite == nil {
return 0, false
}
return r.Offsite.RepoSizeBytes, true
}
// offsiteTile is the compact Dashboard tile: fill% + ratio, band-colored, linking to /offsite.
type offsiteTile struct {
FillPercent float64
RatioStr string
Band string // worst of fill/oversub → tile color
Degraded bool
}
// offsiteBoxTile builds the Dashboard tile, or nil when there is nothing to show (no provider, or the
// first refresh hasn't landed) — the caller then renders NO tile (never a zeroed one).
func (s *Server) offsiteBoxTile() *offsiteTile {
if s.offsiteBox == nil {
return nil
}
snap, ok := s.offsiteBox()
if !ok {
return nil // configured but no snapshot yet — absent, not a zeroed tile
}
band := "ok"
switch {
case snap.FillBand == "critical":
band = "critical"
case snap.FillBand == "warning" || snap.OversubBand == "warning":
band = "warning"
}
return &offsiteTile{
FillPercent: snap.FillPercent,
RatioStr: fmt.Sprintf("%.2f×", snap.Ratio),
Band: band,
Degraded: snap.Degraded,
}
}
// offsiteBoxData builds the panel view + per-customer rows. Nil provider → Configured:false (the panel
// renders the honest "not configured"). Never fetches.
func (s *Server) offsiteBoxData() (offsiteBoxView, []offsiteCustomerRow) {
if s.offsiteBox == nil {
return offsiteBoxView{Configured: false}, nil
}
view := offsiteBoxView{Configured: true}
if snap, ok := s.offsiteBox(); ok {
view.Pending = false
view.Degraded = snap.Degraded
view.CapacityStr = fmtBytesGB(snap.CapacityBytes)
view.UsedStr = fmtBytesGB(snap.UsedBytes)
view.DataStr = fmtBytesGB(snap.DataBytes)
view.SnapshotStr = fmtBytesGB(snap.SnapshotBytes)
view.FillPercent = snap.FillPercent
view.FillBand = snap.FillBand
view.SumQuotaStr = fmt.Sprintf("%d GB", snap.SumQuotaGB)
view.RatioStr = fmt.Sprintf("%.2f×", snap.Ratio)
view.OversubBand = snap.OversubBand
view.BoxType = snap.BoxType
view.FetchedAt = snap.FetchedAt
} else {
view.Pending = true // configured, first fetch pending
}
return view, s.offsiteCustomerRows()
}
// offsiteCustomerRows composes the per-customer rows for every ENABLED-offsite customer: quota from the
// authoritative ConfigJSON Descriptor, usage from the report echo. Shared customers get a usage/quota
// bar; dedicated customers are listed without one. Sorted by usage descending.
func (s *Server) offsiteCustomerRows() []offsiteCustomerRow {
cfgs, err := s.store.ListCustomerConfigs()
if err != nil {
s.logger.Printf("[WARN] offsite box: list configs: %v", err)
return nil
}
usage := map[string]int64{}
hasReport := map[string]bool{}
if custs, cerr := s.store.GetCustomers(); cerr == nil {
for _, c := range custs {
if b, ok := offsiteUsageBytes(c.ReportJSON); ok {
usage[c.CustomerID], hasReport[c.CustomerID] = b, true
}
}
}
var rows []offsiteCustomerRow
for _, cfg := range cfgs {
d, derr := offsite.ReadDescriptor(cfg.ConfigJSON)
if derr != nil || d == nil || !d.Enabled {
continue // only customers actually using an offsite tier appear
}
name := cfg.CustomerName
if name == "" {
name = cfg.CustomerID
}
row := offsiteCustomerRow{CustomerName: name, Dedicated: d.Type == "dedicated"}
if d.Type == "shared" && d.QuotaGB > 0 {
row.QuotaStr = fmt.Sprintf("%d GB", d.QuotaGB)
}
if hasReport[cfg.CustomerID] {
row.UsageBytes = usage[cfg.CustomerID]
row.UsageStr = fmtBytesGB(row.UsageBytes)
if d.Type == "shared" && d.QuotaGB > 0 {
row.UsagePercent = float64(row.UsageBytes) * 100 / float64(int64(d.QuotaGB)<<30)
row.UsageBand = pctBand(row.UsagePercent, 90, 95) // matches the per-customer OffsiteChecker bands
row.HasBar = true
}
} else {
row.NoUsage = true
}
rows = append(rows, row)
}
sort.SliceStable(rows, func(i, j int) bool { return rows[i].UsageBytes > rows[j].UsageBytes })
return rows
}
@@ -0,0 +1,85 @@
package web
import (
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/monitor"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
const gib = int64(1) << 30
// Not configured (no HETZNER_TOKEN) → honest "not configured", never an error or fake zeros.
// (Uses renderOffsite from offsite_test.go — the real handler via httptest.)
func TestOffsiteBoxPanel_NotConfigured(t *testing.T) {
s, _ := newRenderServer(t)
// s.offsiteBox left nil
body := renderOffsite(t, s)
if !strings.Contains(body, "Offsite pool metrics not configured") {
t.Fatalf("unconfigured panel must say 'not configured':\n%s", body)
}
if strings.Contains(body, "0.0 GB") {
t.Fatal("unconfigured panel must not render fake zeros")
}
}
// With data + per-customer rows: the panel renders capacity/fill/ratio; a shared customer with a report
// shows usage, a customer with NO report shows 'no usage reported yet' (never an asserted 0 GB).
func TestOffsiteBoxPanel_WithData(t *testing.T) {
s, st := newRenderServer(t)
s.SetOffsiteBox(func() (monitor.BoxSnapshot, bool) {
return monitor.BoxSnapshot{
CapacityBytes: 1 << 40, UsedBytes: 300 * gib, DataBytes: 300 * gib,
FillPercent: 27, FillBand: "ok", SumQuotaGB: 1200, Ratio: 1.17, OversubBand: "ok",
BoxType: "bx11", FetchedAt: time.Now().UTC(),
}, true
})
// "acme": shared, enabled, 500 GB, WITH a report (200 GiB used). "newbie": shared, enabled, no report.
saveCfg(t, st, "acme", `{"offsite":{"enabled":true,"type":"shared","quota_gb":500}}`)
saveCfg(t, st, "newbie", `{"offsite":{"enabled":true,"type":"shared","quota_gb":300}}`)
if err := st.SaveReport("acme", []byte(`{"customer_id":"acme","offsite":{"enabled":true,"repo_size_bytes":`+itoa(200*gib)+`}}`)); err != nil {
t.Fatal(err)
}
body := renderOffsite(t, s)
for _, want := range []string{"Offsite pool box", "1.00 TB", "1.17×", "acme", "newbie", "no usage reported yet"} {
if !strings.Contains(body, want) {
t.Fatalf("panel missing %q:\n%s", want, body)
}
}
if strings.Contains(body, "not configured") {
t.Fatal("a configured panel must not show the not-configured message")
}
}
func saveCfg(t *testing.T, st *store.Store, id, cfgJSON string) {
t.Helper()
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: id, CustomerName: id, APIKey: "k-" + id, RetrievalPassword: "p", ConfigJSON: cfgJSON}); err != nil {
t.Fatalf("save config %s: %v", id, err)
}
}
// itoa avoids importing strconv just for the report body.
func itoa(v int64) string {
if v == 0 {
return "0"
}
neg := v < 0
if neg {
v = -v
}
var b [20]byte
i := len(b)
for v > 0 {
i--
b[i] = byte('0' + v%10)
v /= 10
}
if neg {
i--
b[i] = '-'
}
return string(b[i:])
}
+6 -2
View File
@@ -92,11 +92,15 @@ func TestTemplates_DashboardCriticalBadge(t *testing.T) {
EventErrors int
EventWarnings int
}
data := []dashboardCustomer{{
// v0.64.0: dashboard.html now takes {Customers, OffsiteTile}; OffsiteTile nil → no tile rendered.
data := struct {
Customers []dashboardCustomer
OffsiteTile any
}{Customers: []dashboardCustomer{{
CustomerSummary: store.CustomerSummary{CustomerID: "c1", CustomerName: "Acme", ReceivedAt: time.Now()},
OverallStatus: "ok", BackupAge: "",
EventCriticals: 2, EventErrors: 1, EventWarnings: 0,
}}
}}}
var buf bytes.Buffer
if err := s.templates.ExecuteTemplate(&buf, "dashboard.html", data); err != nil {
t.Fatalf("render dashboard.html: %v", err)
+13 -1
View File
@@ -19,6 +19,7 @@ import (
"gitea.dooplex.hu/admin/felhom-hub/internal/claim"
"gitea.dooplex.hu/admin/felhom-hub/internal/gitea"
"gitea.dooplex.hu/admin/felhom-hub/internal/intent"
"gitea.dooplex.hu/admin/felhom-hub/internal/monitor"
"gitea.dooplex.hu/admin/felhom-hub/internal/poke"
"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
@@ -66,6 +67,7 @@ type Server struct {
assetsMgr *assets.Manager
gitea *gitea.Client // optional; enables the Day-0 artifact version dropdowns
offsite *offsite.Provisioner // optional; enables Hetzner offsite provisioning (SLICE 1)
offsiteBox func() (monitor.BoxSnapshot, bool) // optional (v0.64.0, R-5); the pool-box aggregate snapshot accessor
tenantsync tenancyProvisioner // optional; enables PBS DR tier provisioning (web/pbsdr.go)
claimEngine *claim.Engine // optional; enables the customer-claim resend button (v0.50.0)
// intentHub (v0.58.0, Direction-2 immediate-sync) is Bumped by every operator-intent handler
@@ -181,6 +183,11 @@ func (s *Server) SetAssetManager(am *assets.Manager) {
// offsite enabled returns an error (offsite not configured on this hub).
func (s *Server) SetOffsiteProvisioner(p *offsite.Provisioner) { s.offsite = p }
// SetOffsiteBox wires the pool-box aggregate snapshot accessor (v0.64.0, R-5): the checker's cached
// snapshot, read on the Offsite tab + the Dashboard tile. nil (no HETZNER_TOKEN/box id) → both render an
// honest "not configured". The web layer NEVER fetches from Hetzner — it only reads this cache.
func (s *Server) SetOffsiteBox(fn func() (monitor.BoxSnapshot, bool)) { s.offsiteBox = fn }
// SetClaimEngine wires the customer-claim code engine for the Setup-tab resend button (v0.50.0).
func (s *Server) SetClaimEngine(e *claim.Engine) { s.claimEngine = e }
@@ -742,8 +749,13 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
data = append(data, dc)
}
payload := struct {
Customers []dashboardCustomer
OffsiteTile *offsiteTile // R-5: nil → no tile rendered
}{Customers: data, OffsiteTile: s.offsiteBoxTile()}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := s.templates.ExecuteTemplate(w, "dashboard.html", data); err != nil {
if err := s.templates.ExecuteTemplate(w, "dashboard.html", payload); err != nil {
s.logger.Printf("[ERROR] Template render: %v", err)
}
}
+10 -2
View File
@@ -22,7 +22,15 @@
</nav>
</header>
{{if not .}}
{{with .OffsiteTile}}
<a href="/offsite" class="offsite-tile offsite-tile-{{.Band}}">
<span class="offsite-tile-label">Offsite pool</span>
<span class="offsite-tile-val">{{formatFloat .FillPercent}}% &middot; {{.RatioStr}}</span>
{{if .Degraded}}<span class="offsite-tile-stale">stale</span>{{end}}
</a>
{{end}}
{{if not .Customers}}
<div class="empty-state">
<p>No customer reports received yet.</p>
<p class="hint">Configure <code>hub.enabled: true</code> in customer controller.yaml to start receiving reports.</p>
@@ -44,7 +52,7 @@
</tr>
</thead>
<tbody>
{{range .}}
{{range .Customers}}
<tr class="status-{{.OverallStatus}}" onclick="window.location='/customers/{{.CustomerID}}'">
<td class="customer-name">
<span class="status-dot status-dot-{{statusColor .OverallStatus}}"></span>
+39
View File
@@ -34,6 +34,45 @@
<div class="flash flash-success">Endpoint deleted.</div>
{{end}}
<!-- R-5 (v0.64.0): shared pool-box aggregate — total fill, oversubscription, per-customer usage -->
<section class="card" style="margin-bottom: 1.5rem;">
<h3 style="margin: 0 0 0.75rem;">Offsite pool box</h3>
{{if not .OffsiteBox.Configured}}
<p class="text-muted" style="font-size: 0.9em;">Offsite pool metrics not configured (no Hetzner token / pool box id on this hub).</p>
{{else if .OffsiteBox.Pending}}
<p class="text-muted" style="font-size: 0.9em;">Pool metrics loading — the first refresh has not landed yet.</p>
{{else}}
<table class="detail-table">
<tr><th style="width: 12rem;">Box</th><td>{{.OffsiteBox.BoxType}}</td></tr>
<tr><th>Capacity</th><td>{{.OffsiteBox.CapacityStr}}</td></tr>
<tr><th>Used</th><td>{{.OffsiteBox.UsedStr}} &middot; {{formatFloat .OffsiteBox.FillPercent}}% full <span class="text-muted">(data {{.OffsiteBox.DataStr}} + snapshots {{.OffsiteBox.SnapshotStr}})</span></td></tr>
</table>
<div class="bar" style="margin: 0.4rem 0 0.9rem;"><div class="bar-fill bar-{{.OffsiteBox.FillBand}}" style="width: {{formatFloat .OffsiteBox.FillPercent}}%;"></div></div>
<table class="detail-table">
<tr><th style="width: 12rem;">&Sigma; shared soft quotas</th><td>{{.OffsiteBox.SumQuotaStr}}</td></tr>
<tr><th>Oversubscription</th><td><span class="status-badge status-badge-{{if eq .OffsiteBox.OversubBand "warning"}}warn{{else}}ok{{end}}">{{.OffsiteBox.RatioStr}}</span> <span class="text-muted">&Sigma;(shared quotas) &divide; capacity</span></td></tr>
<tr><th>Fetched</th><td>{{timeAgo .OffsiteBox.FetchedAt}}{{if .OffsiteBox.Degraded}} <span class="status-badge status-badge-warn">STALE — last refresh failed</span>{{end}}</td></tr>
</table>
{{if .OffsiteCusts}}
<h4 style="margin: 1rem 0 0.5rem; font-size: 0.85rem; color: var(--text-3); text-transform: uppercase; letter-spacing: 0.03em;">Per-customer usage</h4>
<table class="container-table">
<thead><tr><th>Customer</th><th>Usage</th><th>Quota</th><th style="width: 35%;"></th></tr></thead>
<tbody>
{{range .OffsiteCusts}}
<tr>
<td>{{.CustomerName}}{{if .Dedicated}} <span class="text-muted">(dedicated)</span>{{end}}</td>
<td>{{if .NoUsage}}<span class="text-muted">no usage reported yet</span>{{else}}{{.UsageStr}}{{end}}</td>
<td>{{if .Dedicated}}<span class="text-muted">dedicated</span>{{else if .QuotaStr}}{{.QuotaStr}}{{else}}<span class="text-muted"></span>{{end}}</td>
<td>{{if .HasBar}}<div class="bar"><div class="bar-fill bar-{{.UsageBand}}" style="width: {{formatFloat .UsagePercent}}%;"></div></div>{{end}}</td>
</tr>
{{end}}
</tbody>
</table>
{{end}}
{{end}}
</section>
{{if .HasEndpoints}}
{{range .Endpoints}}
<section class="card" style="margin-bottom: 1.5rem;"
+19
View File
@@ -298,6 +298,25 @@ header h1 {
border-radius: var(--radius);
transition: width 0.3s ease;
}
/* Fill bands (v0.64.0, R-5) — exception color: neutral when nominal, amber ≥ warn, red ≥ crit; no green. */
.bar-fill.bar-ok { background: var(--text-3); }
.bar-fill.bar-warning { background: var(--warn); }
.bar-fill.bar-critical { background: var(--crit); }
/* Offsite pool dashboard tile (v0.64.0, R-5) — compact, band-colored, links to /offsite. */
.offsite-tile {
display: inline-flex; align-items: baseline; gap: 0.5rem;
padding: 0.4rem 0.8rem; margin-bottom: 1rem;
border: 1px solid var(--line); border-radius: var(--radius);
text-decoration: none; font-size: 0.85rem;
}
.offsite-tile-label { color: var(--text-3); text-transform: uppercase; font-size: 0.72rem; letter-spacing: 0.03em; }
.offsite-tile-val { color: var(--text-1); font-weight: 600; }
.offsite-tile-stale { color: var(--warn); font-size: 0.72rem; }
.offsite-tile-warning { border-color: var(--warn); }
.offsite-tile-warning .offsite-tile-val { color: var(--warn); }
.offsite-tile-critical { border-color: var(--crit); }
.offsite-tile-critical .offsite-tile-val { color: var(--crit); }
/* Container table */
.container-table {