hub v0.65.0 — PBS DR storage visibility (ep0 usage op) + Offsite tab split + dual dashboard gauges (R-5)

Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.

Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.

- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
  path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
  cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
  no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
  neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
  60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
  the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).

Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
This commit is contained in:
2026-07-17 21:13:30 +02:00
parent 3588a31b78
commit 7f11cfb36c
19 changed files with 856 additions and 22 deletions
+3 -1
View File
@@ -139,7 +139,8 @@ func (s *Server) handleOffsite(w http.ResponseWriter, r *http.Request) {
})
}
boxView, custRows := s.offsiteBoxData() // R-5 pool-box aggregate panel
boxView, custRows := s.offsiteBoxData() // R-5 restic pool-box aggregate panel (Restic tab)
pbsView := s.pbsdrBoxData() // R-5 v0.65.0 PBS-DR datastore panel (PBS DR tab)
data := map[string]interface{}{
"Endpoints": cards,
@@ -147,6 +148,7 @@ func (s *Server) handleOffsite(w http.ResponseWriter, r *http.Request) {
"Peers": rows,
"OffsiteBox": boxView,
"OffsiteCusts": custRows,
"PBSBox": pbsView,
"CSRFToken": s.getCSRFToken(r),
"Flash": r.URL.Query().Get("flash"),
}
+4 -2
View File
@@ -49,8 +49,10 @@ func TestOffsiteBoxPanel_WithData(t *testing.T) {
t.Fatalf("panel missing %q:\n%s", want, body)
}
}
if strings.Contains(body, "not configured") {
t.Fatal("a configured panel must not show the not-configured message")
// The RESTIC panel is configured — its specific not-configured message must be absent. (The PBS DR
// panel legitimately shows its own "not configured" here since s.pbsdrBox is unset.)
if strings.Contains(body, "Offsite pool metrics not configured") {
t.Fatal("a configured restic panel must not show the not-configured message")
}
}
+89
View File
@@ -0,0 +1,89 @@
package web
// PBS DR datastore fill surfaces (v0.65.0, R-5): the Offsite "PBS DR" tab panel + the Dashboard PBS
// gauge. The web layer reads only the checker's cached snapshot (s.pbsdrBox) — it never polls ep0. The
// snapshot carries its own state (ok / unavailable / degraded) so the UI renders each honestly.
import (
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/monitor"
)
// pbsdrBoxView is the PBS-DR panel model.
type pbsdrBoxView struct {
Configured bool // s.pbsdrBox wired (tenantsync client present)
Pending bool // configured but the first poll hasn't landed yet
State string // ok | unavailable | degraded
Unavailable bool // endpoint script predates the usage op (≤ v1.1.0) — expected pre-update
Degraded bool // last poll failed — values shown stale
HasFill bool // numbers available (ok or degraded-with-last-known)
CapacityStr string
UsedStr string
FillPercent float64
FillBand string
FetchedAt time.Time
}
// pbsdrTile is the compact Dashboard PBS gauge: fill %, band-colored; "n/a" when unavailable.
type pbsdrTile struct {
Unavailable bool
FillPercent float64
Band string
Degraded bool
}
// pbsdrBoxData builds the PBS-DR panel view. Nil provider → Configured:false ("not configured"). Never polls.
func (s *Server) pbsdrBoxData() pbsdrBoxView {
if s.pbsdrBox == nil {
return pbsdrBoxView{Configured: false}
}
view := pbsdrBoxView{Configured: true}
snap, ok := s.pbsdrBox()
if !ok {
view.Pending = true // configured, first poll pending
return view
}
view.State = snap.State
view.FetchedAt = snap.FetchedAt
switch snap.State {
case monitor.PBSStateUnavailable:
view.Unavailable = true
case monitor.PBSStateDegraded:
view.Degraded = true
if snap.CapacityBytes > 0 { // last-known values persist across a degraded poll
view.HasFill = true
view.CapacityStr = fmtBytesGB(snap.CapacityBytes)
view.UsedStr = fmtBytesGB(snap.UsedBytes)
view.FillPercent = snap.FillPercent
view.FillBand = snap.FillBand
}
default: // ok
view.HasFill = true
view.CapacityStr = fmtBytesGB(snap.CapacityBytes)
view.UsedStr = fmtBytesGB(snap.UsedBytes)
view.FillPercent = snap.FillPercent
view.FillBand = snap.FillBand
}
return view
}
// pbsdrBoxTile builds the Dashboard PBS gauge, or nil when there is nothing to show (no provider, or the
// first poll hasn't landed). Unavailable → a neutral "n/a" gauge (never a fake 0%).
func (s *Server) pbsdrBoxTile() *pbsdrTile {
if s.pbsdrBox == nil {
return nil
}
snap, ok := s.pbsdrBox()
if !ok {
return nil
}
if snap.State == monitor.PBSStateUnavailable {
return &pbsdrTile{Unavailable: true}
}
return &pbsdrTile{
FillPercent: snap.FillPercent,
Band: snap.FillBand,
Degraded: snap.State == monitor.PBSStateDegraded,
}
}
+60
View File
@@ -0,0 +1,60 @@
package web
import (
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/monitor"
)
// PBS DR panel — configured + ok: renders the datastore, capacity/used, fill bar.
func TestPBSDRPanel_OK(t *testing.T) {
s, _ := newRenderServer(t)
s.SetPBSDRBox(func() (monitor.PBSBoxSnapshot, bool) {
return monitor.PBSBoxSnapshot{
CapacityBytes: 40 * gib, UsedBytes: 8 * gib, FillPercent: 20, FillBand: "ok",
State: monitor.PBSStateOK, FetchedAt: time.Now().UTC(),
}, true
})
body := renderOffsite(t, s)
for _, want := range []string{"PBS DR datastore", "felhom-offsite", "40.0 GB", "8.0 GB", "20% full"} {
if !strings.Contains(body, want) {
t.Fatalf("PBS DR panel missing %q", want)
}
}
}
// PBS DR panel — unavailable (endpoint script ≤ v1.1.0): the honest pending-update message, NOT a fake 0%.
func TestPBSDRPanel_Unavailable(t *testing.T) {
s, _ := newRenderServer(t)
s.SetPBSDRBox(func() (monitor.PBSBoxSnapshot, bool) {
return monitor.PBSBoxSnapshot{State: monitor.PBSStateUnavailable, FetchedAt: time.Now().UTC()}, true
})
body := renderOffsite(t, s)
if !strings.Contains(body, "usage not available") || !strings.Contains(body, "v1.2.0") {
t.Fatalf("unavailable PBS panel must show the pending-update message:\n%s", pbsSection(body))
}
// Never a fake fill for the unavailable box.
if strings.Contains(pbsSection(body), "% full") {
t.Fatal("unavailable PBS panel must not render a fill percentage")
}
}
// PBS DR panel — not configured (no tenantsync client).
func TestPBSDRPanel_NotConfigured(t *testing.T) {
s, _ := newRenderServer(t)
// s.pbsdrBox left nil
body := renderOffsite(t, s)
if !strings.Contains(body, "PBS DR metrics not configured") {
t.Fatalf("unconfigured PBS panel must say 'not configured'")
}
}
// pbsSection returns the slice of the body from the PBS DR datastore heading onward (for scoped asserts).
func pbsSection(body string) string {
if i := strings.Index(body, "PBS DR datastore"); i >= 0 {
return body[i:]
}
return body
}
+2 -1
View File
@@ -92,10 +92,11 @@ func TestTemplates_DashboardCriticalBadge(t *testing.T) {
EventErrors int
EventWarnings int
}
// v0.64.0: dashboard.html now takes {Customers, OffsiteTile}; OffsiteTile nil → no tile rendered.
// v0.65.0: dashboard.html takes {Customers, OffsiteTile, PBSTile}; nil tiles → no gauges rendered.
data := struct {
Customers []dashboardCustomer
OffsiteTile any
PBSTile any
}{Customers: []dashboardCustomer{{
CustomerSummary: store.CustomerSummary{CustomerID: "c1", CustomerName: "Acme", ReceivedAt: time.Now()},
OverallStatus: "ok", BackupAge: "",
+10 -3
View File
@@ -67,7 +67,8 @@ type Server struct {
assetsMgr *assets.Manager
gitea *gitea.Client // optional; enables the Day-0 artifact version dropdowns
offsite *offsite.Provisioner // optional; enables Hetzner offsite provisioning (SLICE 1)
offsiteBox func() (monitor.BoxSnapshot, bool) // optional (v0.64.0, R-5); the pool-box aggregate snapshot accessor
offsiteBox func() (monitor.BoxSnapshot, bool) // optional (v0.64.0, R-5); the restic pool-box aggregate snapshot accessor
pbsdrBox func() (monitor.PBSBoxSnapshot, bool) // optional (v0.65.0, R-5); the PBS-DR datastore fill snapshot accessor
tenantsync tenancyProvisioner // optional; enables PBS DR tier provisioning (web/pbsdr.go)
claimEngine *claim.Engine // optional; enables the customer-claim resend button (v0.50.0)
// intentHub (v0.58.0, Direction-2 immediate-sync) is Bumped by every operator-intent handler
@@ -188,6 +189,11 @@ func (s *Server) SetOffsiteProvisioner(p *offsite.Provisioner) { s.offsite = p }
// honest "not configured". The web layer NEVER fetches from Hetzner — it only reads this cache.
func (s *Server) SetOffsiteBox(fn func() (monitor.BoxSnapshot, bool)) { s.offsiteBox = fn }
// SetPBSDRBox wires the PBS-DR datastore fill snapshot accessor (v0.65.0, R-5): read on the Offsite
// "PBS DR" tab + the Dashboard PBS gauge. nil (no tenantsync client) → "not configured". The snapshot
// carries its own state (ok/unavailable/degraded); the web layer never polls ep0.
func (s *Server) SetPBSDRBox(fn func() (monitor.PBSBoxSnapshot, bool)) { s.pbsdrBox = fn }
// SetClaimEngine wires the customer-claim code engine for the Setup-tab resend button (v0.50.0).
func (s *Server) SetClaimEngine(e *claim.Engine) { s.claimEngine = e }
@@ -751,8 +757,9 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
payload := struct {
Customers []dashboardCustomer
OffsiteTile *offsiteTile // R-5: nil → no tile rendered
}{Customers: data, OffsiteTile: s.offsiteBoxTile()}
OffsiteTile *offsiteTile // R-5: restic pool box; nil → no gauge
PBSTile *pbsdrTile // R-5 v0.65.0: PBS DR datastore; nil → no gauge
}{Customers: data, OffsiteTile: s.offsiteBoxTile(), PBSTile: s.pbsdrBoxTile()}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := s.templates.ExecuteTemplate(w, "dashboard.html", payload); err != nil {
+17 -6
View File
@@ -22,12 +22,23 @@
</nav>
</header>
{{with .OffsiteTile}}
<a href="/offsite" class="offsite-tile offsite-tile-{{.Band}}">
<span class="offsite-tile-label">Offsite pool</span>
<span class="offsite-tile-val">{{formatFloat .FillPercent}}% &middot; {{.RatioStr}}</span>
{{if .Degraded}}<span class="offsite-tile-stale">stale</span>{{end}}
</a>
{{if or .OffsiteTile .PBSTile}}
<div class="offsite-gauges">
{{with .OffsiteTile}}
<a href="/offsite#tab=restic" class="offsite-tile offsite-tile-{{.Band}}">
<span class="offsite-tile-label">Restic</span>
<span class="offsite-tile-val">{{formatFloat .FillPercent}}% &middot; {{.RatioStr}}</span>
{{if .Degraded}}<span class="offsite-tile-stale">stale</span>{{end}}
</a>
{{end}}
{{with .PBSTile}}
<a href="/offsite#tab=pbsdr" class="offsite-tile offsite-tile-{{.Band}}">
<span class="offsite-tile-label">PBS DR</span>
{{if .Unavailable}}<span class="offsite-tile-val" style="color:var(--text-3);">n/a</span>{{else}}<span class="offsite-tile-val">{{formatFloat .FillPercent}}%</span>{{end}}
{{if .Degraded}}<span class="offsite-tile-stale">stale</span>{{end}}
</a>
{{end}}
</div>
{{end}}
{{if not .Customers}}
+73 -5
View File
@@ -21,11 +21,14 @@
</nav>
</header>
<h2 style="margin-bottom: 0.5rem;">Offsite connectivity</h2>
<p class="text-muted" style="margin: 0 0 1rem; font-size: 0.85em;">
Peer allocation and endpoint sync currently use the lowest endpoint id (ep0).
Per-endpoint allocation is a future work item.
</p>
<h2 style="margin-bottom: 0.75rem;">Offsite</h2>
<!-- Two offsite tiers, two distinct stores (v0.65.0, R-5): RESTIC = subaccounts on the shared
Hetzner Storage Box; PBS DR = the felhom-offsite datastore on the ep0 endpoint VM. -->
<nav class="tab-nav" id="tab-nav">
<a href="#tab=restic" data-tab="restic" class="active">Restic (shared box)</a>
<a href="#tab=pbsdr" data-tab="pbsdr">PBS DR</a>
</nav>
{{if eq .Flash "endpoint_saved"}}
<div class="flash flash-success">Endpoint saved.</div>
@@ -34,6 +37,9 @@
<div class="flash flash-success">Endpoint deleted.</div>
{{end}}
<!-- ═══ Restic tab ═══ -->
<div class="tab-panel tab-panel-active" data-tab="restic">
<!-- R-5 (v0.64.0): shared pool-box aggregate — total fill, oversubscription, per-customer usage -->
<section class="card" style="margin-bottom: 1.5rem;">
<h3 style="margin: 0 0 0.75rem;">Offsite pool box</h3>
@@ -73,6 +79,40 @@
{{end}}
</section>
</div><!-- ═══ /Restic tab ═══ -->
<!-- ═══ PBS DR tab ═══ -->
<div class="tab-panel" data-tab="pbsdr">
<!-- R-5 (v0.65.0): PBS DR datastore fill (felhom-offsite on ep0, read via the tenantsync usage op) -->
<section class="card" style="margin-bottom: 1.5rem;">
<h3 style="margin: 0 0 0.75rem;">PBS DR datastore</h3>
{{if not .PBSBox.Configured}}
<p class="text-muted" style="font-size: 0.9em;">PBS DR metrics not configured (no offsite endpoint / tenantsync key on this hub).</p>
{{else if .PBSBox.Pending}}
<p class="text-muted" style="font-size: 0.9em;">PBS DR metrics loading — the first poll has not landed yet.</p>
{{else if .PBSBox.Unavailable}}
<p class="text-muted" style="font-size: 0.9em;">PBS DR usage not available — the endpoint script update (felhom-tenantsync v1.2.0) is pending. The gauge lights up on the next poll once ep0 is updated (no hub redeploy).</p>
{{else if .PBSBox.HasFill}}
<table class="detail-table">
<tr><th style="width: 12rem;">Datastore</th><td><code>felhom-offsite</code> (ep0)</td></tr>
<tr><th>Capacity</th><td>{{.PBSBox.CapacityStr}}</td></tr>
<tr><th>Used</th><td>{{.PBSBox.UsedStr}} &middot; {{formatFloat .PBSBox.FillPercent}}% full</td></tr>
</table>
<div class="bar" style="margin: 0.4rem 0 0.9rem;"><div class="bar-fill bar-{{.PBSBox.FillBand}}" style="width: {{formatFloat .PBSBox.FillPercent}}%;"></div></div>
<table class="detail-table">
<tr><th style="width: 12rem;">Polled</th><td>{{timeAgo .PBSBox.FetchedAt}}{{if .PBSBox.Degraded}} <span class="status-badge status-badge-warn">STALE — last poll failed</span>{{end}}</td></tr>
</table>
{{else}}
<p class="text-muted" style="font-size: 0.9em;">PBS DR usage temporarily unavailable (endpoint poll failed).</p>
{{end}}
</section>
<p class="text-muted" style="margin: 0 0 1rem; font-size: 0.85em;">
The endpoint below IS the PBS DR host. Peer allocation and endpoint sync currently use the
lowest endpoint id (ep0); per-endpoint allocation is a future work item.
</p>
{{if .HasEndpoints}}
{{range .Endpoints}}
<section class="card" style="margin-bottom: 1.5rem;"
@@ -185,11 +225,39 @@
</div>
{{end}}
</div><!-- ═══ /PBS DR tab ═══ -->
<footer style="margin-top: 2rem; color: var(--text-muted); font-size: 0.8rem; text-align: center;">
Felhom Hub <span style="font-family: var(--font-mono)">{{hubVersion}}</span>
</footer>
</div>
<script>
// Hash tabs (v0.65.0), mirrored from customer_unified.html. Without JS this never runs — the body
// never gets .js-tabs, so both panels stay visible and the page reads top-to-bottom (data is
// server-rendered; JS only picks which panel is shown). Default tab = restic.
(function() {
var panels = document.querySelectorAll('.tab-panel');
var links = document.querySelectorAll('#tab-nav a');
if (!panels.length || !links.length) return;
document.body.classList.add('js-tabs');
var known = {};
panels.forEach(function(p) { known[p.getAttribute('data-tab')] = true; });
function currentTab() {
var m = (location.hash || '').match(/^#tab=([a-z-]+)$/);
var t = m ? m[1] : '';
return known[t] ? t : 'restic';
}
function activate() {
var tab = currentTab();
panels.forEach(function(p) { p.classList.toggle('tab-panel-active', p.getAttribute('data-tab') === tab); });
links.forEach(function(a) { a.classList.toggle('active', a.getAttribute('data-tab') === tab); });
}
window.addEventListener('hashchange', activate);
activate();
})();
</script>
<script>
// Endpoint management JS (v0.47.0). The server enforces every guard — this layer only
// fills the edit form from a card's data attributes and adds the pubkey-change confirm.
+4
View File
@@ -303,6 +303,10 @@ header h1 {
.bar-fill.bar-warning { background: var(--warn); }
.bar-fill.bar-critical { background: var(--crit); }
/* Offsite dashboard gauges (v0.65.0, R-5) — two side-by-side tiles (Restic, PBS DR). */
.offsite-gauges { display: flex; gap: 0.75rem; flex-wrap: wrap; margin-bottom: 1rem; }
.offsite-gauges .offsite-tile { margin-bottom: 0; }
/* Offsite pool dashboard tile (v0.64.0, R-5) — compact, band-colored, links to /offsite. */
.offsite-tile {
display: inline-flex; align-items: baseline; gap: 0.5rem;