7f11cfb36c
Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.
Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.
- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).
Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
327 lines
20 KiB
HTML
327 lines
20 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Offsite — Felhom Hub</title>
|
|
<link rel="stylesheet" href="/style.css?v={{hubVersion}}">
|
|
</head>
|
|
<body>
|
|
{{template "icon_sprite"}}
|
|
<div class="container">
|
|
<header>
|
|
<h1>Felhom <span>Hub</span></h1>
|
|
<nav class="nav-links">
|
|
<a href="/" class="nav-link">Dashboard</a>
|
|
<a href="/configs" class="nav-link">Customers</a>
|
|
<a href="/apps" class="nav-link">Apps</a>
|
|
<a href="/hosts" class="nav-link">Hosts</a>
|
|
<a href="/offsite" class="nav-link active">Offsite</a>
|
|
<a href="/configuration" class="nav-link">Configuration</a>
|
|
</nav>
|
|
</header>
|
|
|
|
<h2 style="margin-bottom: 0.75rem;">Offsite</h2>
|
|
|
|
<!-- Two offsite tiers, two distinct stores (v0.65.0, R-5): RESTIC = subaccounts on the shared
|
|
Hetzner Storage Box; PBS DR = the felhom-offsite datastore on the ep0 endpoint VM. -->
|
|
<nav class="tab-nav" id="tab-nav">
|
|
<a href="#tab=restic" data-tab="restic" class="active">Restic (shared box)</a>
|
|
<a href="#tab=pbsdr" data-tab="pbsdr">PBS DR</a>
|
|
</nav>
|
|
|
|
{{if eq .Flash "endpoint_saved"}}
|
|
<div class="flash flash-success">Endpoint saved.</div>
|
|
{{end}}
|
|
{{if eq .Flash "endpoint_deleted"}}
|
|
<div class="flash flash-success">Endpoint deleted.</div>
|
|
{{end}}
|
|
|
|
<!-- ═══ Restic tab ═══ -->
|
|
<div class="tab-panel tab-panel-active" data-tab="restic">
|
|
|
|
<!-- R-5 (v0.64.0): shared pool-box aggregate — total fill, oversubscription, per-customer usage -->
|
|
<section class="card" style="margin-bottom: 1.5rem;">
|
|
<h3 style="margin: 0 0 0.75rem;">Offsite pool box</h3>
|
|
{{if not .OffsiteBox.Configured}}
|
|
<p class="text-muted" style="font-size: 0.9em;">Offsite pool metrics not configured (no Hetzner token / pool box id on this hub).</p>
|
|
{{else if .OffsiteBox.Pending}}
|
|
<p class="text-muted" style="font-size: 0.9em;">Pool metrics loading — the first refresh has not landed yet.</p>
|
|
{{else}}
|
|
<table class="detail-table">
|
|
<tr><th style="width: 12rem;">Box</th><td>{{.OffsiteBox.BoxType}}</td></tr>
|
|
<tr><th>Capacity</th><td>{{.OffsiteBox.CapacityStr}}</td></tr>
|
|
<tr><th>Used</th><td>{{.OffsiteBox.UsedStr}} · {{formatFloat .OffsiteBox.FillPercent}}% full <span class="text-muted">(data {{.OffsiteBox.DataStr}} + snapshots {{.OffsiteBox.SnapshotStr}})</span></td></tr>
|
|
</table>
|
|
<div class="bar" style="margin: 0.4rem 0 0.9rem;"><div class="bar-fill bar-{{.OffsiteBox.FillBand}}" style="width: {{formatFloat .OffsiteBox.FillPercent}}%;"></div></div>
|
|
<table class="detail-table">
|
|
<tr><th style="width: 12rem;">Σ shared soft quotas</th><td>{{.OffsiteBox.SumQuotaStr}}</td></tr>
|
|
<tr><th>Oversubscription</th><td><span class="status-badge status-badge-{{if eq .OffsiteBox.OversubBand "warning"}}warn{{else}}ok{{end}}">{{.OffsiteBox.RatioStr}}</span> <span class="text-muted">Σ(shared quotas) ÷ capacity</span></td></tr>
|
|
<tr><th>Fetched</th><td>{{timeAgo .OffsiteBox.FetchedAt}}{{if .OffsiteBox.Degraded}} <span class="status-badge status-badge-warn">STALE — last refresh failed</span>{{end}}</td></tr>
|
|
</table>
|
|
|
|
{{if .OffsiteCusts}}
|
|
<h4 style="margin: 1rem 0 0.5rem; font-size: 0.85rem; color: var(--text-3); text-transform: uppercase; letter-spacing: 0.03em;">Per-customer usage</h4>
|
|
<table class="container-table">
|
|
<thead><tr><th>Customer</th><th>Usage</th><th>Quota</th><th style="width: 35%;"></th></tr></thead>
|
|
<tbody>
|
|
{{range .OffsiteCusts}}
|
|
<tr>
|
|
<td>{{.CustomerName}}{{if .Dedicated}} <span class="text-muted">(dedicated)</span>{{end}}</td>
|
|
<td>{{if .NoUsage}}<span class="text-muted">no usage reported yet</span>{{else}}{{.UsageStr}}{{end}}</td>
|
|
<td>{{if .Dedicated}}<span class="text-muted">dedicated</span>{{else if .QuotaStr}}{{.QuotaStr}}{{else}}<span class="text-muted">—</span>{{end}}</td>
|
|
<td>{{if .HasBar}}<div class="bar"><div class="bar-fill bar-{{.UsageBand}}" style="width: {{formatFloat .UsagePercent}}%;"></div></div>{{end}}</td>
|
|
</tr>
|
|
{{end}}
|
|
</tbody>
|
|
</table>
|
|
{{end}}
|
|
{{end}}
|
|
</section>
|
|
|
|
</div><!-- ═══ /Restic tab ═══ -->
|
|
|
|
<!-- ═══ PBS DR tab ═══ -->
|
|
<div class="tab-panel" data-tab="pbsdr">
|
|
|
|
<!-- R-5 (v0.65.0): PBS DR datastore fill (felhom-offsite on ep0, read via the tenantsync usage op) -->
|
|
<section class="card" style="margin-bottom: 1.5rem;">
|
|
<h3 style="margin: 0 0 0.75rem;">PBS DR datastore</h3>
|
|
{{if not .PBSBox.Configured}}
|
|
<p class="text-muted" style="font-size: 0.9em;">PBS DR metrics not configured (no offsite endpoint / tenantsync key on this hub).</p>
|
|
{{else if .PBSBox.Pending}}
|
|
<p class="text-muted" style="font-size: 0.9em;">PBS DR metrics loading — the first poll has not landed yet.</p>
|
|
{{else if .PBSBox.Unavailable}}
|
|
<p class="text-muted" style="font-size: 0.9em;">PBS DR usage not available — the endpoint script update (felhom-tenantsync v1.2.0) is pending. The gauge lights up on the next poll once ep0 is updated (no hub redeploy).</p>
|
|
{{else if .PBSBox.HasFill}}
|
|
<table class="detail-table">
|
|
<tr><th style="width: 12rem;">Datastore</th><td><code>felhom-offsite</code> (ep0)</td></tr>
|
|
<tr><th>Capacity</th><td>{{.PBSBox.CapacityStr}}</td></tr>
|
|
<tr><th>Used</th><td>{{.PBSBox.UsedStr}} · {{formatFloat .PBSBox.FillPercent}}% full</td></tr>
|
|
</table>
|
|
<div class="bar" style="margin: 0.4rem 0 0.9rem;"><div class="bar-fill bar-{{.PBSBox.FillBand}}" style="width: {{formatFloat .PBSBox.FillPercent}}%;"></div></div>
|
|
<table class="detail-table">
|
|
<tr><th style="width: 12rem;">Polled</th><td>{{timeAgo .PBSBox.FetchedAt}}{{if .PBSBox.Degraded}} <span class="status-badge status-badge-warn">STALE — last poll failed</span>{{end}}</td></tr>
|
|
</table>
|
|
{{else}}
|
|
<p class="text-muted" style="font-size: 0.9em;">PBS DR usage temporarily unavailable (endpoint poll failed).</p>
|
|
{{end}}
|
|
</section>
|
|
|
|
<p class="text-muted" style="margin: 0 0 1rem; font-size: 0.85em;">
|
|
The endpoint below IS the PBS DR host. Peer allocation and endpoint sync currently use the
|
|
lowest endpoint id (ep0); per-endpoint allocation is a future work item.
|
|
</p>
|
|
|
|
{{if .HasEndpoints}}
|
|
{{range .Endpoints}}
|
|
<section class="card" style="margin-bottom: 1.5rem;"
|
|
data-endpoint-id="{{.EndpointID}}" data-dns-name="{{.DNSName}}" data-wg-port="{{.WGPort}}"
|
|
data-server-pubkey="{{.ServerPubkey}}" data-tunnel-subnet="{{.TunnelSubnet}}" data-pbs-tunnel-ip="{{.PBSTunnelIP}}">
|
|
<div style="display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 0.5rem;">
|
|
<h3 style="margin: 0;">Endpoint <code>{{.EndpointID}}</code></h3>
|
|
<div style="display: flex; gap: 0.5rem;">
|
|
<button type="button" class="btn btn-outline btn-sm" onclick="epEdit('{{.EndpointID}}')">Edit</button>
|
|
<button type="button" class="btn btn-danger btn-sm" onclick="epDeleteConfirm('{{.EndpointID}}')">Remove…</button>
|
|
</div>
|
|
</div>
|
|
<table class="detail-table" style="margin-top: 0.75rem;">
|
|
<tr><th>Address</th><td><code>{{.DNSName}}:{{.WGPort}}</code> (WireGuard, UDP)</td></tr>
|
|
<tr><th>Server public key</th><td><code title="{{.ServerPubkey}}">{{.ServerPubkeyShort}}</code></td></tr>
|
|
<tr><th>Tunnel subnet</th><td><code>{{.TunnelSubnet}}</code></td></tr>
|
|
<tr><th>PBS tunnel address</th><td><code>{{.PBSTunnelIP}}:8007</code></td></tr>
|
|
<tr><th>Peers in subnet</th><td>{{.PeerCount}}</td></tr>
|
|
</table>
|
|
<div id="ep-delete-confirm-{{.EndpointID}}" style="display: none; margin-top: 0.75rem; padding: 0.75rem; border: 1px solid var(--crit); background: var(--crit-dim); border-radius: var(--radius); max-width: 44em;">
|
|
<p style="margin: 0 0 0.5rem; font-size: 0.85em;">
|
|
Deleting <code>{{.EndpointID}}</code> removes only the endpoint record — registered peers stay.
|
|
Deletion is refused while any peer is allocated in its subnet. Type the endpoint id to confirm:
|
|
</p>
|
|
<form method="POST" action="/offsite/endpoints/{{.EndpointID}}/delete" id="ep-delete-form-{{.EndpointID}}" style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
|
|
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}">
|
|
<input type="text" id="ep-delete-input-{{.EndpointID}}" placeholder="retype the endpoint id…" style="padding: 0.3em 0.5em; width: 12em;">
|
|
<button type="button" class="btn btn-danger btn-sm" onclick="epDeleteSubmit('{{.EndpointID}}')">Confirm & delete</button>
|
|
<button type="button" class="btn btn-sm btn-outline" onclick="document.getElementById('ep-delete-confirm-{{.EndpointID}}').style.display='none';">Cancel</button>
|
|
<span id="ep-delete-err-{{.EndpointID}}" style="font-size: 0.8em; color: var(--crit);"></span>
|
|
</form>
|
|
</div>
|
|
</section>
|
|
{{end}}
|
|
{{else}}
|
|
<section class="card" style="margin-bottom: 1.5rem;">
|
|
<h3>Endpoint</h3>
|
|
<p class="text-muted">Not configured. Add one below (or via <code>PUT /api/v1/admin/wg/endpoint</code>, runbook: offsite-endpoint.md).</p>
|
|
</section>
|
|
{{end}}
|
|
|
|
<!-- Add / edit endpoint (v0.47.0). Plain form post — server-side validation is
|
|
authoritative; the JS layer only adds the pubkey-change type-to-confirm. -->
|
|
<section class="card" style="margin-bottom: 1.5rem;">
|
|
<h3 id="ep-form-title">Add endpoint</h3>
|
|
<form method="POST" action="/offsite/endpoints" id="ep-form" onsubmit="return epFormSubmitCheck()"
|
|
style="display: grid; grid-template-columns: auto 1fr; gap: 0.5rem; align-items: center; max-width: 44em; margin-top: 0.75rem;">
|
|
<input type="hidden" name="_csrf" value="{{.CSRFToken}}">
|
|
<label style="font-size: 0.9em;">Endpoint id</label>
|
|
<input type="text" name="endpoint_id" id="ep-id" placeholder="ep1" style="padding: 0.3em 0.5em;">
|
|
<label style="font-size: 0.9em;">DNS name</label>
|
|
<input type="text" name="dns_name" id="ep-dns" placeholder="ep1.felhom.eu" style="padding: 0.3em 0.5em;">
|
|
<label style="font-size: 0.9em;">WG port</label>
|
|
<input type="number" name="wg_port" id="ep-port" min="1" max="65535" placeholder="443" style="padding: 0.3em 0.5em;">
|
|
<label style="font-size: 0.9em;">Server public key</label>
|
|
<input type="text" name="server_pubkey" id="ep-pubkey" placeholder="base64 WireGuard pubkey" style="padding: 0.3em 0.5em; font-family: var(--font-data);">
|
|
<label style="font-size: 0.9em;">Tunnel subnet</label>
|
|
<input type="text" name="tunnel_subnet" id="ep-subnet" placeholder="10.78.0.0/24" style="padding: 0.3em 0.5em;">
|
|
<label style="font-size: 0.9em;">PBS tunnel IP</label>
|
|
<input type="text" name="pbs_tunnel_ip" id="ep-pbsip" placeholder="10.78.0.1 (inside the subnet)" style="padding: 0.3em 0.5em;">
|
|
<span></span>
|
|
<div style="display: flex; gap: 0.5rem; align-items: center;">
|
|
<button type="submit" class="btn btn-sm">Save endpoint</button>
|
|
<button type="button" class="btn btn-sm btn-outline" id="ep-form-reset" style="display: none;" onclick="epFormReset()">Cancel edit</button>
|
|
</div>
|
|
</form>
|
|
<div id="ep-pubkey-confirm" style="display: none; margin-top: 0.75rem; padding: 0.75rem; border: 1px solid var(--warn); background: var(--warn-dim); border-radius: var(--radius); max-width: 44em;">
|
|
<p style="margin: 0 0 0.5rem; font-size: 0.85em;">
|
|
The server public key is CHANGING. Peers keep using the old key until they pull their
|
|
next desired-state (they converge on their own cycle — no push). Type the endpoint id to confirm:
|
|
</p>
|
|
<input type="text" id="ep-pubkey-confirm-input" placeholder="retype the endpoint id…" style="padding: 0.3em 0.5em; width: 12em;">
|
|
<button type="button" class="btn btn-sm" onclick="epPubkeyConfirmSubmit()">Confirm & save</button>
|
|
<button type="button" class="btn btn-sm btn-outline" onclick="document.getElementById('ep-pubkey-confirm').style.display='none';">Cancel</button>
|
|
<p id="ep-pubkey-confirm-err" style="margin: 0.4em 0 0; font-size: 0.8em; color: var(--crit);"></p>
|
|
</div>
|
|
</section>
|
|
|
|
{{if .Peers}}
|
|
<section class="card" style="padding: 0; overflow: hidden;">
|
|
<table class="data-table">
|
|
<thead>
|
|
<tr>
|
|
<th>Public key</th>
|
|
<th>Assigned IP</th>
|
|
<th>Endpoint</th>
|
|
<th>Host</th>
|
|
<th>Note</th>
|
|
<th>Created</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{{range .Peers}}
|
|
<tr>
|
|
<td><code title="{{.Pubkey}}">{{.PubkeyShort}}</code></td>
|
|
<td><code>{{.AssignedIP}}</code></td>
|
|
<td>{{if .EndpointID}}<code>{{.EndpointID}}</code>{{else}}—{{end}}</td>
|
|
<td>{{if .HostID}}<a href="/hosts/{{.HostID}}">{{.HostID}}</a>{{else}}—{{end}}</td>
|
|
<td>{{if .Note}}{{.Note}}{{else}}—{{end}}</td>
|
|
<td>{{.CreatedAt}}</td>
|
|
</tr>
|
|
{{end}}
|
|
</tbody>
|
|
</table>
|
|
</section>
|
|
{{else}}
|
|
<div class="empty-state">
|
|
<p>No WireGuard peers registered.</p>
|
|
<p class="hint">A peer appears here when a box registers via <code>POST /hosts/<id>/wg</code> or the operator adds one via the admin API.</p>
|
|
</div>
|
|
{{end}}
|
|
|
|
</div><!-- ═══ /PBS DR tab ═══ -->
|
|
|
|
<footer style="margin-top: 2rem; color: var(--text-muted); font-size: 0.8rem; text-align: center;">
|
|
Felhom Hub <span style="font-family: var(--font-mono)">{{hubVersion}}</span>
|
|
</footer>
|
|
</div>
|
|
|
|
<script>
|
|
// Hash tabs (v0.65.0), mirrored from customer_unified.html. Without JS this never runs — the body
|
|
// never gets .js-tabs, so both panels stay visible and the page reads top-to-bottom (data is
|
|
// server-rendered; JS only picks which panel is shown). Default tab = restic.
|
|
(function() {
|
|
var panels = document.querySelectorAll('.tab-panel');
|
|
var links = document.querySelectorAll('#tab-nav a');
|
|
if (!panels.length || !links.length) return;
|
|
document.body.classList.add('js-tabs');
|
|
var known = {};
|
|
panels.forEach(function(p) { known[p.getAttribute('data-tab')] = true; });
|
|
function currentTab() {
|
|
var m = (location.hash || '').match(/^#tab=([a-z-]+)$/);
|
|
var t = m ? m[1] : '';
|
|
return known[t] ? t : 'restic';
|
|
}
|
|
function activate() {
|
|
var tab = currentTab();
|
|
panels.forEach(function(p) { p.classList.toggle('tab-panel-active', p.getAttribute('data-tab') === tab); });
|
|
links.forEach(function(a) { a.classList.toggle('active', a.getAttribute('data-tab') === tab); });
|
|
}
|
|
window.addEventListener('hashchange', activate);
|
|
activate();
|
|
})();
|
|
</script>
|
|
|
|
<script>
|
|
// Endpoint management JS (v0.47.0). The server enforces every guard — this layer only
|
|
// fills the edit form from a card's data attributes and adds the pubkey-change confirm.
|
|
var epOriginalPubkey = null; // non-null = editing an existing endpoint
|
|
|
|
function epEdit(id) {
|
|
var card = document.querySelector('[data-endpoint-id="' + id + '"]');
|
|
if (!card) return;
|
|
document.getElementById('ep-form-title').textContent = 'Edit endpoint ' + id;
|
|
var f = {
|
|
'ep-id': 'data-endpoint-id', 'ep-dns': 'data-dns-name', 'ep-port': 'data-wg-port',
|
|
'ep-pubkey': 'data-server-pubkey', 'ep-subnet': 'data-tunnel-subnet', 'ep-pbsip': 'data-pbs-tunnel-ip'
|
|
};
|
|
for (var el in f) { document.getElementById(el).value = card.getAttribute(f[el]) || ''; }
|
|
// endpoint_id is the primary key — immutable while editing.
|
|
document.getElementById('ep-id').readOnly = true;
|
|
epOriginalPubkey = card.getAttribute('data-server-pubkey') || '';
|
|
document.getElementById('ep-form-reset').style.display = 'inline-block';
|
|
document.getElementById('ep-form').scrollIntoView({behavior: 'smooth', block: 'center'});
|
|
}
|
|
|
|
function epFormReset() {
|
|
document.getElementById('ep-form').reset();
|
|
document.getElementById('ep-form-title').textContent = 'Add endpoint';
|
|
document.getElementById('ep-id').readOnly = false;
|
|
document.getElementById('ep-id').value = '';
|
|
epOriginalPubkey = null;
|
|
document.getElementById('ep-form-reset').style.display = 'none';
|
|
document.getElementById('ep-pubkey-confirm').style.display = 'none';
|
|
}
|
|
|
|
function epFormSubmitCheck() {
|
|
// Editing + pubkey changed → intercept with the type-to-confirm (floor pattern).
|
|
if (epOriginalPubkey !== null && document.getElementById('ep-pubkey').value.trim() !== epOriginalPubkey) {
|
|
document.getElementById('ep-pubkey-confirm-input').value = '';
|
|
document.getElementById('ep-pubkey-confirm-err').textContent = '';
|
|
document.getElementById('ep-pubkey-confirm').style.display = 'block';
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function epPubkeyConfirmSubmit() {
|
|
var typed = document.getElementById('ep-pubkey-confirm-input').value.trim();
|
|
var expected = document.getElementById('ep-id').value.trim();
|
|
var err = document.getElementById('ep-pubkey-confirm-err');
|
|
if (typed !== expected) { err.textContent = 'Confirmation does not match the endpoint id.'; return; }
|
|
document.getElementById('ep-form').submit();
|
|
}
|
|
|
|
function epDeleteConfirm(id) {
|
|
var box = document.getElementById('ep-delete-confirm-' + id);
|
|
document.getElementById('ep-delete-input-' + id).value = '';
|
|
document.getElementById('ep-delete-err-' + id).textContent = '';
|
|
box.style.display = 'block';
|
|
}
|
|
|
|
function epDeleteSubmit(id) {
|
|
var typed = document.getElementById('ep-delete-input-' + id).value.trim();
|
|
var err = document.getElementById('ep-delete-err-' + id);
|
|
if (typed !== id) { err.textContent = 'Confirmation does not match the endpoint id.'; return; }
|
|
document.getElementById('ep-delete-form-' + id).submit();
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|