4bb2df0dc4
The operator sees the shared pool box's real state on the hub: total box fill vs
capacity, Σ(shared soft quotas) vs capacity (the oversubscription ratio), per-customer
usage/quota bars, and a box-level operator alert (fill % + oversub ratio) on the existing
dispatcher's operator channel. Per-customer fill alerts already existed; the box-level
aggregate was the gap. READ-ONLY against Hetzner (GET only).
Phase-0 probe (gate PASSED): the live pool box 611714 returns capacity via
storage_box_type.size (1 TiB / bx11) and usage via a stats object (size/size_data/
size_snapshots), all bytes; our token reads it (200).
- hetznerapi: additive StorageBoxType + StorageBoxStats on StorageBox (no existing field/
method changed); fake carries them + a GetBoxCalls counter; golden decode test.
- monitor.OffsiteBoxChecker: OffsiteChecker-sibling for the box; fetch-throttled (1 GET/
15min), cached BoxSnapshot, escalation-only + recovery re-arm. FILL (used/capacity 80/90)
+ OVERSUB (Σ shared+enabled quotas / capacity, 2.0x) — independent. Σ from the ConfigJSON
Descriptor (offsite.ReadDescriptor, new), never the report echo; dedicated+disabled
excluded. Scope "pool-box" -> operator channel only, no SaveEvent. Failed fetch keeps the
last snapshot degraded; missing data never becomes 0% and never transitions a band.
- config: Alerting.OffsiteBoxFill{Warn,Crit}Percent + OffsiteOversubWarnRatio (80/90/2.0
defaults; thresholds pending Viktor's ruling). Constructed in the HETZNER_TOKEN branch,
60s sweep, snapshot handed to the web server.
- web: Offsite-tab panel (fill bar, Σ+ratio, per-customer usage/quota rows) + a compact
dashboard tile; reads the cached snapshot only, never fetches; nil -> "not configured".
Tests: 10 new + 4 red-proofs (throttle, Σ filter, escalation-only, failed-fetch honesty),
all confirmed red then restored. go build/vet/test all pass; hub confirm gate OK.
354 lines
13 KiB
Go
354 lines
13 KiB
Go
// Package hetznerapi is a small typed client for the Hetzner unified API storage-box surface, used by the
|
|
// hub to provision the offsite tier. SPIKE 996d403 (authoritative, measured live): the base URL is
|
|
// https://api.hetzner.com/v1 — NOT api.hetzner.cloud (the classic Cloud API 404s for every storage-box
|
|
// route). Async writes return an action object; poll it to status "success".
|
|
//
|
|
// Provisioning depends on the CloudAPI interface (not *Client) so tests inject a fake — NO live Hetzner
|
|
// calls in CI. The bearer token is supplied by an injected func (read from an out-of-band secret, never a
|
|
// committed file); it is never logged.
|
|
package hetznerapi
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
)
|
|
|
|
// DefaultBaseURL is the storage-box API base (spike §2).
|
|
const DefaultBaseURL = "https://api.hetzner.com/v1"
|
|
|
|
// Action is the async-write result the API returns; poll to status "success".
|
|
type Action struct {
|
|
ID int64 `json:"id"`
|
|
Command string `json:"command"`
|
|
Status string `json:"status"` // "running" | "success" | "error"
|
|
Progress int `json:"progress"`
|
|
Error *struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
|
|
// AccessSettings mirrors the storage-box/subaccount access_settings object.
|
|
type AccessSettings struct {
|
|
SSHEnabled bool `json:"ssh_enabled"`
|
|
ReachableExternally bool `json:"reachable_externally"`
|
|
SambaEnabled bool `json:"samba_enabled"`
|
|
WebDAVEnabled bool `json:"webdav_enabled"`
|
|
Readonly bool `json:"readonly"`
|
|
}
|
|
|
|
// Subaccount mirrors a storage-box subaccount (the non-secret fields; the API never returns a password).
|
|
type Subaccount struct {
|
|
ID int64 `json:"id"`
|
|
StorageBox int64 `json:"storage_box"`
|
|
Username string `json:"username"` // uXXXXXX-subN
|
|
Server string `json:"server"` // …-subN.your-storagebox.de
|
|
HomeDirectory string `json:"home_directory"`
|
|
AccessSettings AccessSettings `json:"access_settings"`
|
|
Labels map[string]string `json:"labels"`
|
|
}
|
|
|
|
// StorageBox mirrors a dedicated storage box (non-secret fields).
|
|
type StorageBox struct {
|
|
ID int64 `json:"id"`
|
|
Username string `json:"username"` // uXXXXXX (empty until active)
|
|
Name string `json:"name"`
|
|
Status string `json:"status"` // initializing | active | …
|
|
Server string `json:"server"` // uXXXXXX.your-storagebox.de
|
|
Labels map[string]string `json:"labels"`
|
|
|
|
// StorageBoxType + Stats added v0.64.0 (R-5 pool-box aggregate). Shape pinned by the Phase-0 probe
|
|
// of the live pool box 611714 (2026-07-17): GET /storage_boxes/{id} returns a `storage_box_type`
|
|
// object whose `size` is the CAPACITY in bytes (1099511627776 = 1 TiB for bx11), and a `stats` object
|
|
// carrying total/data/snapshot usage in bytes. Additive — no existing field or method changed.
|
|
StorageBoxType StorageBoxType `json:"storage_box_type"`
|
|
Stats StorageBoxStats `json:"stats"`
|
|
}
|
|
|
|
// StorageBoxType is the box's plan; its Size is the CAPACITY in bytes (never use stats.size for capacity).
|
|
type StorageBoxType struct {
|
|
ID int64 `json:"id"`
|
|
Name string `json:"name"` // e.g. "bx11"
|
|
Description string `json:"description"` // e.g. "BX11"
|
|
Size int64 `json:"size"` // capacity in BYTES
|
|
}
|
|
|
|
// StorageBoxStats is the box's live usage, all in BYTES (probe-confirmed): total, the data portion, and
|
|
// the snapshot portion (Size == SizeData + SizeSnapshots on the live box).
|
|
type StorageBoxStats struct {
|
|
Size int64 `json:"size"` // total used bytes
|
|
SizeData int64 `json:"size_data"` // data bytes
|
|
SizeSnapshots int64 `json:"size_snapshots"` // snapshot bytes
|
|
}
|
|
|
|
// CreateSubaccountRequest — POST /storage_boxes/{box}/subaccounts. Password satisfies the 4-class policy.
|
|
type CreateSubaccountRequest struct {
|
|
HomeDirectory string `json:"home_directory"`
|
|
Password string `json:"password"`
|
|
AccessSettings AccessSettings `json:"access_settings"`
|
|
Labels map[string]string `json:"labels,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
}
|
|
|
|
// CreateBoxRequest — POST /storage_boxes.
|
|
type CreateBoxRequest struct {
|
|
Name string `json:"name"`
|
|
StorageBoxType string `json:"storage_box_type"` // e.g. "bx11"
|
|
Location string `json:"location"` // e.g. "fsn1"
|
|
Password string `json:"password"`
|
|
AccessSettings AccessSettings `json:"access_settings"`
|
|
Labels map[string]string `json:"labels,omitempty"`
|
|
}
|
|
|
|
// CloudAPI is the storage-box provisioning surface the hub depends on. A fake implements it in tests.
|
|
type CloudAPI interface {
|
|
ListSubaccounts(ctx context.Context, boxID int64, labelSelector string) ([]Subaccount, error)
|
|
GetSubaccount(ctx context.Context, boxID, subID int64) (Subaccount, error)
|
|
CreateSubaccount(ctx context.Context, boxID int64, req CreateSubaccountRequest) (createdID int64, action Action, err error)
|
|
ResetSubaccountPassword(ctx context.Context, boxID, subID int64, password string) (Action, error)
|
|
UpdateSubaccountAccess(ctx context.Context, boxID, subID int64, as AccessSettings) (Action, error)
|
|
DeleteSubaccount(ctx context.Context, boxID, subID int64) (Action, error)
|
|
|
|
ListStorageBoxes(ctx context.Context, labelSelector string) ([]StorageBox, error)
|
|
GetStorageBox(ctx context.Context, boxID int64) (StorageBox, error)
|
|
CreateStorageBox(ctx context.Context, req CreateBoxRequest) (createdID int64, action Action, err error)
|
|
ChangeType(ctx context.Context, boxID int64, boxType string) (Action, error)
|
|
ResetBoxPassword(ctx context.Context, boxID int64, password string) (Action, error)
|
|
DeleteStorageBox(ctx context.Context, boxID int64) (Action, error)
|
|
|
|
// WaitAction polls the action to "success" (bounded); errors on "error" or timeout.
|
|
WaitAction(ctx context.Context, action Action) error
|
|
}
|
|
|
|
// Client implements CloudAPI against the live API.
|
|
type Client struct {
|
|
BaseURL string
|
|
Token func() string // injected; read from an out-of-band secret; never logged
|
|
HC *http.Client
|
|
PollEvery time.Duration
|
|
PollMax time.Duration
|
|
}
|
|
|
|
// NewClient builds a Client with sane defaults. token must return the bearer (from env/mounted secret).
|
|
func NewClient(token func() string) *Client {
|
|
return &Client{
|
|
BaseURL: DefaultBaseURL,
|
|
Token: token,
|
|
HC: &http.Client{Timeout: 30 * time.Second},
|
|
PollEvery: 3 * time.Second,
|
|
PollMax: 3 * time.Minute,
|
|
}
|
|
}
|
|
|
|
// apiError is the API's error envelope {error:{code,message,details}}.
|
|
type apiError struct {
|
|
Error struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
Details json.RawMessage `json:"details"`
|
|
} `json:"error"`
|
|
}
|
|
|
|
func (e *apiError) String() string { return fmt.Sprintf("%s: %s", e.Error.Code, e.Error.Message) }
|
|
|
|
// do performs a request and decodes the JSON body into out. On a non-2xx it returns a typed error carrying
|
|
// the API's error code/message. The token is set on the header only — never logged.
|
|
func (c *Client) do(ctx context.Context, method, path string, body, out any) error {
|
|
var rdr io.Reader
|
|
if body != nil {
|
|
b, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rdr = bytes.NewReader(b)
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, rdr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.Token())
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := c.HC.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("hetznerapi: %s %s: %w", method, path, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
var ae apiError
|
|
if json.Unmarshal(raw, &ae) == nil && ae.Error.Code != "" {
|
|
return fmt.Errorf("hetznerapi: %s %s: HTTP %d: %s", method, path, resp.StatusCode, ae.String())
|
|
}
|
|
return fmt.Errorf("hetznerapi: %s %s: HTTP %d", method, path, resp.StatusCode)
|
|
}
|
|
if out != nil && len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, out); err != nil {
|
|
return fmt.Errorf("hetznerapi: %s %s: decode: %w", method, path, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func labelQuery(labelSelector string) string {
|
|
if labelSelector == "" {
|
|
return ""
|
|
}
|
|
return "?label_selector=" + url.QueryEscape(labelSelector)
|
|
}
|
|
|
|
// --- subaccounts ---
|
|
|
|
func (c *Client) ListSubaccounts(ctx context.Context, boxID int64, labelSelector string) ([]Subaccount, error) {
|
|
var out struct {
|
|
Subaccounts []Subaccount `json:"subaccounts"`
|
|
}
|
|
err := c.do(ctx, http.MethodGet, fmt.Sprintf("/storage_boxes/%d/subaccounts%s", boxID, labelQuery(labelSelector)), nil, &out)
|
|
return out.Subaccounts, err
|
|
}
|
|
|
|
func (c *Client) GetSubaccount(ctx context.Context, boxID, subID int64) (Subaccount, error) {
|
|
var out struct {
|
|
Subaccount Subaccount `json:"subaccount"`
|
|
}
|
|
err := c.do(ctx, http.MethodGet, fmt.Sprintf("/storage_boxes/%d/subaccounts/%d", boxID, subID), nil, &out)
|
|
return out.Subaccount, err
|
|
}
|
|
|
|
func (c *Client) CreateSubaccount(ctx context.Context, boxID int64, req CreateSubaccountRequest) (int64, Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
Subaccount struct {
|
|
ID int64 `json:"id"`
|
|
} `json:"subaccount"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, fmt.Sprintf("/storage_boxes/%d/subaccounts", boxID), req, &out)
|
|
return out.Subaccount.ID, out.Action, err
|
|
}
|
|
|
|
func (c *Client) ResetSubaccountPassword(ctx context.Context, boxID, subID int64, password string) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, fmt.Sprintf("/storage_boxes/%d/subaccounts/%d/actions/reset_subaccount_password", boxID, subID), map[string]string{"password": password}, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
func (c *Client) UpdateSubaccountAccess(ctx context.Context, boxID, subID int64, as AccessSettings) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, fmt.Sprintf("/storage_boxes/%d/subaccounts/%d/actions/update_access_settings", boxID, subID), as, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
func (c *Client) DeleteSubaccount(ctx context.Context, boxID, subID int64) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodDelete, fmt.Sprintf("/storage_boxes/%d/subaccounts/%d", boxID, subID), nil, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
// --- storage boxes ---
|
|
|
|
func (c *Client) ListStorageBoxes(ctx context.Context, labelSelector string) ([]StorageBox, error) {
|
|
var out struct {
|
|
StorageBoxes []StorageBox `json:"storage_boxes"`
|
|
}
|
|
err := c.do(ctx, http.MethodGet, "/storage_boxes"+labelQuery(labelSelector), nil, &out)
|
|
return out.StorageBoxes, err
|
|
}
|
|
|
|
func (c *Client) GetStorageBox(ctx context.Context, boxID int64) (StorageBox, error) {
|
|
var out struct {
|
|
StorageBox StorageBox `json:"storage_box"`
|
|
}
|
|
err := c.do(ctx, http.MethodGet, fmt.Sprintf("/storage_boxes/%d", boxID), nil, &out)
|
|
return out.StorageBox, err
|
|
}
|
|
|
|
func (c *Client) CreateStorageBox(ctx context.Context, req CreateBoxRequest) (int64, Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
StorageBox struct {
|
|
ID int64 `json:"id"`
|
|
} `json:"storage_box"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, "/storage_boxes", req, &out)
|
|
return out.StorageBox.ID, out.Action, err
|
|
}
|
|
|
|
func (c *Client) ChangeType(ctx context.Context, boxID int64, boxType string) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, fmt.Sprintf("/storage_boxes/%d/actions/change_type", boxID), map[string]string{"storage_box_type": boxType}, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
// ResetBoxPassword resets a dedicated box's (main-account) password. The password value is only ever in
|
|
// the request body — never logged by the caller (F4 re-issue).
|
|
func (c *Client) ResetBoxPassword(ctx context.Context, boxID int64, password string) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodPost, fmt.Sprintf("/storage_boxes/%d/actions/reset_password", boxID), map[string]string{"password": password}, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
func (c *Client) DeleteStorageBox(ctx context.Context, boxID int64) (Action, error) {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
err := c.do(ctx, http.MethodDelete, fmt.Sprintf("/storage_boxes/%d", boxID), nil, &out)
|
|
return out.Action, err
|
|
}
|
|
|
|
// WaitAction polls the action to "success" (bounded by PollMax). An already-terminal action returns
|
|
// immediately. It NEVER assumes create == ready.
|
|
func (c *Client) WaitAction(ctx context.Context, action Action) error {
|
|
if s := action.Status; s == "success" {
|
|
return nil
|
|
} else if s == "error" {
|
|
return actionErr(action)
|
|
}
|
|
deadline := time.Now().Add(c.PollMax)
|
|
for {
|
|
var out struct {
|
|
Action Action `json:"action"`
|
|
}
|
|
if err := c.do(ctx, http.MethodGet, fmt.Sprintf("/storage_boxes/actions/%d", action.ID), nil, &out); err != nil {
|
|
return err
|
|
}
|
|
switch out.Action.Status {
|
|
case "success":
|
|
return nil
|
|
case "error":
|
|
return actionErr(out.Action)
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("hetznerapi: action %d (%s) did not reach success within %s", action.ID, action.Command, c.PollMax)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(c.PollEvery):
|
|
}
|
|
}
|
|
}
|
|
|
|
func actionErr(a Action) error {
|
|
if a.Error != nil {
|
|
return fmt.Errorf("hetznerapi: action %d (%s) failed: %s: %s", a.ID, a.Command, a.Error.Code, a.Error.Message)
|
|
}
|
|
return fmt.Errorf("hetznerapi: action %d (%s) failed", a.ID, a.Command)
|
|
}
|