4bb2df0dc4
The operator sees the shared pool box's real state on the hub: total box fill vs
capacity, Σ(shared soft quotas) vs capacity (the oversubscription ratio), per-customer
usage/quota bars, and a box-level operator alert (fill % + oversub ratio) on the existing
dispatcher's operator channel. Per-customer fill alerts already existed; the box-level
aggregate was the gap. READ-ONLY against Hetzner (GET only).
Phase-0 probe (gate PASSED): the live pool box 611714 returns capacity via
storage_box_type.size (1 TiB / bx11) and usage via a stats object (size/size_data/
size_snapshots), all bytes; our token reads it (200).
- hetznerapi: additive StorageBoxType + StorageBoxStats on StorageBox (no existing field/
method changed); fake carries them + a GetBoxCalls counter; golden decode test.
- monitor.OffsiteBoxChecker: OffsiteChecker-sibling for the box; fetch-throttled (1 GET/
15min), cached BoxSnapshot, escalation-only + recovery re-arm. FILL (used/capacity 80/90)
+ OVERSUB (Σ shared+enabled quotas / capacity, 2.0x) — independent. Σ from the ConfigJSON
Descriptor (offsite.ReadDescriptor, new), never the report echo; dedicated+disabled
excluded. Scope "pool-box" -> operator channel only, no SaveEvent. Failed fetch keeps the
last snapshot degraded; missing data never becomes 0% and never transitions a band.
- config: Alerting.OffsiteBoxFill{Warn,Crit}Percent + OffsiteOversubWarnRatio (80/90/2.0
defaults; thresholds pending Viktor's ruling). Constructed in the HETZNER_TOKEN branch,
60s sweep, snapshot handed to the web server.
- web: Offsite-tab panel (fill bar, Σ+ratio, per-customer usage/quota rows) + a compact
dashboard tile; reads the cached snapshot only, never fetches; nil -> "not configured".
Tests: 10 new + 4 red-proofs (throttle, Σ filter, escalation-only, failed-fetch honesty),
all confirmed red then restored. go build/vet/test all pass; hub confirm gate OK.
207 lines
5.6 KiB
Go
207 lines
5.6 KiB
Go
package hetznerapi
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
)
|
|
|
|
// Fake is an in-memory CloudAPI for tests (no live Hetzner calls). It records created resources by label
|
|
// (so idempotency lookups work) and lets a test force a failure. Actions it returns are already-terminal
|
|
// (success unless FailCreate/FailAction is set), so WaitAction resolves without a network poll.
|
|
type Fake struct {
|
|
mu sync.Mutex
|
|
|
|
Subaccounts map[int64]Subaccount // by id
|
|
Boxes map[int64]StorageBox // by id
|
|
nextID int64
|
|
|
|
// Records of calls (for assertions).
|
|
CreatedSubaccounts int
|
|
CreatedBoxes int
|
|
ResetCalls int
|
|
BoxResetCalls int
|
|
DeletedSubaccounts int
|
|
DeletedBoxes int
|
|
GetBoxCalls int // v0.64.0: the fetch-throttle assertion (Scenario A) counts these
|
|
|
|
// Failure injection.
|
|
FailCreate error // if set, CreateSubaccount/CreateStorageBox return this error
|
|
FailAction bool // if set, created actions come back status "error" (WaitAction fails)
|
|
FailGetBox error // v0.64.0: if set, GetStorageBox returns this (the box-poll failure path, Scenario D)
|
|
}
|
|
|
|
// NewFake returns an empty Fake.
|
|
func NewFake() *Fake {
|
|
return &Fake{Subaccounts: map[int64]Subaccount{}, Boxes: map[int64]StorageBox{}, nextID: 1000}
|
|
}
|
|
|
|
func (f *Fake) newAction(cmd string) Action {
|
|
if f.FailAction {
|
|
a := Action{ID: f.nextID, Command: cmd, Status: "error"}
|
|
a.Error = &struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
}{Code: "action_failed", Message: "injected action failure"}
|
|
f.nextID++
|
|
return a
|
|
}
|
|
a := Action{ID: f.nextID, Command: cmd, Status: "success", Progress: 100}
|
|
f.nextID++
|
|
return a
|
|
}
|
|
|
|
func labelMatch(labels map[string]string, selector string) bool {
|
|
if selector == "" {
|
|
return true
|
|
}
|
|
// minimal k=v matcher (the hub only uses single-key equality selectors)
|
|
for i := 0; i < len(selector); i++ {
|
|
if selector[i] == '=' {
|
|
k, v := selector[:i], selector[i+1:]
|
|
return labels[k] == v
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (f *Fake) ListSubaccounts(_ context.Context, boxID int64, sel string) ([]Subaccount, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []Subaccount
|
|
for _, s := range f.Subaccounts {
|
|
if s.StorageBox == boxID && labelMatch(s.Labels, sel) {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *Fake) GetSubaccount(_ context.Context, _ int64, subID int64) (Subaccount, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
s, ok := f.Subaccounts[subID]
|
|
if !ok {
|
|
return Subaccount{}, fmt.Errorf("hetznerapi(fake): subaccount %d not found", subID)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (f *Fake) CreateSubaccount(_ context.Context, boxID int64, req CreateSubaccountRequest) (int64, Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.FailCreate != nil {
|
|
return 0, Action{}, f.FailCreate
|
|
}
|
|
f.CreatedSubaccounts++
|
|
id := f.nextID
|
|
f.nextID++
|
|
f.Subaccounts[id] = Subaccount{
|
|
ID: id, StorageBox: boxID,
|
|
Username: fmt.Sprintf("u629193-sub%d", id),
|
|
Server: fmt.Sprintf("u629193-sub%d.your-storagebox.de", id),
|
|
HomeDirectory: req.HomeDirectory,
|
|
AccessSettings: req.AccessSettings,
|
|
Labels: req.Labels,
|
|
}
|
|
return id, f.newAction("create_subaccount"), nil
|
|
}
|
|
|
|
func (f *Fake) ResetSubaccountPassword(_ context.Context, _, _ int64, _ string) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.ResetCalls++
|
|
return f.newAction("reset_subaccount_password"), nil
|
|
}
|
|
|
|
func (f *Fake) UpdateSubaccountAccess(_ context.Context, _, subID int64, as AccessSettings) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if s, ok := f.Subaccounts[subID]; ok {
|
|
s.AccessSettings = as
|
|
f.Subaccounts[subID] = s
|
|
}
|
|
return f.newAction("update_access_settings"), nil
|
|
}
|
|
|
|
func (f *Fake) DeleteSubaccount(_ context.Context, _, subID int64) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.Subaccounts, subID)
|
|
f.DeletedSubaccounts++
|
|
return f.newAction("delete_subaccount"), nil
|
|
}
|
|
|
|
func (f *Fake) ListStorageBoxes(_ context.Context, sel string) ([]StorageBox, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []StorageBox
|
|
for _, b := range f.Boxes {
|
|
if labelMatch(b.Labels, sel) {
|
|
out = append(out, b)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *Fake) GetStorageBox(_ context.Context, boxID int64) (StorageBox, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.GetBoxCalls++
|
|
if f.FailGetBox != nil {
|
|
return StorageBox{}, f.FailGetBox
|
|
}
|
|
b, ok := f.Boxes[boxID]
|
|
if !ok {
|
|
return StorageBox{}, fmt.Errorf("hetznerapi(fake): box %d not found", boxID)
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *Fake) CreateStorageBox(_ context.Context, req CreateBoxRequest) (int64, Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.FailCreate != nil {
|
|
return 0, Action{}, f.FailCreate
|
|
}
|
|
f.CreatedBoxes++
|
|
id := f.nextID
|
|
f.nextID++
|
|
f.Boxes[id] = StorageBox{
|
|
ID: id, Name: req.Name, Status: "active",
|
|
Username: fmt.Sprintf("u6294%d", id),
|
|
Server: fmt.Sprintf("u6294%d.your-storagebox.de", id),
|
|
Labels: req.Labels,
|
|
}
|
|
return id, f.newAction("create_storage_box"), nil
|
|
}
|
|
|
|
func (f *Fake) ChangeType(_ context.Context, _ int64, _ string) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.newAction("change_type"), nil
|
|
}
|
|
|
|
func (f *Fake) ResetBoxPassword(_ context.Context, _ int64, _ string) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.BoxResetCalls++
|
|
return f.newAction("reset_password"), nil
|
|
}
|
|
|
|
func (f *Fake) DeleteStorageBox(_ context.Context, boxID int64) (Action, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.Boxes, boxID)
|
|
f.DeletedBoxes++
|
|
return f.newAction("delete_storage_box"), nil
|
|
}
|
|
|
|
// WaitAction resolves based on the (already-terminal) fake action status — no polling.
|
|
func (f *Fake) WaitAction(_ context.Context, a Action) error {
|
|
if a.Status == "error" {
|
|
return actionErr(a)
|
|
}
|
|
return nil
|
|
}
|