From fad5573dd377d492e059d7cb80f305419ae8d655 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 9 Jul 2026 23:57:12 +0200 Subject: [PATCH] hub v0.41.0: OffsiteChecker (fill 90/95 + staleness >48h) + operator freeze lever (SLICE 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sibling checker over the controller report's offsite object: quota-fill warn/crit + the silently-stuck staleness detector (escrowed-only, red-proofed; nil-safe on pre-v0.109 reports; same-second tie-guard). SetOffsiteFrozen flips ONLY readonly on the exactly-1 labelled sub-account (SSH preserved); Freeze/Unfreeze buttons — manual only, never automatic. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- hub/CHANGELOG.md | 19 ++ hub/cmd/hub/main.go | 5 + hub/internal/monitor/offsite.go | 244 ++++++++++++++++++ hub/internal/monitor/offsite_test.go | 148 +++++++++++ hub/internal/offsite/offsite.go | 29 +++ hub/internal/offsite/offsite_test.go | 36 +++ hub/internal/web/configs.go | 40 +++ hub/internal/web/server.go | 8 + hub/internal/web/templates/config_form.html | 12 + .../web/templates/customer_unified.html | 2 + 10 files changed, 543 insertions(+) create mode 100644 hub/internal/monitor/offsite.go create mode 100644 hub/internal/monitor/offsite_test.go diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 65ce71e..f02005e 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,24 @@ # Felhom Hub — Changelog +## v0.41.0 — SLICE 4: OffsiteChecker (fill + staleness) + operator freeze lever (2026-07-09) + +The last build item of the offsite arc (pairs with controller v0.109.0's soft-quota gate + report object). + +- **`internal/monitor.OffsiteChecker`** — a SIBLING of StorageFillChecker (same born/persistent, + escalation-only, recovery-re-arm shape; NOT bolted onto the disk checkers), reading the controller + report's new `offsite` object. Two signals: **fill** (`repo_size_bytes` vs `quota_gb` at warn 90 / crit + 95 — quota 0 = dedicated, never alerts) and **staleness** (`offsite_stale`, warning): enabled+**escrowed** + but no run in >48h (or never) — the silently-STUCK detector; a recently-FAILING offsite is not stale + (`backup_failed` owns that), and pending/disabled targets never alert (normal onboarding — **companion + red-proof:** dropped the escrowed-only filter → the pending customer alerted → test FAILED). Nil-safe on + reports without the object (pre-v0.109 controllers). Tie-guard: duplicate same-second latest reports are + processed once per sweep. Same 60s sweep as the other checkers. +- **Freeze lever (operator, MANUAL only):** `Provisioner.SetOffsiteFrozen` — flips ONLY `readonly` on the + exactly-1 labelled sub-account via `UpdateSubaccountAccess` (SSH stays on; ambiguity refuses — tested), + wired to confirm-gated **Freeze/Unfreeze offsite** buttons next to Re-issue (shared model only; dedicated + is Hetzner-enforced). NEVER automatic — freezing also blocks prune, the customer's only way DOWN from + over-quota. Route `POST /configs/{id}/offsite-freeze` (`unfreeze=1` reverses); action logged, value-free. + ## v0.40.0 — SLICE 3: store the escrow password-hash + serve escrow status in the report ACK (2026-07-09) The hub-verified escrow auto-confirm chain, hub third (pairs with agent v0.79.0 + controller v0.108.0). diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 245b963..f98b7f8 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -404,6 +404,10 @@ func main() { // TASK H1: alert when a host's OPERATOR ACCESS is degraded — felhom-sshd down (with the operator // peer configured) or its config invalid. Transition-based, same 60s sweep. hostOOBChecker := monitor.NewHostOOBChecker(dataStore, dispatcher.ProcessEvent, logger) + // SLICE 4: offsite backup health from the controller report's `offsite` object — soft-quota fill + // (90/95% of quota_gb) + staleness (enabled+escrowed but no run >48h — the silently-stuck detector; + // run FAILURES already alert via backup_failed). Nil-safe on pre-v0.109 reports. Same sweep. + offsiteChecker := monitor.NewOffsiteChecker(dataStore, 0, dispatcher.ProcessEvent, logger) go func() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() @@ -420,6 +424,7 @@ func main() { storageFillChecker.Check() hostMgmtPlaneChecker.Check() hostOOBChecker.Check() + offsiteChecker.Check() } } }() diff --git a/hub/internal/monitor/offsite.go b/hub/internal/monitor/offsite.go new file mode 100644 index 0000000..5b6cba7 --- /dev/null +++ b/hub/internal/monitor/offsite.go @@ -0,0 +1,244 @@ +package monitor + +import ( + "encoding/json" + "fmt" + "log" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// OffsiteChecker (SLICE 4) watches each customer's offsite backup health from the controller report's +// `offsite` status object. Two independent signals, one checker (sibling of StorageFillChecker — same +// born/persistent, escalation-only emit, recovery re-arm shape; NOT bolted onto the disk checkers — +// different data source, different remedy text): +// +// - FILL: repo_size_bytes vs the shared-model soft quota (quota_gb>0) at warn 90% / crit 95% — the +// operator's early warning before the controller's own 100% run-refusal bites the customer. +// - STALENESS: enabled + escrowed but no run in >48h (or never) — the silently-STUCK detector. A +// RECENTLY-failing offsite is NOT stale (backup_failed already alerts it); staleness is the +// complement: nothing is even trying. Pending/disabled targets are normal onboarding, never stale. +// +// Reports without an `offsite` object (pre-v0.109 controllers, offbox not enabled) are skipped nil-safe. +type OffsiteChecker struct { + store *store.Store + logger *log.Logger + onEvent EventNotifyFunc + staleAfter time.Duration + now func() time.Time // injectable clock (tests) + + mu sync.Mutex + fillStates map[string]string // customerID → fill band + staleStates map[string]string // customerID → "ok" | "stale" +} + +const defaultOffsiteStaleAfter = 48 * time.Hour + +// offsiteReport mirrors the controller report's `offsite` object (v0.109.0). +type offsiteReport struct { + Enabled bool `json:"enabled"` + EscrowState string `json:"escrow_state"` + LastRun string `json:"last_run"` + LastStatus string `json:"last_status"` + SnapshotCount int `json:"snapshot_count"` + RepoSizeBytes int64 `json:"repo_size_bytes"` + QuotaGB int `json:"quota_gb"` +} + +// NewOffsiteChecker builds the checker. Same seeding philosophy as StorageFillChecker: already-breached +// customers are left UNSEEDED so their first Check emits (born/persistent); the dispatcher's cooldown +// dedups a hub restart. +func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteChecker { + if staleAfter <= 0 { + staleAfter = defaultOffsiteStaleAfter + } + oc := &OffsiteChecker{ + store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now, + fillStates: make(map[string]string), staleStates: make(map[string]string), + } + customers, err := s.GetCustomers() + if err != nil { + logger.Printf("[WARN] Offsite checker: failed to seed states: %v", err) + return oc + } + var seeded int + for _, c := range customers { + off := parseOffsite(c.ReportJSON) + if off == nil || s.IsCustomerBlocked(c.CustomerID) { + continue + } + if band := oc.fillBand(off); band == bandOK { + oc.fillStates[c.CustomerID] = bandOK + seeded++ + } + if !oc.isStale(off) { + oc.staleStates[c.CustomerID] = "ok" + } + } + logger.Printf("[INFO] Offsite checker initialized: fill warn=90%% crit=95%%, stale after %s, %d ok-seeded", staleAfter, seeded) + return oc +} + +func parseOffsite(reportJSON string) *offsiteReport { + var r struct { + Offsite *offsiteReport `json:"offsite"` + } + if json.Unmarshal([]byte(reportJSON), &r) != nil { + return nil + } + return r.Offsite // nil when absent (old controller / offbox not enabled) — the caller skips +} + +// fillBand maps the quota usage to a band. quota<=0 (dedicated/unset) never alerts. +func (oc *OffsiteChecker) fillBand(off *offsiteReport) string { + if off.QuotaGB <= 0 || off.RepoSizeBytes <= 0 { + return bandOK + } + pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) + return bandForPercent(pct, 90, 95) +} + +// isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or +// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale +// (backup_failed owns that signal). +func (oc *OffsiteChecker) isStale(off *offsiteReport) bool { + if !off.Enabled || off.EscrowState != "escrowed" { + return false + } + if off.LastRun == "" { + return true + } + t, err := time.Parse(time.RFC3339, off.LastRun) + if err != nil { + return true // unparseable = unknown-old — fail toward visibility + } + return oc.now().Sub(t) > oc.staleAfter +} + +// Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently. +func (oc *OffsiteChecker) Check() { + customers, err := oc.store.GetCustomers() + if err != nil { + oc.logger.Printf("[WARN] Offsite check failed: %v", err) + return + } + oc.mu.Lock() + defer oc.mu.Unlock() + + seen := make(map[string]bool, len(customers)) + for _, c := range customers { + // GetCustomers can return the same customer twice when two reports tie on received_at + // (second-resolution timestamps) — process each customer once per sweep. + if seen[c.CustomerID] { + continue + } + seen[c.CustomerID] = true + off := parseOffsite(c.ReportJSON) + if off == nil { + delete(oc.fillStates, c.CustomerID) // vanished object (disabled / downgraded) → re-arm + delete(oc.staleStates, c.CustomerID) + continue + } + if oc.store.IsCustomerBlocked(c.CustomerID) { + delete(oc.fillStates, c.CustomerID) + delete(oc.staleStates, c.CustomerID) + continue + } + + // FILL (quota>0 only) + newBand := oc.fillBand(off) + if bandRank(newBand) > bandRank(oc.fillStates[c.CustomerID]) { + oc.emitFill(c.CustomerID, off, newBand) + } + oc.fillStates[c.CustomerID] = newBand + + // STALENESS (binary, warn-severity) + newStale := "ok" + if oc.isStale(off) { + newStale = "stale" + } + if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" { + oc.emitStale(c.CustomerID, off) + } + oc.staleStates[c.CustomerID] = newStale + } + for k := range oc.fillStates { + if !seen[k] { + delete(oc.fillStates, k) + } + } + for k := range oc.staleStates { + if !seen[k] { + delete(oc.staleStates, k) + } + } +} + +// GetFillState / GetStaleState expose current states for tests. +func (oc *OffsiteChecker) GetFillState(customerID string) string { + oc.mu.Lock() + defer oc.mu.Unlock() + if s := oc.fillStates[customerID]; s != "" { + return s + } + return "unknown" +} + +func (oc *OffsiteChecker) GetStaleState(customerID string) string { + oc.mu.Lock() + defer oc.mu.Unlock() + if s := oc.staleStates[customerID]; s != "" { + return s + } + return "unknown" +} + +func (oc *OffsiteChecker) emitFill(customerID string, off *offsiteReport, band string) { + usedGB := off.RepoSizeBytes >> 30 + pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) + var eventType, severity, message string + switch band { + case bandCritical: + eventType, severity = "offsite_fill_critical", "critical" + message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used) — at 100%% new offsite runs are refused; consider the freeze lever or a bigger quota", customerID, pct, off.QuotaGB, usedGB) + case bandWarning: + eventType, severity = "offsite_fill_warning", "warning" + message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used)", customerID, pct, off.QuotaGB, usedGB) + default: + return + } + details, _ := json.Marshal(map[string]any{ + "customer_id": customerID, "quota_gb": off.QuotaGB, "repo_size_bytes": off.RepoSizeBytes, "percent": pct, + }) + oc.logger.Printf("[INFO] Offsite fill: %s %.0f%% (%s)", customerID, pct, eventType) + if _, err := oc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { + oc.logger.Printf("[WARN] Failed to save offsite fill event for %s: %v", customerID, err) + return + } + if oc.onEvent != nil { + oc.onEvent(customerID, eventType, severity, message, string(details), "hub") + } +} + +func (oc *OffsiteChecker) emitStale(customerID string, off *offsiteReport) { + age := "never ran" + if off.LastRun != "" { + if t, err := time.Parse(time.RFC3339, off.LastRun); err == nil { + age = fmt.Sprintf("last run %s ago", oc.now().Sub(t).Round(time.Hour)) + } + } + message := fmt.Sprintf("Customer %s: offsite backup is STALE — enabled + escrowed but %s (threshold %s). The offsite leg is silently not running; check the controller/schedule", customerID, age, oc.staleAfter) + details, _ := json.Marshal(map[string]any{ + "customer_id": customerID, "last_run": off.LastRun, "last_status": off.LastStatus, "stale_after": oc.staleAfter.String(), + }) + oc.logger.Printf("[INFO] Offsite staleness: %s (%s)", customerID, age) + if _, err := oc.store.SaveEvent(customerID, "offsite_stale", "warning", message, string(details), "hub"); err != nil { + oc.logger.Printf("[WARN] Failed to save offsite staleness event for %s: %v", customerID, err) + return + } + if oc.onEvent != nil { + oc.onEvent(customerID, "offsite_stale", "warning", message, string(details), "hub") + } +} diff --git a/hub/internal/monitor/offsite_test.go b/hub/internal/monitor/offsite_test.go new file mode 100644 index 0000000..ed71141 --- /dev/null +++ b/hub/internal/monitor/offsite_test.go @@ -0,0 +1,148 @@ +package monitor + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// saveOffsiteReport records a customer report whose report_json carries the given offsite object +// (or none, when raw==""). The 1.1s sleep before a RE-save of the same customer makes received_at +// strictly increasing (second resolution) so GetCustomers' MAX(received_at) picks the new row +// deterministically. +var offsiteSaved = map[string]bool{} + +func saveOffsiteReport(t *testing.T, st interface { + SaveReport(string, []byte) error +}, customerID, raw string) { + t.Helper() + if offsiteSaved[customerID] { + time.Sleep(1100 * time.Millisecond) + } + offsiteSaved[customerID] = true + body := `{"customer_id":"` + customerID + `"` + if raw != "" { + body += `,"offsite":` + raw + } + body += `}` + if err := st.SaveReport(customerID, []byte(body)); err != nil { + t.Fatal(err) + } +} + +func offsiteJSON(enabled bool, escrow, lastRun, lastStatus string, sizeBytes int64, quotaGB int) string { + return fmt.Sprintf(`{"enabled":%v,"escrow_state":%q,"last_run":%q,"last_status":%q,"snapshot_count":3,"repo_size_bytes":%d,"quota_gb":%d}`, + enabled, escrow, lastRun, lastStatus, sizeBytes, quotaGB) +} + +// Scenario D — staleness: enabled+escrowed with no run >48h (or never) alerts once (deduped across +// sweeps); a pending customer NEVER alerts (normal onboarding — the red-proofed filter); recovery re-arms. +func TestOffsite_StalenessAlert(t *testing.T) { + st := newDiskStore(t) + now := time.Now().UTC() + old := now.Add(-72 * time.Hour).Format(time.RFC3339) + fresh := now.Add(-1 * time.Hour).Format(time.RFC3339) + + // stale-c: escrowed, last run 72h ago → stale. pend-c: PENDING with the same old run → silent. + saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50)) + saveOffsiteReport(t, st, "pend-c", offsiteJSON(true, "pending", old, "", 1<<30, 50)) + + var types []string + oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog()) + oc.Check() + if got := count(types, "offsite_stale"); got != 1 { + t.Fatalf("want exactly 1 offsite_stale (stale-c only; pending must NEVER alert), got %d (%v)", got, types) + } + if oc.GetStaleState("pend-c") != "ok" { + t.Fatalf("a pending customer must not be stale, got %s", oc.GetStaleState("pend-c")) + } + // repeated sweep with the same data → NO re-page (dedupe) + oc.Check() + if got := count(types, "offsite_stale"); got != 1 { + t.Fatalf("staleness must not re-page every sweep, got %d", got) + } + // recovery: a fresh run clears + re-arms; going stale again re-alerts + saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", fresh, "ok", 1<<30, 50)) + oc.Check() + if oc.GetStaleState("stale-c") != "ok" { + t.Fatal("recovery must clear the stale state") + } + saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50)) + oc.Check() + if got := count(types, "offsite_stale"); got != 2 { + t.Fatalf("re-staleness after recovery must alert again, got %d", got) + } + // never-ran escrowed customer is stale too + saveOffsiteReport(t, st, "never-c", offsiteJSON(true, "escrowed", "", "", 0, 50)) + oc.Check() + if got := count(types, "offsite_stale"); got != 3 { + t.Fatalf("a never-ran escrowed target must be stale, got %d", got) + } +} + +// Scenario E (fill) — 90/95 of quota_gb; quota 0 never alerts; escalation-only; recovery re-arms. +func TestOffsite_FillAlert(t *testing.T) { + st := newDiskStore(t) + fresh := time.Now().UTC().Format(time.RFC3339) + gb := int64(1) << 30 + + var types, sevs []string + onEvent := func(_, et, sev, _, _, _ string) { types = append(types, et); sevs = append(sevs, sev) } + saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 40*gb, 50)) // 80% — under warn + saveOffsiteReport(t, st, "noq-c", offsiteJSON(true, "escrowed", fresh, "ok", 900*gb, 0)) // dedicated: quota 0 + oc := NewOffsiteChecker(st, 48*time.Hour, onEvent, quietLog()) + oc.Check() + if len(types) != 0 { + t.Fatalf("80%% and quota-0 must not alert, got %v", types) + } + + saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 46*gb, 50)) // 92% warn + oc.Check() + if count(types, "offsite_fill_warning") != 1 { + t.Fatalf("92%% must warn once, got %v", types) + } + oc.Check() // same data — no re-page + if len(types) != 1 { + t.Fatalf("no re-page on an unchanged band, got %v", types) + } + saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 48*gb, 50)) // 96% crit + oc.Check() + if count(types, "offsite_fill_critical") != 1 || sevs[len(sevs)-1] != "critical" { + t.Fatalf("96%% must escalate to critical, got types=%v sevs=%v", types, sevs) + } + // recovery re-arms + saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 10*gb, 50)) + oc.Check() + if oc.GetFillState("fill-c") != bandOK { + t.Fatal("recovery must re-arm the fill state") + } +} + +// Nil-safety — reports without an offsite object (pre-v0.109 controllers / offbox disabled) are skipped: +// no alert, no state. +func TestOffsite_NilSafeOnOldReports(t *testing.T) { + st := newDiskStore(t) + saveOffsiteReport(t, st, "old-c", "") // no offsite key at all + var types []string + oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog()) + oc.Check() + if len(types) != 0 { + t.Fatalf("a report without an offsite object must never alert, got %v", types) + } + if oc.GetStaleState("old-c") != "unknown" || oc.GetFillState("old-c") != "unknown" { + t.Fatal("no state may exist for a customer without an offsite object") + } +} + +func count(list []string, want string) int { + n := 0 + for _, s := range list { + if s == want { + n++ + } + } + return n +} + +var _ = strings.Contains // keep strings import if unused by future edits diff --git a/hub/internal/offsite/offsite.go b/hub/internal/offsite/offsite.go index bc42bb9..456e9d6 100644 --- a/hub/internal/offsite/offsite.go +++ b/hub/internal/offsite/offsite.go @@ -283,6 +283,35 @@ func (p *Provisioner) provisionDedicated(ctx context.Context, customerID string, return &Descriptor{Enabled: true, Type: "dedicated", Host: box.Server, User: box.Username, Port: sftpPort, RepoPath: repoPath, BoxType: boxType}, nil } +// SetOffsiteFrozen freezes/unfreezes the customer's SHARED sub-account (SLICE 4: readonly access) — an +// OPERATOR lever, NEVER automatic: freezing also blocks prune/forget, which is the customer's only way +// DOWN from over-quota, so only a human weighs that trade-off. Same exactly-1 label guard as the +// re-issue. Preserves the sub-account's other access settings (SSH must stay on — only readonly flips). +// Dedicated boxes have no freeze path (Hetzner enforces their size physically; the UI hides the button). +func (p *Provisioner) SetOffsiteFrozen(ctx context.Context, customerID string, frozen bool) error { + if p.PoolBoxID == 0 { + return fmt.Errorf("offsite: no shared pool box configured") + } + subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) + if err != nil { + return fmt.Errorf("offsite: freeze lookup: %w", err) + } + if len(subs) != 1 { + return fmt.Errorf("offsite: freeze needs exactly 1 sub-account labelled for %s, found %d — refusing", customerID, len(subs)) + } + as := subs[0].AccessSettings + as.Readonly = frozen + act, err := p.API.UpdateSubaccountAccess(ctx, p.PoolBoxID, subs[0].ID, as) + if err != nil { + return fmt.Errorf("offsite: update access: %w", err) + } + if err := p.API.WaitAction(ctx, act); err != nil { + return fmt.Errorf("offsite: freeze action: %w", err) + } + p.logf("[offsite] set frozen=%v (readonly) for %s (subaccount %d)", frozen, customerID, subs[0].ID) + return nil +} + // MergeDescriptor merges the offsite descriptor under the "offsite" key of a ConfigJSON object, preserving // all other keys. Returns the new ConfigJSON string. NEVER carries a secret (Descriptor is non-secret). func MergeDescriptor(configJSON string, d *Descriptor) (string, error) { diff --git a/hub/internal/offsite/offsite_test.go b/hub/internal/offsite/offsite_test.go index e9c49ce..b3b789e 100644 --- a/hub/internal/offsite/offsite_test.go +++ b/hub/internal/offsite/offsite_test.go @@ -211,6 +211,42 @@ func TestReissue_RefusesAmbiguousLookup(t *testing.T) { } } +// Scenario E (SLICE 4) — the freeze lever flips ONLY readonly on the exactly-1 labelled sub-account +// (SSH stays on — a freeze must not cut access, just writes); ambiguity refuses; unfreeze reverses. +func TestFreeze_SharedTogglesReadonlyOnly(t *testing.T) { + p, fake, _ := newTestProvisioner(t) + if _, err := p.ProvisionOffsite(context.Background(), "cust-fz", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { + t.Fatal(err) + } + if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", true); err != nil { + t.Fatalf("freeze: %v", err) + } + subs, _ := fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz") + if len(subs) != 1 || !subs[0].AccessSettings.Readonly { + t.Fatalf("freeze must set readonly on the labelled sub-account: %+v", subs) + } + if !subs[0].AccessSettings.SSHEnabled { + t.Fatal("freeze must NOT disable SSH — only readonly flips") + } + if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", false); err != nil { + t.Fatalf("unfreeze: %v", err) + } + subs, _ = fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz") + if subs[0].AccessSettings.Readonly { + t.Fatal("unfreeze must clear readonly") + } + // nothing provisioned → refuse + if err := p.SetOffsiteFrozen(context.Background(), "cust-none", true); err == nil { + t.Fatal("freeze with no labelled sub-account must refuse") + } + // ambiguous → refuse + fake.Subaccounts[71] = hetznerapi.Subaccount{ID: 71, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}} + fake.Subaccounts[72] = hetznerapi.Subaccount{ID: 72, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}} + if err := p.SetOffsiteFrozen(context.Background(), "cust-amb2", true); err == nil || !strings.Contains(err.Error(), "exactly 1") { + t.Fatalf("ambiguous freeze must refuse, got %v", err) + } +} + // Scenario B — enable dedicated → box provisioned. func TestProvision_Dedicated(t *testing.T) { p, fake, st := newTestProvisioner(t) diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index 12019a2..2a7ea45 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -581,6 +581,46 @@ func (s *Server) handleOffsiteReissue(w http.ResponseWriter, r *http.Request, cu http.Redirect(w, r, "/customers/"+customerID+"?flash=offsite_reissued", http.StatusSeeOther) } +// handleOffsiteFreeze (SLICE 4) freezes/unfreezes the customer's shared sub-account (readonly) — an +// OPERATOR lever, never automatic (freezing also blocks prune, the customer's only way down from +// over-quota). Shared model only; the exactly-1 label guard lives in the provisioner. Action logged, +// no secrets involved. +func (s *Server) handleOffsiteFreeze(w http.ResponseWriter, r *http.Request, customerID string, frozen bool) { + if s.offsite == nil { + http.Error(w, "Offsite provisioning is not configured on this hub", http.StatusBadGateway) + return + } + cfg, err := s.store.GetCustomerConfig(customerID) + if err != nil || cfg == nil { + http.NotFound(w, r) + return + } + var overrides struct { + Offsite struct { + Enabled bool `json:"enabled"` + Type string `json:"type"` + } `json:"offsite"` + } + _ = json.Unmarshal([]byte(cfg.ConfigJSON), &overrides) + if !overrides.Offsite.Enabled || overrides.Offsite.Type != "shared" { + http.Error(w, "Freeze applies to a provisioned SHARED offsite tier only", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 2*time.Minute) + defer cancel() + if err := s.offsite.SetOffsiteFrozen(ctx, customerID, frozen); err != nil { + s.logger.Printf("[ERROR] offsite freeze(%v) for %s: %v", frozen, customerID, err) + http.Error(w, "Offsite freeze/unfreeze failed: "+err.Error(), http.StatusBadGateway) + return + } + s.logger.Printf("[INFO] offsite frozen=%v (readonly) for %s (operator action)", frozen, customerID) + flash := "offsite_frozen" + if !frozen { + flash = "offsite_unfrozen" + } + http.Redirect(w, r, "/customers/"+customerID+"?flash="+flash, http.StatusSeeOther) +} + // handleConfigDelete deletes a customer config. func (s *Server) handleConfigDelete(w http.ResponseWriter, r *http.Request, customerID string) { if err := s.store.DeleteCustomerConfig(customerID); err != nil { diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index facd202..444e8d0 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -351,6 +351,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } else { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) } + case strings.HasPrefix(path, "/configs/") && strings.HasSuffix(path, "/offsite-freeze"): + customerID := strings.TrimPrefix(path, "/configs/") + customerID = strings.TrimSuffix(customerID, "/offsite-freeze") + if r.Method == http.MethodPost { + s.handleOffsiteFreeze(w, r, customerID, r.FormValue("unfreeze") != "1") + } else { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } case strings.HasPrefix(path, "/configs/") && strings.HasSuffix(path, "/preview"): customerID := strings.TrimPrefix(path, "/configs/") customerID = strings.TrimSuffix(customerID, "/preview") diff --git a/hub/internal/web/templates/config_form.html b/hub/internal/web/templates/config_form.html index ee0b566..8c3175a 100644 --- a/hub/internal/web/templates/config_form.html +++ b/hub/internal/web/templates/config_form.html @@ -141,6 +141,18 @@ formaction="/configs/{{$.Config.CustomerID}}/offsite-reissue" formmethod="POST" onclick="return confirm('Re-issue the offsite credentials?\n\nThe box password is reset and a fresh one-time password is staged for the controller. Guests with a working installed key are unaffected (key-auth-first); a stuck fresh guest picks the new password up on its next config refresh.')"> Re-issue offsite credentials + {{if eq (index . "type") "shared"}} + + + + {{end}} {{end}}{{end}}{{end}} diff --git a/hub/internal/web/templates/customer_unified.html b/hub/internal/web/templates/customer_unified.html index 397f190..54d8e98 100644 --- a/hub/internal/web/templates/customer_unified.html +++ b/hub/internal/web/templates/customer_unified.html @@ -45,6 +45,8 @@ {{else if eq .Flash "updated"}}Configuration updated. {{else if eq .Flash "password_regenerated"}}Retrieval password regenerated. {{else if eq .Flash "offsite_reissued"}}Offsite credentials re-issued — a fresh one-time password is staged; the controller picks it up on its next config refresh. + {{else if eq .Flash "offsite_frozen"}}Offsite storage FROZEN (read-only) — new backups and prune will fail until unfrozen. + {{else if eq .Flash "offsite_unfrozen"}}Offsite storage unfrozen — read-write restored. {{else if eq .Flash "blocked"}}Customer blocked — hidden from Dashboard. {{else if eq .Flash "unblocked"}}Customer unblocked — visible on Dashboard again. {{end}}