hub v0.129.0: operator raises ONE clean-up window's cap (R-833); restore-beside script + runbooks (R-834)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/api/offsite_test.go

@@ -138,3 +138,25 @@ func TestOffsiteKeyEndpoints_AuthAndNoPasswordInAnyResponse(t *testing.T) {
 		t.Fatalf("service not reached: pub=%q fp=%q", f.gotPub, f.gotFP)
 	}
 }
+
+// R-833: a box cannot raise its own window cap. No box-authenticated route sets a grant, and a
+// window-open body that names a cap is not a grant. Asserted on the consequence: the store holds no
+// grant after every box call.
+func TestOffsiteWindow_BoxCannotGrantItself(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "pp"})
+	h.SetOffsiteKeyService(&fakeKeySvc{})
+	for _, c := range []struct{ path, body string }{
+		{"/api/v1/offsite/window-open/c1", `{"count_before":40,"max_remove":400}`},
+		{"/api/v1/offsite/window-grant/c1", `{"max_remove":400}`},
+		{"/api/v1/offsite/window-large-grant/c1", `{"max_remove":400}`},
+		{"/offsite/window-grant/c1", `max_remove=400`},
+	} {
+		req := httptest.NewRequest(http.MethodPost, c.path, strings.NewReader(c.body))
+		req.Header.Set("Authorization", "Bearer ckey")
+		h.ServeHTTP(httptest.NewRecorder(), req)
+	}
+	if ok, n := st.TakeOffsiteWindowGrant("c1"); ok || n != 0 {
+		t.Fatalf("a box call left a grant (ok=%v max=%d)", ok, n)
+	}
+}

hub/internal/notify/dispatcher.go

@@ -678,6 +678,8 @@ var operatorOnlyEvents = map[string]bool{
 	"offsite_window_drop":         true,
 	"offsite_window_failed":       true,
 	"offsite_prune_guard_refused": true,
+	// R-833 (v0.129.0): the operator raised ONE window's removal cap — an operator act, logged.
+	"offsite_window_large_grant": true,
 	// R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow
 	// blobs. A customer can take no action on it — the remedy is the operator's inspection of the
 	// off-site tier — and the text is operator-grade English naming host ids and retained-blob

hub/internal/offsitekeys/service.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"strconv"
 	"time"
 
 	"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
@@ -208,6 +209,7 @@ const (
 	EventWindowDrop   = "offsite_window_drop"         // error: more snapshots went than a window may remove
 	EventWindowFailed = "offsite_window_failed"       // warning: a window errored or was left open
 	EventGuardRefused = "offsite_prune_guard_refused" // error: the box's fake-snapshot guard refused (R-822)
+	EventWindowLargeGrant = "offsite_window_large_grant" // warning: the operator raised one window's cap (R-833)
 	windowLength      = 20 * time.Minute
 	windowCadence     = 6*24*time.Hour + 12*time.Hour // "weekly", with slack for the night chain's drift
 )
@@ -243,10 +245,44 @@ func MaxRemove(countBefore int) int {
 	return n
 }
 
+// windowCap is the cap a window was OPENED with (an operator grant may have raised it, R-833); rows
+// from before v0.129.0 carry none and fall back to the default.
+func windowCap(w *store.OffsiteWindow) int {
+	if w.MaxRemove > 0 {
+		return w.MaxRemove
+	}
+	return MaxRemove(w.CountBefore)
+}
+
+// MaxRemoveGrantCeiling bounds an operator's raised cap: a typo of an extra zero must not turn one
+// window into "remove anything". A real backlog above it is cleared over several granted windows.
+const MaxRemoveGrantCeiling = 500
+
+// GrantLargeWindow is the OPERATOR's one-shot grant with a raised cap for that one window (R-833).
+// It is reachable only from the operator's hub login (internal/web), never from the box API. The grant
+// is consumed by the next window; the window after it has the default cap again. Logged as an
+// operator event.
+func (s *Service) GrantLargeWindow(customerID string, maxRemove int) error {
+	if maxRemove < 1 || maxRemove > MaxRemoveGrantCeiling {
+		return fmt.Errorf("offsitekeys: max_remove %d is outside 1..%d", maxRemove, MaxRemoveGrantCeiling)
+	}
+	if c, err := s.Store.GetCustomerConfig(customerID); err != nil || c == nil {
+		return fmt.Errorf("offsitekeys: no customer %q", customerID)
+	}
+	if err := s.Store.GrantOffsiteWindowOnceMax(customerID, maxRemove); err != nil {
+		return err
+	}
+	s.logf("[WARN] offsitekeys: operator granted ONE clean-up window for %s with a raised cap of %d", customerID, maxRemove)
+	s.event(customerID, EventWindowLargeGrant, "warning",
+		fmt.Sprintf("Off-site clean-up: the operator granted one window with a raised removal cap of %d (the fake-snapshot guard still applies).", maxRemove),
+		map[string]any{"max_remove": maxRemove})
+	return nil
+}
+
 // OpenWindowFor decides and, if due, opens the window: a deleting line for the box's CONFIRMED key is
 // prepended (first match wins), and a ledger row bounds it to 20 minutes.
 func (s *Service) OpenWindowFor(ctx context.Context, customerID string, countBefore int) (WindowGrant, error) {
-	oneShot := s.Store.TakeOffsiteWindowGrant(customerID)
+	oneShot, raisedMax := s.Store.TakeOffsiteWindowGrant(customerID)
 	last := s.Store.LastOffsiteWindowOpened(customerID)
 	due := last.IsZero() || s.now().Sub(last) >= windowCadence
 	if !oneShot && !(s.Store.OffsiteWindowsEnabled() && due) {
@@ -268,15 +304,23 @@ func (s *Service) OpenWindowFor(ctx context.Context, customerID string, countBef
 		return WindowGrant{}, err
 	}
 	now := s.now()
-	id, err := s.Store.OpenOffsiteWindowRow(customerID, now.Add(windowLength), countBefore)
+	// R-833: an operator grant may RAISE the cap for this one window (never lower it). Only the count
+	// cap moves; the box's fake-snapshot guard still runs in full against NewestAllowed.
+	maxRemove := MaxRemove(countBefore)
+	raised := raisedMax > maxRemove
+	if raised {
+		maxRemove = raisedMax
+	}
+	id, err := s.Store.OpenOffsiteWindowRow(customerID, now.Add(windowLength), countBefore, maxRemove)
 	if err != nil {
 		// The line is written; close it rather than leave a deleting line without a ledger row.
 		_ = s.Reg.CloseWindow(ctx, t, pw)
 		return WindowGrant{}, err
 	}
-	g := WindowGrant{Granted: true, WindowID: id, NewestAllowed: now.UTC(), MaxRemove: MaxRemove(countBefore)}
-	s.logf("[WARN] offsitekeys: clean-up window %d OPENED for %s (key %s, %d snapshot(s), max %d removed, closes by %s, one-shot=%v)",
-		id, customerID, k.Fingerprint, countBefore, g.MaxRemove, now.Add(windowLength).UTC().Format(time.RFC3339), oneShot)
+	g := WindowGrant{Granted: true, WindowID: id, NewestAllowed: now.UTC(), MaxRemove: maxRemove}
+	s.logf("[WARN] offsitekeys: clean-up window %d OPENED for %s (key %s, %d snapshot(s), max %d removed%s, closes by %s, one-shot=%v)",
+		id, customerID, k.Fingerprint, countBefore, g.MaxRemove, map[bool]string{true: " — OPERATOR-RAISED cap (default " + strconv.Itoa(MaxRemove(countBefore)) + ")", false: ""}[raised],
+		now.Add(windowLength).UTC().Format(time.RFC3339), oneShot)
 	return g, nil
 }
 
@@ -297,13 +341,14 @@ func (s *Service) CloseWindowFor(ctx context.Context, customerID string, r Windo
 		s.logf("[WARN] offsitekeys: ledger close %d: %v", w.ID, err)
 	}
 	drop := w.CountBefore - r.CountAfter
+	allowed := windowCap(w)
 	s.logf("[INFO] offsitekeys: clean-up window %d CLOSED for %s: outcome=%s, %d -> %d (drop %d, allowed %d)",
-		w.ID, customerID, r.Outcome, w.CountBefore, r.CountAfter, drop, MaxRemove(w.CountBefore))
+		w.ID, customerID, r.Outcome, w.CountBefore, r.CountAfter, drop, allowed)
 	details := map[string]any{"window_id": w.ID, "count_before": w.CountBefore, "count_after": r.CountAfter, "outcome": r.Outcome, "reason": r.Reason}
 	switch {
-	case drop > MaxRemove(w.CountBefore):
+	case drop > allowed:
 		s.event(customerID, EventWindowDrop, "error",
-			fmt.Sprintf("Off-site clean-up window %d: the snapshot count fell %d -> %d, more than a window may remove (%d).", w.ID, w.CountBefore, r.CountAfter, MaxRemove(w.CountBefore)), details)
+			fmt.Sprintf("Off-site clean-up window %d: the snapshot count fell %d -> %d, more than a window may remove (%d).", w.ID, w.CountBefore, r.CountAfter, allowed), details)
 	case r.Outcome == "guard-refused":
 		s.event(customerID, EventGuardRefused, "error",
 			fmt.Sprintf("Off-site clean-up window %d: the box's fake-snapshot guard refused to prune — nothing was deleted: %s", w.ID, r.Reason), details)

hub/internal/offsitekeys/service_test.go

@@ -220,3 +220,76 @@ func TestMaxRemove_HonestWeekFits(t *testing.T) {
 		t.Fatal("floor of 5 lost")
 	}
 }
+
+// R-833: after a long gap the honest backlog exceeds half the snapshots, and the default cap makes the
+// box's guard refuse every window. The operator's raised-cap grant opens ONE window with a larger cap;
+// it is consumed, the next window has the default cap again, the close check uses the raised cap (no
+// false drop alarm), and the grant is an operator event. Red-proof: drop the `if raised` assignment in
+// OpenWindowFor and the first assertion fails.
+func TestWindow_RaisedCapIsOneWindowOnly(t *testing.T) {
+	s, _, events := svcFixture(t)
+	ctx := context.Background()
+	pub, fp := newKey(t)
+	if _, err := s.RegisterKey(ctx, "c1", pub); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.ConfirmKey(ctx, "c1", fp); err != nil {
+		t.Fatal(err)
+	}
+	// Without the raised grant: a plain one-shot gives half of 40 = 20.
+	_ = s.Store.GrantOffsiteWindowOnce("c1")
+	g0, err := s.OpenWindowFor(ctx, "c1", 40)
+	if err != nil || !g0.Granted || g0.MaxRemove != 20 {
+		t.Fatalf("plain grant: %+v %v — want the default cap 20", g0, err)
+	}
+	_ = s.CloseWindowFor(ctx, "c1", WindowResult{WindowID: g0.WindowID, CountAfter: 40, Outcome: "guard-refused"})
+
+	if err := s.GrantLargeWindow("c1", 30); err != nil {
+		t.Fatal(err)
+	}
+	g, err := s.OpenWindowFor(ctx, "c1", 40)
+	if err != nil || !g.Granted || g.MaxRemove != 30 {
+		t.Fatalf("raised grant: %+v %v — want MaxRemove 30", g, err)
+	}
+	*events = nil
+	if err := s.CloseWindowFor(ctx, "c1", WindowResult{WindowID: g.WindowID, CountAfter: 12, Outcome: "pruned"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range *events {
+		if e == EventWindowDrop {
+			t.Fatal("a drop of 28 under a raised cap of 30 alarmed — the close check ignored the window's own cap")
+		}
+	}
+	// The grant was consumed: no window without a new grant (weekly windows are off here) …
+	if g2, _ := s.OpenWindowFor(ctx, "c1", 12); g2.Granted {
+		t.Fatal("the raised grant was not consumed")
+	}
+	// … and the next granted window is back on the default cap.
+	_ = s.Store.GrantOffsiteWindowOnce("c1")
+	g3, _ := s.OpenWindowFor(ctx, "c1", 40)
+	if !g3.Granted || g3.MaxRemove != 20 {
+		t.Fatalf("next window: %+v — want the default cap 20 again", g3)
+	}
+}
+
+// The grant is an operator event, bounded, and only for a known customer.
+func TestGrantLargeWindow_EventAndBounds(t *testing.T) {
+	s, _, events := svcFixture(t)
+	for _, bad := range []int{0, -1, MaxRemoveGrantCeiling + 1} {
+		if err := s.GrantLargeWindow("c1", bad); err == nil {
+			t.Fatalf("max_remove %d accepted", bad)
+		}
+	}
+	if err := s.GrantLargeWindow("nobody", 10); err == nil {
+		t.Fatal("a grant for an unknown customer was accepted")
+	}
+	if ok, _ := s.Store.TakeOffsiteWindowGrant("c1"); ok {
+		t.Fatal("a refused grant left a grant behind")
+	}
+	if err := s.GrantLargeWindow("c1", 10); err != nil {
+		t.Fatal(err)
+	}
+	if len(*events) != 1 || (*events)[0] != EventWindowLargeGrant {
+		t.Fatalf("events = %v, want one %s", *events, EventWindowLargeGrant)
+	}
+}

hub/internal/store/offsite_keys.go

@@ -2,6 +2,9 @@ package store
 
 import (
 	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -72,12 +75,15 @@ type OffsiteWindow struct {
 	CountAfter  int
 	BoxResult   string
 	CloseReason string
+	// MaxRemove is the cap this window was opened with (R-833: an operator grant may raise it for one
+	// window). 0 on rows written before v0.129.0 — readers fall back to the default cap then.
+	MaxRemove int
 }
 
-// OpenOffsiteWindowRow records a window the hub just opened.
-func (s *Store) OpenOffsiteWindowRow(customerID string, closesBy time.Time, countBefore int) (int64, error) {
-	res, err := s.db.Exec(`INSERT INTO offsite_windows (customer_id, opened_at, closes_by, count_before) VALUES (?, datetime('now'), ?, ?)`,
-		customerID, closesBy.UTC().Format("2006-01-02 15:04:05"), countBefore)
+// OpenOffsiteWindowRow records a window the hub just opened, with the cap it was opened under.
+func (s *Store) OpenOffsiteWindowRow(customerID string, closesBy time.Time, countBefore, maxRemove int) (int64, error) {
+	res, err := s.db.Exec(`INSERT INTO offsite_windows (customer_id, opened_at, closes_by, count_before, max_remove) VALUES (?, datetime('now'), ?, ?, ?)`,
+		customerID, closesBy.UTC().Format("2006-01-02 15:04:05"), countBefore, maxRemove)
 	if err != nil {
 		return 0, err
 	}
@@ -100,9 +106,9 @@ func (s *Store) GetOffsiteWindow(id int64) (*OffsiteWindow, error) {
 	var w OffsiteWindow
 	var opened, closesBy string
 	var closed, boxRes, reason sql.NullString
-	var before, after sql.NullInt64
-	err := s.db.QueryRow(`SELECT id, customer_id, opened_at, closes_by, closed_at, count_before, count_after, box_result, close_reason FROM offsite_windows WHERE id = ?`, id).
-		Scan(&w.ID, &w.CustomerID, &opened, &closesBy, &closed, &before, &after, &boxRes, &reason)
+	var before, after, maxRm sql.NullInt64
+	err := s.db.QueryRow(`SELECT id, customer_id, opened_at, closes_by, closed_at, count_before, count_after, box_result, close_reason, max_remove FROM offsite_windows WHERE id = ?`, id).
+		Scan(&w.ID, &w.CustomerID, &opened, &closesBy, &closed, &before, &after, &boxRes, &reason, &maxRm)
 	if err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -115,6 +121,7 @@ func (s *Store) GetOffsiteWindow(id int64) (*OffsiteWindow, error) {
 	}
 	w.CountBefore, w.CountAfter = int(before.Int64), int(after.Int64)
 	w.BoxResult, w.CloseReason = boxRes.String, reason.String
+	w.MaxRemove = int(maxRm.Int64)
 	return &w, nil
 }
 
@@ -166,14 +173,34 @@ func (s *Store) GrantOffsiteWindowOnce(customerID string) error {
 	return s.setSetting("offsite_window_grant:"+customerID, "1")
 }
 
-// TakeOffsiteWindowGrant consumes a one-shot grant; true when one was present.
-func (s *Store) TakeOffsiteWindowGrant(customerID string) bool {
+// GrantOffsiteWindowOnceMax is the operator's one-shot grant that ALSO raises the removal cap for that
+// one window (R-833: after a long gap the honest backlog exceeds half the snapshots and the default cap
+// refuses every window). The raised cap is consumed with the grant; the next window has the default.
+func (s *Store) GrantOffsiteWindowOnceMax(customerID string, maxRemove int) error {
+	if maxRemove <= 0 {
+		return fmt.Errorf("max_remove must be positive, got %d", maxRemove)
+	}
+	return s.setSetting("offsite_window_grant:"+customerID, "max:"+strconv.Itoa(maxRemove))
+}
+
+// TakeOffsiteWindowGrant consumes a one-shot grant: granted is true when one was present, and
+// maxRemove is the operator's raised cap for that window (0 = none, use the default).
+func (s *Store) TakeOffsiteWindowGrant(customerID string) (granted bool, maxRemove int) {
 	k := "offsite_window_grant:" + customerID
-	if s.getSetting(k) != "1" {
-		return false
+	v := s.getSetting(k)
+	switch {
+	case v == "1":
+	case strings.HasPrefix(v, "max:"):
+		n, err := strconv.Atoi(strings.TrimPrefix(v, "max:"))
+		if err != nil || n <= 0 {
+			n = 0 // a malformed value still grants the window, at the default cap
+		}
+		maxRemove = n
+	default:
+		return false, 0
 	}
 	_ = s.setSetting(k, "")
-	return true
+	return true, maxRemove
 }
 
 // ForceOffsiteWindowDueForTest back-dates a window's closes_by. TEST-ONLY.

hub/internal/store/store.go

@@ -856,6 +856,8 @@ func (s *Store) migrate() error {
 	`); err != nil {
 		return fmt.Errorf("offsite_keys/offsite_windows: %w", err)
 	}
+	// R-833 (v0.129.0): the cap each window was opened with — an operator grant may raise it once.
+	s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN max_remove INTEGER")
 
 	return nil
 }

hub/internal/web/server.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -71,6 +72,7 @@ type Server struct {
 	offsiteKeyAudit func(ctx context.Context) any
 	// offsiteWindowAdmin: operator one-shot grant / weekly switch (decision 68). nil → 503.
 	offsiteWindowGrant  func(customerID string) error
+	offsiteWindowGrantMax func(customerID string, maxRemove int) error
 	offsiteWindowSwitch func(on bool) error
 	// operator key-file clean-up (decision 72) and abandonment cancel (decision 74). nil → 503.
 	offsiteRemoveUnpinned func(ctx context.Context, customerID string) (int, error)
@@ -210,6 +212,9 @@ func (s *Server) SetOffsiteKeyAdmin(remove func(context.Context, string) (int, e
 	s.offsiteRemoveUnpinned, s.offsiteAbandonCancel = remove, cancel
 }
 
+// SetOffsiteWindowLargeGrant wires the operator's one-window raised cap (R-833).
+func (s *Server) SetOffsiteWindowLargeGrant(fn func(string, int) error) { s.offsiteWindowGrantMax = fn }
+
 // SetOffsiteWindowAdmin wires the operator's window controls (decision 68).
 func (s *Server) SetOffsiteWindowAdmin(grant func(string) error, sw func(bool) error) {
 	s.offsiteWindowGrant, s.offsiteWindowSwitch = grant, sw
@@ -655,6 +660,18 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		var err error
 		if path == "/offsite/windows-enabled" {
 			err = s.offsiteWindowSwitch(r.FormValue("on") == "1")
+		} else if mr := r.FormValue("max_remove"); mr != "" {
+			// R-833: one window with a raised removal cap (operator only — this server is behind the
+			// operator's login; the box API has no route to it).
+			n, perr := strconv.Atoi(mr)
+			if perr != nil || s.offsiteWindowGrantMax == nil {
+				http.Error(w, "max_remove must be a number", http.StatusBadRequest)
+				return
+			}
+			if err = s.offsiteWindowGrantMax(strings.TrimPrefix(path, "/offsite/window-grant/"), n); err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
 		} else {
 			err = s.offsiteWindowGrant(strings.TrimPrefix(path, "/offsite/window-grant/"))
 		}