Files
felhom.eu/hub/internal/store/offsite_keys.go
T
admin d34dfc84c0 hub: one lost off-site snapshot outside a clean-up window is an error (R-435, D7)
On a pinned tier (the hub holds a CONFIRMED append-only key) the only
legitimate fall of the box's snapshot count is a clean-up window the hub
opened. The checker now compares prev - cur with what the windows closed
since the previous trustworthy report removed (store.RemovedByWindowsBetween,
2 h slack for the in-run count lag); any unexplained fall, even one snapshot,
raises offsite_snapshots_dropped (error) saying 'outside any clean-up window
the hub opened'. A window that cannot say what it removed (timeout, still
open) explains anything: no alarm, one INFO line. Non-pinned tiers keep the
half-rule. Untrustworthy reports: unchanged (no alarm, baseline kept).

Red tests: r435_pinned_drop_test.go (3 fail with the pinned rule disabled;
WindowExplainsFall fails when windows are ignored), r435_windows_between_test.go.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-08 15:17:05 +02:00

367 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package store
import (
"database/sql"
"fmt"
"strconv"
"strings"
"time"
)
// OffsiteKey is the registrar's record of the box key the hub installed pinned (decision 69).
type OffsiteKey struct {
CustomerID string
Fingerprint string
InstalledAt time.Time
ConfirmedAt time.Time // zero = the box has not confirmed it yet
}
// RecordOffsiteKeyInstalled records (last-write-wins) the key the hub just installed; confirmation resets.
func (s *Store) RecordOffsiteKeyInstalled(customerID, fp string) error {
_, err := s.db.Exec(`
INSERT INTO offsite_keys (customer_id, fingerprint, installed_at, confirmed_at) VALUES (?, ?, datetime('now'), NULL)
ON CONFLICT(customer_id) DO UPDATE SET fingerprint = excluded.fingerprint, installed_at = datetime('now'), confirmed_at = NULL`,
customerID, fp)
return err
}
// RecordOffsiteKeyConfirmed marks the installed key confirmed by the box; false when fp is not the key on record.
func (s *Store) RecordOffsiteKeyConfirmed(customerID, fp string) (bool, error) {
res, err := s.db.Exec(`UPDATE offsite_keys SET confirmed_at = datetime('now') WHERE customer_id = ? AND fingerprint = ?`, customerID, fp)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// GetOffsiteKey returns the record, or (nil, nil) when none exists.
func (s *Store) GetOffsiteKey(customerID string) (*OffsiteKey, error) {
var k OffsiteKey
var inst string
var conf sql.NullString
err := s.db.QueryRow(`SELECT customer_id, fingerprint, installed_at, confirmed_at FROM offsite_keys WHERE customer_id = ?`, customerID).
Scan(&k.CustomerID, &k.Fingerprint, &inst, &conf)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
k.InstalledAt = parseSQLiteTime(inst)
if conf.Valid {
k.ConfirmedAt = parseSQLiteTime(conf.String)
}
return &k, nil
}
// OffsiteWindowOpen reports whether a clean-up window is open for the customer right now (decision 68):
// a window row not closed and not past its closes_by. Errors read as "closed" — the key check then
// alarms on a window line, which is the safe side.
func (s *Store) OffsiteWindowOpen(customerID string) bool {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM offsite_windows WHERE customer_id = ? AND closed_at IS NULL AND closes_by > datetime('now')`, customerID).Scan(&n)
return err == nil && n > 0
}
// OffsiteWindow is one clean-up window (decision 68).
type OffsiteWindow struct {
ID int64
CustomerID string
OpenedAt time.Time
ClosesBy time.Time
ClosedAt time.Time
CountBefore int
CountAfter int
BoxResult string
CloseReason string
// MaxRemove is the cap this window was opened with (R-833: an operator grant may raise it for one
// window). 0 on rows written before v0.129.0 — readers fall back to the default cap then.
MaxRemove int
}
// OpenOffsiteWindowRow records a window the hub just opened, with the cap it was opened under.
func (s *Store) OpenOffsiteWindowRow(customerID string, closesBy time.Time, countBefore, maxRemove int) (int64, error) {
res, err := s.db.Exec(`INSERT INTO offsite_windows (customer_id, opened_at, closes_by, count_before, max_remove) VALUES (?, datetime('now'), ?, ?, ?)`,
customerID, closesBy.UTC().Format("2006-01-02 15:04:05"), countBefore, maxRemove)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
// CloseOffsiteWindowRow closes a window (idempotent: an already-closed row is not touched).
func (s *Store) CloseOffsiteWindowRow(id int64, countAfter int, boxResult, reason string) (bool, error) {
res, err := s.db.Exec(`UPDATE offsite_windows SET closed_at = datetime('now'), count_after = ?, box_result = ?, close_reason = ? WHERE id = ? AND closed_at IS NULL`,
countAfter, boxResult, reason, id)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// GetOffsiteWindow returns one window row, or (nil, nil).
func (s *Store) GetOffsiteWindow(id int64) (*OffsiteWindow, error) {
var w OffsiteWindow
var opened, closesBy string
var closed, boxRes, reason sql.NullString
var before, after, maxRm sql.NullInt64
err := s.db.QueryRow(`SELECT id, customer_id, opened_at, closes_by, closed_at, count_before, count_after, box_result, close_reason, max_remove FROM offsite_windows WHERE id = ?`, id).
Scan(&w.ID, &w.CustomerID, &opened, &closesBy, &closed, &before, &after, &boxRes, &reason, &maxRm)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
w.OpenedAt, w.ClosesBy = parseSQLiteTime(opened), parseSQLiteTime(closesBy)
if closed.Valid {
w.ClosedAt = parseSQLiteTime(closed.String)
}
w.CountBefore, w.CountAfter = int(before.Int64), int(after.Int64)
w.BoxResult, w.CloseReason = boxRes.String, reason.String
w.MaxRemove = int(maxRm.Int64)
return &w, nil
}
// LastOffsiteWindowOpened returns when the customer's most recent window was opened (zero = never).
func (s *Store) LastOffsiteWindowOpened(customerID string) time.Time {
var v sql.NullString
if err := s.db.QueryRow(`SELECT MAX(opened_at) FROM offsite_windows WHERE customer_id = ?`, customerID).Scan(&v); err != nil || !v.Valid {
return time.Time{}
}
return parseSQLiteTime(v.String)
}
// ExpiredOffsiteWindows lists windows still open past their closes_by.
func (s *Store) ExpiredOffsiteWindows() ([]OffsiteWindow, error) {
rows, err := s.db.Query(`SELECT id, customer_id FROM offsite_windows WHERE closed_at IS NULL AND closes_by <= datetime('now')`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []OffsiteWindow
for rows.Next() {
var w OffsiteWindow
if err := rows.Scan(&w.ID, &w.CustomerID); err != nil {
return nil, err
}
out = append(out, w)
}
return out, rows.Err()
}
const offsiteWindowsEnabledKey = "offsite_prune_windows_enabled"
// OffsiteWindowsEnabled — the operator switch for WEEKLY windows (decision 68). Off by default: the
// interim is "nothing prunes" until the operator turns the weekly window on.
func (s *Store) OffsiteWindowsEnabled() bool { return s.getSetting(offsiteWindowsEnabledKey) == "on" }
// SetOffsiteWindowsEnabled flips the weekly switch.
func (s *Store) SetOffsiteWindowsEnabled(on bool) error {
v := ""
if on {
v = "on"
}
return s.setSetting(offsiteWindowsEnabledKey, v)
}
// GrantOffsiteWindowOnce lets the customer's NEXT window request through regardless of the weekly
// cadence (operator one-shot).
func (s *Store) GrantOffsiteWindowOnce(customerID string) error {
return s.setSetting("offsite_window_grant:"+customerID, "1")
}
// GrantOffsiteWindowOnceMax is the operator's one-shot grant that ALSO raises the removal cap for that
// one window (R-833: after a long gap the honest backlog exceeds half the snapshots and the default cap
// refuses every window). The raised cap is consumed with the grant; the next window has the default.
func (s *Store) GrantOffsiteWindowOnceMax(customerID string, maxRemove int) error {
if maxRemove <= 0 {
return fmt.Errorf("max_remove must be positive, got %d", maxRemove)
}
return s.setSetting("offsite_window_grant:"+customerID, "max:"+strconv.Itoa(maxRemove))
}
// TakeOffsiteWindowGrant consumes a one-shot grant: granted is true when one was present, and
// maxRemove is the operator's raised cap for that window (0 = none, use the default).
func (s *Store) TakeOffsiteWindowGrant(customerID string) (granted bool, maxRemove int) {
k := "offsite_window_grant:" + customerID
v := s.getSetting(k)
switch {
case v == "1":
case strings.HasPrefix(v, "max:"):
n, err := strconv.Atoi(strings.TrimPrefix(v, "max:"))
if err != nil || n <= 0 {
n = 0 // a malformed value still grants the window, at the default cap
}
maxRemove = n
default:
return false, 0
}
_ = s.setSetting(k, "")
return true, maxRemove
}
// ForceOffsiteWindowDueForTest back-dates a window's closes_by. TEST-ONLY.
func (s *Store) ForceOffsiteWindowDueForTest(id int64) error {
_, err := s.db.Exec(`UPDATE offsite_windows SET closes_by = datetime('now', '-1 minute') WHERE id = ?`, id)
return err
}
// OffsiteAbandon is one household request to delete a set-aside off-site copy (decision 74, R-823). The
// hub deletes it only after DueAt, and only if nobody cancelled it.
type OffsiteAbandon struct {
ID int64
CustomerID string
Path string
RequestedAt time.Time
DueAt time.Time
CancelledAt time.Time
CancelledBy string
DeletedAt time.Time
LastError string
}
func (a *OffsiteAbandon) State() string {
switch {
case a == nil:
return "none"
case !a.DeletedAt.IsZero():
return "deleted"
case !a.CancelledAt.IsZero():
return "cancelled"
}
return "pending"
}
const abandonCols = `id, customer_id, path, requested_at, due_at, cancelled_at, cancelled_by, deleted_at, last_error`
func scanAbandon(sc interface{ Scan(...any) error }) (*OffsiteAbandon, error) {
var a OffsiteAbandon
var req, due string
var canc, by, del, lerr sql.NullString
if err := sc.Scan(&a.ID, &a.CustomerID, &a.Path, &req, &due, &canc, &by, &del, &lerr); err != nil {
return nil, err
}
a.RequestedAt, a.DueAt = parseSQLiteTime(req), parseSQLiteTime(due)
if canc.Valid {
a.CancelledAt = parseSQLiteTime(canc.String)
}
if del.Valid {
a.DeletedAt = parseSQLiteTime(del.String)
}
a.CancelledBy, a.LastError = by.String, lerr.String
return &a, nil
}
// LatestOffsiteAbandon returns the customer's most recent request for path ("" = any), or (nil, nil).
func (s *Store) LatestOffsiteAbandon(customerID, path string) (*OffsiteAbandon, error) {
q := `SELECT ` + abandonCols + ` FROM offsite_abandon_requests WHERE customer_id = ?`
args := []any{customerID}
if path != "" {
q += ` AND path = ?`
args = append(args, path)
}
a, err := scanAbandon(s.db.QueryRow(q+` ORDER BY id DESC LIMIT 1`, args...))
if err == sql.ErrNoRows {
return nil, nil
}
return a, err
}
// CreateOffsiteAbandon records a request due at dueAt.
func (s *Store) CreateOffsiteAbandon(customerID, path string, dueAt time.Time) (int64, error) {
res, err := s.db.Exec(`INSERT INTO offsite_abandon_requests (customer_id, path, requested_at, due_at) VALUES (?, ?, datetime('now'), ?)`,
customerID, path, dueAt.UTC().Format("2006-01-02 15:04:05"))
if err != nil {
return 0, err
}
return res.LastInsertId()
}
// CancelOffsiteAbandon cancels every PENDING request of the customer; returns how many.
func (s *Store) CancelOffsiteAbandon(customerID, by string) (int, error) {
res, err := s.db.Exec(`UPDATE offsite_abandon_requests SET cancelled_at = datetime('now'), cancelled_by = ? WHERE customer_id = ? AND cancelled_at IS NULL AND deleted_at IS NULL`, by, customerID)
if err != nil {
return 0, err
}
n, _ := res.RowsAffected()
return int(n), nil
}
// DueOffsiteAbandons lists pending requests whose due time has passed.
func (s *Store) DueOffsiteAbandons() ([]*OffsiteAbandon, error) {
rows, err := s.db.Query(`SELECT ` + abandonCols + ` FROM offsite_abandon_requests WHERE cancelled_at IS NULL AND deleted_at IS NULL AND due_at <= datetime('now')`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*OffsiteAbandon
for rows.Next() {
a, err := scanAbandon(rows)
if err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
// MarkOffsiteAbandonDeleted / MarkOffsiteAbandonError record the sweep's outcome.
func (s *Store) MarkOffsiteAbandonDeleted(id int64) error {
_, err := s.db.Exec(`UPDATE offsite_abandon_requests SET deleted_at = datetime('now'), last_error = NULL WHERE id = ?`, id)
return err
}
func (s *Store) MarkOffsiteAbandonError(id int64, msg string) error {
_, err := s.db.Exec(`UPDATE offsite_abandon_requests SET last_error = ? WHERE id = ?`, msg, id)
return err
}
// ForceOffsiteAbandonDueForTest back-dates a request. TEST-ONLY.
func (s *Store) ForceOffsiteAbandonDueForTest(id int64) error {
_, err := s.db.Exec(`UPDATE offsite_abandon_requests SET due_at = datetime('now', '-1 minute') WHERE id = ?`, id)
return err
}
// RemovedByWindowsBetween — R-435 (D7, `09` §3 decision 191). How many snapshots the clean-up windows
// the hub opened for this customer EXPLAIN between two reports: the sum of count_before − count_after
// over windows closed in (from, to]. On a pinned tier these windows are the only legitimate way the
// count can fall, so anything beyond the sum is unexplained.
//
// unknown = true when a window in the interval cannot say what it removed: closed by timeout (no box
// result, count_after −1 or NULL), or still open (opened at or before `to`, not closed). The caller
// then treats the fall as explained — the safe side for noise, recorded as a limit in `08`. A query
// error is unknown too.
func (s *Store) RemovedByWindowsBetween(customerID string, from, to time.Time) (removed int, unknown bool) {
const f = "2006-01-02 15:04:05"
rows, err := s.db.Query(`
SELECT count_before, count_after, closed_at IS NULL FROM offsite_windows
WHERE customer_id = ?
AND ((closed_at IS NULL AND opened_at <= ?) OR (closed_at > ? AND closed_at <= ?))`,
customerID, to.UTC().Format(f), from.UTC().Format(f), to.UTC().Format(f))
if err != nil {
return 0, true
}
defer rows.Close()
for rows.Next() {
var before, after sql.NullInt64
var open bool
if err := rows.Scan(&before, &after, &open); err != nil {
return 0, true
}
if open || !before.Valid || !after.Valid || after.Int64 < 0 {
unknown = true
continue
}
if d := int(before.Int64 - after.Int64); d > 0 {
removed += d
}
}
if rows.Err() != nil {
return 0, true
}
return removed, unknown
}