Files
felhom.eu/hub/internal/offsitekeys/service.go
T
2026-10-04 08:56:56 +02:00

520 lines
22 KiB
Go

package offsitekeys
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"strconv"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// Event types this package raises — all OPERATOR-ONLY (registered in notify.operatorOnlyEvents).
const (
EventKeyInstalled = "offsite_key_installed" // info: the registrar pinned a box key
EventKeyUnlocked = "offsite_key_unlocked" // error: the daily check saw a line that can delete
EventAuditFailed = "offsite_key_audit_failed" // warning: the daily check could not read the file
EventMovedAside = "offsite_repo_moved_aside" // info: the hub set an orphaned repository aside
)
// Service binds the Registrar to the hub's store: the descriptor (where), the sealed password (how),
// the key record, and the operator events.
type Service struct {
Store *store.Store
Reg *Registrar
Logger *log.Logger
// Emit routes an event to the dispatcher (operator mail). nil → events are only saved.
Emit func(customerID, eventType, severity, message, detailsJSON, source string)
Now func() time.Time
// AbandonDelay is the hub-enforced wait before a set-aside copy is deleted (decision 74). 0 → 7 days.
AbandonDelay time.Duration
}
// ErrNotProvisioned — the customer has no provisioned off-site target (nothing to register against).
var ErrNotProvisioned = errors.New("offsitekeys: customer has no provisioned off-site target")
func (s *Service) logf(f string, a ...any) {
if s.Logger != nil {
s.Logger.Printf(f, a...)
}
}
func (s *Service) now() time.Time {
if s.Now != nil {
return s.Now()
}
return time.Now()
}
func (s *Service) event(customerID, typ, sev, msg string, details any) {
dj := ""
if details != nil {
if b, err := json.Marshal(details); err == nil {
dj = string(b)
}
}
if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil {
s.logf("[WARN] offsitekeys: save event %s for %s: %v", typ, customerID, err)
}
if s.Emit != nil {
s.Emit(customerID, typ, sev, msg, dj, "hub")
}
}
// TargetFor resolves the customer's sub-account and the hub's (decrypted) password for it.
func (s *Service) TargetFor(customerID string) (Target, string, error) {
cfg, err := s.Store.GetCustomerConfig(customerID)
if err != nil || cfg == nil {
return Target{}, "", fmt.Errorf("offsitekeys: customer %s: %v", customerID, err)
}
d, err := offsite.ReadDescriptor(cfg.ConfigJSON)
if err != nil {
return Target{}, "", err
}
if d == nil || !d.Enabled || d.Host == "" || d.User == "" || d.RepoPath == "" {
return Target{}, "", ErrNotProvisioned
}
pw, err := s.Store.OffsitePassword(customerID)
if err != nil {
return Target{}, "", fmt.Errorf("offsitekeys: no usable stored credential for %s: %w", customerID, err)
}
return Target{Host: d.Host, User: d.User, Port: d.Port, RepoPath: d.RepoPath, Fingerprint: d.HostFingerprint}, pw, nil
}
// RegisterKey installs the box's public key pinned append-only (the ONLY way a box gets off-site access
// from hub v0.127.0 on — no box ever receives the password).
func (s *Service) RegisterKey(ctx context.Context, customerID, pub string) (InstallResult, error) {
t, pw, err := s.TargetFor(customerID)
if err != nil {
return InstallResult{}, err
}
start := s.now()
res, err := s.Reg.Install(ctx, t, pw, pub)
if err != nil {
s.logf("[ERROR] offsitekeys: install key for %s (%s@%s): %v", customerID, t.User, t.Host, err)
return InstallResult{}, err
}
if err := s.Store.RecordOffsiteKeyInstalled(customerID, res.Fingerprint); err != nil {
s.logf("[WARN] offsitekeys: record key for %s: %v", customerID, err)
}
if err := s.Store.MarkOffsiteSecretDelivered(customerID); err != nil {
s.logf("[WARN] offsitekeys: mark delivered for %s: %v", customerID, err)
}
s.logf("[INFO] offsitekeys: installed box key %s for %s pinned append-only (%s@%s, dropped %d unpinned line(s)) in %s",
res.Fingerprint, customerID, t.User, t.Host, res.RemovedUnpinned, s.now().Sub(start).Round(time.Millisecond))
s.event(customerID, EventKeyInstalled, "info",
fmt.Sprintf("Off-site: the box's key %s was installed append-only on %s (%d unpinned line(s) removed).", res.Fingerprint, t.User, res.RemovedUnpinned),
map[string]any{"fingerprint": res.Fingerprint, "removed_unpinned": res.RemovedUnpinned})
return res, nil
}
// ConfirmKey is the rotation's last step: only the confirmed key's pinned line stays.
func (s *Service) ConfirmKey(ctx context.Context, customerID, fp string) (int, error) {
t, pw, err := s.TargetFor(customerID)
if err != nil {
return 0, err
}
removed, err := s.Reg.Confirm(ctx, t, pw, fp)
if err != nil {
return 0, err
}
if ok, err := s.Store.RecordOffsiteKeyConfirmed(customerID, fp); err != nil || !ok {
s.logf("[WARN] offsitekeys: confirm record for %s (fp %s): matched=%v err=%v", customerID, fp, ok, err)
}
s.logf("[INFO] offsitekeys: box confirmed key %s for %s; %d other line(s) removed", fp, customerID, removed)
return removed, nil
}
// MoveAside sets the repository aside (never deletes) on the box's request.
func (s *Service) MoveAside(ctx context.Context, customerID string) (string, error) {
t, pw, err := s.TargetFor(customerID)
if err != nil {
return "", err
}
name, err := s.Reg.MoveAside(ctx, t, pw, s.now().UTC().Format("20060102"))
if err != nil {
return "", err
}
s.logf("[WARN] offsitekeys: moved %s's repository aside: %s -> %s (nothing deleted)", customerID, t.RepoPath, name)
s.event(customerID, EventMovedAside, "info",
fmt.Sprintf("Off-site: the box asked to set its orphaned repository aside; %s was moved to %s. Nothing was deleted.", t.RepoPath, name),
map[string]any{"from": t.RepoPath, "to": name})
return name, nil
}
// AuditOutcome is one customer's daily-check result.
type AuditOutcome struct {
CustomerID string
Result AuditResult
Err error
}
// WindowOpenFunc reports whether a clean-up window is open for the customer (Part E). nil → never.
type WindowOpenFunc func(customerID string) bool
// AuditAll is the DAILY CHECK (decision 69): every provisioned customer's authorized_keys is read, and
// any line that can delete outside an open window raises `offsite_key_unlocked` (error, operator-only),
// naming the line by fingerprint only. An unreadable file raises `offsite_key_audit_failed`.
func (s *Service) AuditAll(ctx context.Context, windowOpen WindowOpenFunc) []AuditOutcome {
cfgs, err := s.Store.ListCustomerConfigs()
if err != nil {
s.logf("[ERROR] offsitekeys: audit: list configs: %v", err)
return nil
}
var out []AuditOutcome
for _, c := range cfgs {
d, derr := offsite.ReadDescriptor(c.ConfigJSON)
if derr != nil || d == nil || !d.Enabled || d.Host == "" {
continue
}
o := AuditOutcome{CustomerID: c.CustomerID}
t, pw, terr := s.TargetFor(c.CustomerID)
if terr != nil {
o.Err = terr
} else {
open := windowOpen != nil && windowOpen(c.CustomerID)
o.Result, o.Err = s.Reg.Audit(ctx, t, pw, open)
}
out = append(out, o)
switch {
case o.Err != nil:
s.logf("[WARN] offsitekeys: audit %s: %v", c.CustomerID, o.Err)
s.event(c.CustomerID, EventAuditFailed, "warning",
fmt.Sprintf("Off-site key check: could not read the key file of %s: %v", c.CustomerID, o.Err), nil)
case len(o.Result.Findings) > 0:
var parts []string
for _, f := range o.Result.Findings {
parts = append(parts, f.Kind+" "+f.Fingerprint)
}
s.logf("[ERROR] offsitekeys: audit %s: %d line(s) can delete off-site history: %s", c.CustomerID, len(o.Result.Findings), strings.Join(parts, "; "))
s.event(c.CustomerID, EventKeyUnlocked, "error",
fmt.Sprintf("Off-site key check: %d key line(s) on %s are NOT append-only and can delete this household's off-site history: %s",
len(o.Result.Findings), t.User, strings.Join(parts, "; ")),
map[string]any{"findings": o.Result.Findings, "lines": o.Result.Lines, "pinned": o.Result.Pinned})
default:
s.logf("[INFO] offsitekeys: audit %s: %d line(s), all pinned append-only", c.CustomerID, o.Result.Lines)
}
}
return out
}
// ── Decision 68: the clean-up window ──────────────────────────────────────────────────────────────
const (
EventWindowDrop = "offsite_window_drop" // error: more snapshots went than a window may remove
EventWindowFailed = "offsite_window_failed" // warning: a window errored or was left open
EventGuardRefused = "offsite_prune_guard_refused" // error: the box's fake-snapshot guard refused (R-822)
EventWindowLargeGrant = "offsite_window_large_grant" // warning: the operator raised one window's cap (R-833)
windowLength = 20 * time.Minute
windowCadence = 6*24*time.Hour + 12*time.Hour // "weekly", with slack for the night chain's drift
)
// WindowGrant is the hub's answer to the box.
type WindowGrant struct {
Granted bool `json:"granted"`
WindowID int64 `json:"window_id,omitempty"`
NewestAllowed time.Time `json:"newest_allowed,omitempty"`
MaxRemove int `json:"max_remove,omitempty"`
Reason string `json:"reason,omitempty"`
}
// WindowResult is the box's report when it is done.
type WindowResult struct {
WindowID int64 `json:"window_id"`
CountBefore int `json:"count_before"`
CountAfter int `json:"count_after"`
Removed int `json:"removed"`
Outcome string `json:"outcome"`
Reason string `json:"reason"`
}
// MaxRemove is the most snapshots one window may remove: HALF of what was there, at least 5. The ruled
// policy (7 daily + 4 weekly + 6 monthly per app) removes ~7 of ~17 per app per week (~41 %) — v0.127.0's
// 40 % would have refused every honest week once the box refuses above the cap (controller v0.290.0).
// Half is also the line R-431's detector draws for "an unexplained fall". Pinned by TestMaxRemove_*.
func MaxRemove(countBefore int) int {
n := countBefore / 2
if n < 5 {
n = 5
}
return n
}
// windowCap is the cap a window was OPENED with (an operator grant may have raised it, R-833); rows
// from before v0.129.0 carry none and fall back to the default.
func windowCap(w *store.OffsiteWindow) int {
if w.MaxRemove > 0 {
return w.MaxRemove
}
return MaxRemove(w.CountBefore)
}
// MaxRemoveGrantCeiling bounds an operator's raised cap: a typo of an extra zero must not turn one
// window into "remove anything". A real backlog above it is cleared over several granted windows.
const MaxRemoveGrantCeiling = 500
// GrantLargeWindow is the OPERATOR's one-shot grant with a raised cap for that one window (R-833).
// It is reachable only from the operator's hub login (internal/web), never from the box API. The grant
// is consumed by the next window; the window after it has the default cap again. Logged as an
// operator event.
func (s *Service) GrantLargeWindow(customerID string, maxRemove int) error {
if maxRemove < 1 || maxRemove > MaxRemoveGrantCeiling {
return fmt.Errorf("offsitekeys: max_remove %d is outside 1..%d", maxRemove, MaxRemoveGrantCeiling)
}
if c, err := s.Store.GetCustomerConfig(customerID); err != nil || c == nil {
return fmt.Errorf("offsitekeys: no customer %q", customerID)
}
if err := s.Store.GrantOffsiteWindowOnceMax(customerID, maxRemove); err != nil {
return err
}
s.logf("[WARN] offsitekeys: operator granted ONE clean-up window for %s with a raised cap of %d", customerID, maxRemove)
s.event(customerID, EventWindowLargeGrant, "warning",
fmt.Sprintf("Off-site clean-up: the operator granted one window with a raised removal cap of %d (the fake-snapshot guard still applies).", maxRemove),
map[string]any{"max_remove": maxRemove})
return nil
}
// OpenWindowFor decides and, if due, opens the window: a deleting line for the box's CONFIRMED key is
// prepended (first match wins), and a ledger row bounds it to 20 minutes.
func (s *Service) OpenWindowFor(ctx context.Context, customerID string, countBefore int) (WindowGrant, error) {
oneShot, raisedMax := s.Store.TakeOffsiteWindowGrant(customerID)
last := s.Store.LastOffsiteWindowOpened(customerID)
due := last.IsZero() || s.now().Sub(last) >= windowCadence
if !oneShot && !(s.Store.OffsiteWindowsEnabled() && due) {
reason := "weekly windows are off"
if s.Store.OffsiteWindowsEnabled() {
reason = "not due (last window " + last.UTC().Format(time.RFC3339) + ")"
}
return WindowGrant{Reason: reason}, nil
}
k, err := s.Store.GetOffsiteKey(customerID)
if err != nil || k == nil || k.ConfirmedAt.IsZero() {
return WindowGrant{Reason: "no confirmed append-only key on record"}, nil
}
t, pw, err := s.TargetFor(customerID)
if err != nil {
return WindowGrant{}, err
}
if err := s.Reg.OpenWindow(ctx, t, pw, k.Fingerprint); err != nil {
return WindowGrant{}, err
}
now := s.now()
// R-833: an operator grant may RAISE the cap for this one window (never lower it). Only the count
// cap moves; the box's fake-snapshot guard still runs in full against NewestAllowed.
maxRemove := MaxRemove(countBefore)
raised := raisedMax > maxRemove
if raised {
maxRemove = raisedMax
}
id, err := s.Store.OpenOffsiteWindowRow(customerID, now.Add(windowLength), countBefore, maxRemove)
if err != nil {
// The line is written; close it rather than leave a deleting line without a ledger row.
_ = s.Reg.CloseWindow(ctx, t, pw)
return WindowGrant{}, err
}
g := WindowGrant{Granted: true, WindowID: id, NewestAllowed: now.UTC(), MaxRemove: maxRemove}
s.logf("[WARN] offsitekeys: clean-up window %d OPENED for %s (key %s, %d snapshot(s), max %d removed%s, closes by %s, one-shot=%v)",
id, customerID, k.Fingerprint, countBefore, g.MaxRemove, map[bool]string{true: " — OPERATOR-RAISED cap (default " + strconv.Itoa(MaxRemove(countBefore)) + ")", false: ""}[raised],
now.Add(windowLength).UTC().Format(time.RFC3339), oneShot)
return g, nil
}
// CloseWindowFor closes the window on the box's report and checks the count.
func (s *Service) CloseWindowFor(ctx context.Context, customerID string, r WindowResult) error {
w, err := s.Store.GetOffsiteWindow(r.WindowID)
if err != nil || w == nil || w.CustomerID != customerID {
return fmt.Errorf("offsitekeys: window %d is not this customer's", r.WindowID)
}
t, pw, err := s.TargetFor(customerID)
if err != nil {
return err
}
if err := s.Reg.CloseWindow(ctx, t, pw); err != nil {
return err // the sweep retries at closes_by
}
if _, err := s.Store.CloseOffsiteWindowRow(w.ID, r.CountAfter, r.Outcome, "box"); err != nil {
s.logf("[WARN] offsitekeys: ledger close %d: %v", w.ID, err)
}
drop := w.CountBefore - r.CountAfter
allowed := windowCap(w)
s.logf("[INFO] offsitekeys: clean-up window %d CLOSED for %s: outcome=%s, %d -> %d (drop %d, allowed %d)",
w.ID, customerID, r.Outcome, w.CountBefore, r.CountAfter, drop, allowed)
details := map[string]any{"window_id": w.ID, "count_before": w.CountBefore, "count_after": r.CountAfter, "outcome": r.Outcome, "reason": r.Reason}
switch {
case drop > allowed:
s.event(customerID, EventWindowDrop, "error",
fmt.Sprintf("Off-site clean-up window %d: the snapshot count fell %d -> %d, more than a window may remove (%d).", w.ID, w.CountBefore, r.CountAfter, allowed), details)
case r.Outcome == "guard-refused":
s.event(customerID, EventGuardRefused, "error",
fmt.Sprintf("Off-site clean-up window %d: the box's fake-snapshot guard refused to prune — nothing was deleted: %s", w.ID, r.Reason), details)
case r.Outcome == "error":
s.event(customerID, EventWindowFailed, "warning",
fmt.Sprintf("Off-site clean-up window %d: the prune failed on the box: %s", w.ID, r.Reason), details)
}
return nil
}
// SweepExpiredWindows closes every window left open past its 20 minutes (a box that crashed or lied).
func (s *Service) SweepExpiredWindows(ctx context.Context) {
ws, err := s.Store.ExpiredOffsiteWindows()
if err != nil {
s.logf("[WARN] offsitekeys: window sweep: %v", err)
return
}
for _, w := range ws {
t, pw, err := s.TargetFor(w.CustomerID)
if err == nil {
err = s.Reg.CloseWindow(ctx, t, pw)
}
if err != nil {
s.logf("[ERROR] offsitekeys: window %d for %s is past its time and could NOT be closed: %v (retrying next sweep)", w.ID, w.CustomerID, err)
continue
}
_, _ = s.Store.CloseOffsiteWindowRow(w.ID, -1, "", "timeout")
s.logf("[WARN] offsitekeys: clean-up window %d for %s was left open — closed by the hub", w.ID, w.CustomerID)
s.event(w.CustomerID, EventWindowFailed, "warning",
fmt.Sprintf("Off-site clean-up window %d was not closed by the box within %s; the hub closed it (the deleting key line is removed).", w.ID, windowLength), nil)
}
}
// ── Decision 74 (R-823): the household's "delete my set-aside history" — done by the HUB, after a delay ──
//
// The box's key cannot delete (decision 69), so a due abandonment is a REQUEST to the hub. The hub waits
// AbandonDelay (default 7 days) from the request — the household (via the box's recovery, which cancels)
// and the operator can cancel in that time — then deletes ONLY `<repo>.orphaned-<…>`, never the live
// repository. A broken-into box can therefore make a set-aside copy disappear only after a week of
// operator mails. Every request, cancel and deletion is an operator event.
const (
EventAbandonRequested = "offsite_abandon_requested"
EventAbandonCancelled = "offsite_abandon_cancelled"
EventAbandonDeleted = "offsite_abandon_deleted"
EventAbandonFailed = "offsite_abandon_failed"
DefaultAbandonDelay = 7 * 24 * time.Hour
)
func (s *Service) abandonDelay() time.Duration {
if s.AbandonDelay > 0 {
return s.AbandonDelay
}
return DefaultAbandonDelay
}
// AbandonStatus is what the box (and the operator) sees.
type AbandonStatus struct {
State string `json:"state"` // none | pending | cancelled | deleted
Path string `json:"path,omitempty"`
DueAt time.Time `json:"due_at,omitempty"`
}
func statusOf(a *store.OffsiteAbandon) AbandonStatus {
if a == nil {
return AbandonStatus{State: "none"}
}
return AbandonStatus{State: a.State(), Path: a.Path, DueAt: a.DueAt.UTC()}
}
// RequestAbandon records (idempotently) the household's request to delete path, due after the delay.
func (s *Service) RequestAbandon(ctx context.Context, customerID, path string) (AbandonStatus, error) {
t, pw, err := s.TargetFor(customerID)
if err != nil {
return AbandonStatus{}, err
}
if !IsSetAsidePath(t.RepoPath, path) {
return AbandonStatus{}, fmt.Errorf("offsitekeys: %q is not a set-aside copy of %s", path, t.RepoPath)
}
if cur, err := s.Store.LatestOffsiteAbandon(customerID, path); err == nil && cur != nil && cur.State() == "pending" {
return statusOf(cur), nil // already requested — the clock is NOT restarted
}
// The copy must exist now (a request for nothing is refused, so a typo cannot sit armed for a week).
sh, err := s.Reg.open(ctx, t, pw)
if err != nil {
return AbandonStatus{}, err
}
_, lerr := sh.Run(ctx, "ls -d "+path, nil)
sh.Close()
if lerr != nil {
return AbandonStatus{}, fmt.Errorf("offsitekeys: set-aside copy %s not found", path)
}
due := s.now().Add(s.abandonDelay())
if _, err := s.Store.CreateOffsiteAbandon(customerID, path, due); err != nil {
return AbandonStatus{}, err
}
s.logf("[WARN] offsitekeys: %s asked to DELETE its set-aside off-site copy %s — the hub deletes it at %s unless cancelled (delay %s)",
customerID, path, due.UTC().Format(time.RFC3339), s.abandonDelay())
s.event(customerID, EventAbandonRequested, "warning",
fmt.Sprintf("Off-site: the box asked to delete the set-aside copy %s (the household's choice). The hub deletes it on %s unless the household or the operator cancels.", path, due.UTC().Format("2006-01-02 15:04 UTC")),
map[string]any{"path": path, "due_at": due.UTC()})
a, _ := s.Store.LatestOffsiteAbandon(customerID, path)
return statusOf(a), nil
}
// CancelAbandon cancels every pending request of the customer (by = "box" | "operator").
func (s *Service) CancelAbandon(customerID, by string) (int, error) {
n, err := s.Store.CancelOffsiteAbandon(customerID, by)
if err != nil || n == 0 {
return n, err
}
s.logf("[INFO] offsitekeys: %s's set-aside deletion CANCELLED by %s (%d request(s)) — nothing deleted", customerID, by, n)
s.event(customerID, EventAbandonCancelled, "info",
fmt.Sprintf("Off-site: the pending deletion of the set-aside copy was cancelled by the %s. Nothing was deleted.", by), nil)
return n, nil
}
// AbandonStatusFor returns the latest request's state.
func (s *Service) AbandonStatusFor(customerID string) (AbandonStatus, error) {
a, err := s.Store.LatestOffsiteAbandon(customerID, "")
if err != nil {
return AbandonStatus{}, err
}
return statusOf(a), nil
}
// SweepAbandons deletes every request that is due, not cancelled and not yet deleted.
func (s *Service) SweepAbandons(ctx context.Context) {
due, err := s.Store.DueOffsiteAbandons()
if err != nil {
s.logf("[WARN] offsitekeys: abandon sweep: %v", err)
return
}
for _, a := range due {
t, pw, terr := s.TargetFor(a.CustomerID)
if terr == nil {
terr = s.Reg.DeleteSetAside(ctx, t, pw, a.Path)
}
if terr != nil {
_ = s.Store.MarkOffsiteAbandonError(a.ID, terr.Error())
s.logf("[ERROR] offsitekeys: deleting %s's set-aside copy %s failed (retrying): %v", a.CustomerID, a.Path, terr)
if a.LastError == "" {
s.event(a.CustomerID, EventAbandonFailed, "warning",
fmt.Sprintf("Off-site: deleting the set-aside copy %s failed and is retried: %v", a.Path, terr), nil)
}
continue
}
_ = s.Store.MarkOffsiteAbandonDeleted(a.ID)
s.logf("[WARN] offsitekeys: DELETED %s's set-aside off-site copy %s (requested %s, due %s)", a.CustomerID, a.Path,
a.RequestedAt.UTC().Format(time.RFC3339), a.DueAt.UTC().Format(time.RFC3339))
s.event(a.CustomerID, EventAbandonDeleted, "info",
fmt.Sprintf("Off-site: the set-aside copy %s was deleted, as the household chose (requested %s).", a.Path, a.RequestedAt.UTC().Format("2006-01-02")), nil)
}
}
// RemoveUnpinnedKeys is the operator's clean-up of a sub-account's key file (decision 72).
func (s *Service) RemoveUnpinnedKeys(ctx context.Context, customerID string) (int, error) {
t, pw, err := s.TargetFor(customerID)
if err != nil {
return 0, err
}
n, err := s.Reg.RemoveUnpinned(ctx, t, pw, s.Store.OffsiteWindowOpen(customerID))
if err != nil {
return 0, err
}
s.logf("[INFO] offsitekeys: removed %d unpinned key line(s) from %s's sub-account (%s) on the operator's request", n, customerID, t.User)
return n, nil
}