package offsitekeys import ( "context" "encoding/json" "errors" "fmt" "log" "strings" "strconv" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // Event types this package raises — all OPERATOR-ONLY (registered in notify.operatorOnlyEvents). const ( EventKeyInstalled = "offsite_key_installed" // info: the registrar pinned a box key EventKeyUnlocked = "offsite_key_unlocked" // error: the daily check saw a line that can delete EventAuditFailed = "offsite_key_audit_failed" // warning: the daily check could not read the file EventMovedAside = "offsite_repo_moved_aside" // info: the hub set an orphaned repository aside ) // Service binds the Registrar to the hub's store: the descriptor (where), the sealed password (how), // the key record, and the operator events. type Service struct { Store *store.Store Reg *Registrar Logger *log.Logger // Emit routes an event to the dispatcher (operator mail). nil → events are only saved. Emit func(customerID, eventType, severity, message, detailsJSON, source string) Now func() time.Time // AbandonDelay is the hub-enforced wait before a set-aside copy is deleted (decision 74). 0 → 7 days. AbandonDelay time.Duration } // ErrNotProvisioned — the customer has no provisioned off-site target (nothing to register against). var ErrNotProvisioned = errors.New("offsitekeys: customer has no provisioned off-site target") func (s *Service) logf(f string, a ...any) { if s.Logger != nil { s.Logger.Printf(f, a...) } } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) event(customerID, typ, sev, msg string, details any) { dj := "" if details != nil { if b, err := json.Marshal(details); err == nil { dj = string(b) } } if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil { s.logf("[WARN] offsitekeys: save event %s for %s: %v", typ, customerID, err) } if s.Emit != nil { s.Emit(customerID, typ, sev, msg, dj, "hub") } } // TargetFor resolves the customer's sub-account and the hub's (decrypted) password for it. func (s *Service) TargetFor(customerID string) (Target, string, error) { cfg, err := s.Store.GetCustomerConfig(customerID) if err != nil || cfg == nil { return Target{}, "", fmt.Errorf("offsitekeys: customer %s: %v", customerID, err) } d, err := offsite.ReadDescriptor(cfg.ConfigJSON) if err != nil { return Target{}, "", err } if d == nil || !d.Enabled || d.Host == "" || d.User == "" || d.RepoPath == "" { return Target{}, "", ErrNotProvisioned } pw, err := s.Store.OffsitePassword(customerID) if err != nil { return Target{}, "", fmt.Errorf("offsitekeys: no usable stored credential for %s: %w", customerID, err) } return Target{Host: d.Host, User: d.User, Port: d.Port, RepoPath: d.RepoPath, Fingerprint: d.HostFingerprint}, pw, nil } // RegisterKey installs the box's public key pinned append-only (the ONLY way a box gets off-site access // from hub v0.127.0 on — no box ever receives the password). func (s *Service) RegisterKey(ctx context.Context, customerID, pub string) (InstallResult, error) { t, pw, err := s.TargetFor(customerID) if err != nil { return InstallResult{}, err } start := s.now() res, err := s.Reg.Install(ctx, t, pw, pub) if err != nil { s.logf("[ERROR] offsitekeys: install key for %s (%s@%s): %v", customerID, t.User, t.Host, err) return InstallResult{}, err } if err := s.Store.RecordOffsiteKeyInstalled(customerID, res.Fingerprint); err != nil { s.logf("[WARN] offsitekeys: record key for %s: %v", customerID, err) } if err := s.Store.MarkOffsiteSecretDelivered(customerID); err != nil { s.logf("[WARN] offsitekeys: mark delivered for %s: %v", customerID, err) } s.logf("[INFO] offsitekeys: installed box key %s for %s pinned append-only (%s@%s, dropped %d unpinned line(s)) in %s", res.Fingerprint, customerID, t.User, t.Host, res.RemovedUnpinned, s.now().Sub(start).Round(time.Millisecond)) s.event(customerID, EventKeyInstalled, "info", fmt.Sprintf("Off-site: the box's key %s was installed append-only on %s (%d unpinned line(s) removed).", res.Fingerprint, t.User, res.RemovedUnpinned), map[string]any{"fingerprint": res.Fingerprint, "removed_unpinned": res.RemovedUnpinned}) return res, nil } // ConfirmKey is the rotation's last step: only the confirmed key's pinned line stays. func (s *Service) ConfirmKey(ctx context.Context, customerID, fp string) (int, error) { t, pw, err := s.TargetFor(customerID) if err != nil { return 0, err } removed, err := s.Reg.Confirm(ctx, t, pw, fp) if err != nil { return 0, err } if ok, err := s.Store.RecordOffsiteKeyConfirmed(customerID, fp); err != nil || !ok { s.logf("[WARN] offsitekeys: confirm record for %s (fp %s): matched=%v err=%v", customerID, fp, ok, err) } s.logf("[INFO] offsitekeys: box confirmed key %s for %s; %d other line(s) removed", fp, customerID, removed) return removed, nil } // MoveAside sets the repository aside (never deletes) on the box's request. func (s *Service) MoveAside(ctx context.Context, customerID string) (string, error) { t, pw, err := s.TargetFor(customerID) if err != nil { return "", err } name, err := s.Reg.MoveAside(ctx, t, pw, s.now().UTC().Format("20060102")) if err != nil { return "", err } s.logf("[WARN] offsitekeys: moved %s's repository aside: %s -> %s (nothing deleted)", customerID, t.RepoPath, name) s.event(customerID, EventMovedAside, "info", fmt.Sprintf("Off-site: the box asked to set its orphaned repository aside; %s was moved to %s. Nothing was deleted.", t.RepoPath, name), map[string]any{"from": t.RepoPath, "to": name}) return name, nil } // AuditOutcome is one customer's daily-check result. type AuditOutcome struct { CustomerID string Result AuditResult Err error } // WindowOpenFunc reports whether a clean-up window is open for the customer (Part E). nil → never. type WindowOpenFunc func(customerID string) bool // AuditAll is the DAILY CHECK (decision 69): every provisioned customer's authorized_keys is read, and // any line that can delete outside an open window raises `offsite_key_unlocked` (error, operator-only), // naming the line by fingerprint only. An unreadable file raises `offsite_key_audit_failed`. func (s *Service) AuditAll(ctx context.Context, windowOpen WindowOpenFunc) []AuditOutcome { cfgs, err := s.Store.ListCustomerConfigs() if err != nil { s.logf("[ERROR] offsitekeys: audit: list configs: %v", err) return nil } var out []AuditOutcome for _, c := range cfgs { d, derr := offsite.ReadDescriptor(c.ConfigJSON) if derr != nil || d == nil || !d.Enabled || d.Host == "" { continue } o := AuditOutcome{CustomerID: c.CustomerID} t, pw, terr := s.TargetFor(c.CustomerID) if terr != nil { o.Err = terr } else { open := windowOpen != nil && windowOpen(c.CustomerID) o.Result, o.Err = s.Reg.Audit(ctx, t, pw, open) } out = append(out, o) switch { case o.Err != nil: s.logf("[WARN] offsitekeys: audit %s: %v", c.CustomerID, o.Err) s.event(c.CustomerID, EventAuditFailed, "warning", fmt.Sprintf("Off-site key check: could not read the key file of %s: %v", c.CustomerID, o.Err), nil) case len(o.Result.Findings) > 0: var parts []string for _, f := range o.Result.Findings { parts = append(parts, f.Kind+" "+f.Fingerprint) } s.logf("[ERROR] offsitekeys: audit %s: %d line(s) can delete off-site history: %s", c.CustomerID, len(o.Result.Findings), strings.Join(parts, "; ")) s.event(c.CustomerID, EventKeyUnlocked, "error", fmt.Sprintf("Off-site key check: %d key line(s) on %s are NOT append-only and can delete this household's off-site history: %s", len(o.Result.Findings), t.User, strings.Join(parts, "; ")), map[string]any{"findings": o.Result.Findings, "lines": o.Result.Lines, "pinned": o.Result.Pinned}) default: s.logf("[INFO] offsitekeys: audit %s: %d line(s), all pinned append-only", c.CustomerID, o.Result.Lines) } } return out } // ── Decision 68: the clean-up window ────────────────────────────────────────────────────────────── const ( EventWindowDrop = "offsite_window_drop" // error: more snapshots went than a window may remove EventWindowFailed = "offsite_window_failed" // warning: a window errored or was left open EventGuardRefused = "offsite_prune_guard_refused" // error: the box's fake-snapshot guard refused (R-822) EventWindowLargeGrant = "offsite_window_large_grant" // warning: the operator raised one window's cap (R-833) windowLength = 20 * time.Minute windowCadence = 6*24*time.Hour + 12*time.Hour // "weekly", with slack for the night chain's drift ) // WindowGrant is the hub's answer to the box. type WindowGrant struct { Granted bool `json:"granted"` WindowID int64 `json:"window_id,omitempty"` NewestAllowed time.Time `json:"newest_allowed,omitempty"` MaxRemove int `json:"max_remove,omitempty"` Reason string `json:"reason,omitempty"` } // WindowResult is the box's report when it is done. type WindowResult struct { WindowID int64 `json:"window_id"` CountBefore int `json:"count_before"` CountAfter int `json:"count_after"` Removed int `json:"removed"` Outcome string `json:"outcome"` Reason string `json:"reason"` } // MaxRemove is the most snapshots one window may remove: HALF of what was there, at least 5. The ruled // policy (7 daily + 4 weekly + 6 monthly per app) removes ~7 of ~17 per app per week (~41 %) — v0.127.0's // 40 % would have refused every honest week once the box refuses above the cap (controller v0.290.0). // Half is also the line R-431's detector draws for "an unexplained fall". Pinned by TestMaxRemove_*. func MaxRemove(countBefore int) int { n := countBefore / 2 if n < 5 { n = 5 } return n } // windowCap is the cap a window was OPENED with (an operator grant may have raised it, R-833); rows // from before v0.129.0 carry none and fall back to the default. func windowCap(w *store.OffsiteWindow) int { if w.MaxRemove > 0 { return w.MaxRemove } return MaxRemove(w.CountBefore) } // MaxRemoveGrantCeiling bounds an operator's raised cap: a typo of an extra zero must not turn one // window into "remove anything". A real backlog above it is cleared over several granted windows. const MaxRemoveGrantCeiling = 500 // GrantLargeWindow is the OPERATOR's one-shot grant with a raised cap for that one window (R-833). // It is reachable only from the operator's hub login (internal/web), never from the box API. The grant // is consumed by the next window; the window after it has the default cap again. Logged as an // operator event. func (s *Service) GrantLargeWindow(customerID string, maxRemove int) error { if maxRemove < 1 || maxRemove > MaxRemoveGrantCeiling { return fmt.Errorf("offsitekeys: max_remove %d is outside 1..%d", maxRemove, MaxRemoveGrantCeiling) } if c, err := s.Store.GetCustomerConfig(customerID); err != nil || c == nil { return fmt.Errorf("offsitekeys: no customer %q", customerID) } if err := s.Store.GrantOffsiteWindowOnceMax(customerID, maxRemove); err != nil { return err } s.logf("[WARN] offsitekeys: operator granted ONE clean-up window for %s with a raised cap of %d", customerID, maxRemove) s.event(customerID, EventWindowLargeGrant, "warning", fmt.Sprintf("Off-site clean-up: the operator granted one window with a raised removal cap of %d (the fake-snapshot guard still applies).", maxRemove), map[string]any{"max_remove": maxRemove}) return nil } // OpenWindowFor decides and, if due, opens the window: a deleting line for the box's CONFIRMED key is // prepended (first match wins), and a ledger row bounds it to 20 minutes. func (s *Service) OpenWindowFor(ctx context.Context, customerID string, countBefore int) (WindowGrant, error) { oneShot, raisedMax := s.Store.TakeOffsiteWindowGrant(customerID) last := s.Store.LastOffsiteWindowOpened(customerID) due := last.IsZero() || s.now().Sub(last) >= windowCadence if !oneShot && !(s.Store.OffsiteWindowsEnabled() && due) { reason := "weekly windows are off" if s.Store.OffsiteWindowsEnabled() { reason = "not due (last window " + last.UTC().Format(time.RFC3339) + ")" } return WindowGrant{Reason: reason}, nil } k, err := s.Store.GetOffsiteKey(customerID) if err != nil || k == nil || k.ConfirmedAt.IsZero() { return WindowGrant{Reason: "no confirmed append-only key on record"}, nil } t, pw, err := s.TargetFor(customerID) if err != nil { return WindowGrant{}, err } if err := s.Reg.OpenWindow(ctx, t, pw, k.Fingerprint); err != nil { return WindowGrant{}, err } now := s.now() // R-833: an operator grant may RAISE the cap for this one window (never lower it). Only the count // cap moves; the box's fake-snapshot guard still runs in full against NewestAllowed. maxRemove := MaxRemove(countBefore) raised := raisedMax > maxRemove if raised { maxRemove = raisedMax } id, err := s.Store.OpenOffsiteWindowRow(customerID, now.Add(windowLength), countBefore, maxRemove) if err != nil { // The line is written; close it rather than leave a deleting line without a ledger row. _ = s.Reg.CloseWindow(ctx, t, pw) return WindowGrant{}, err } g := WindowGrant{Granted: true, WindowID: id, NewestAllowed: now.UTC(), MaxRemove: maxRemove} s.logf("[WARN] offsitekeys: clean-up window %d OPENED for %s (key %s, %d snapshot(s), max %d removed%s, closes by %s, one-shot=%v)", id, customerID, k.Fingerprint, countBefore, g.MaxRemove, map[bool]string{true: " — OPERATOR-RAISED cap (default " + strconv.Itoa(MaxRemove(countBefore)) + ")", false: ""}[raised], now.Add(windowLength).UTC().Format(time.RFC3339), oneShot) return g, nil } // CloseWindowFor closes the window on the box's report and checks the count. func (s *Service) CloseWindowFor(ctx context.Context, customerID string, r WindowResult) error { w, err := s.Store.GetOffsiteWindow(r.WindowID) if err != nil || w == nil || w.CustomerID != customerID { return fmt.Errorf("offsitekeys: window %d is not this customer's", r.WindowID) } t, pw, err := s.TargetFor(customerID) if err != nil { return err } if err := s.Reg.CloseWindow(ctx, t, pw); err != nil { return err // the sweep retries at closes_by } if _, err := s.Store.CloseOffsiteWindowRow(w.ID, r.CountAfter, r.Outcome, "box"); err != nil { s.logf("[WARN] offsitekeys: ledger close %d: %v", w.ID, err) } drop := w.CountBefore - r.CountAfter allowed := windowCap(w) s.logf("[INFO] offsitekeys: clean-up window %d CLOSED for %s: outcome=%s, %d -> %d (drop %d, allowed %d)", w.ID, customerID, r.Outcome, w.CountBefore, r.CountAfter, drop, allowed) details := map[string]any{"window_id": w.ID, "count_before": w.CountBefore, "count_after": r.CountAfter, "outcome": r.Outcome, "reason": r.Reason} switch { case drop > allowed: s.event(customerID, EventWindowDrop, "error", fmt.Sprintf("Off-site clean-up window %d: the snapshot count fell %d -> %d, more than a window may remove (%d).", w.ID, w.CountBefore, r.CountAfter, allowed), details) case r.Outcome == "guard-refused": s.event(customerID, EventGuardRefused, "error", fmt.Sprintf("Off-site clean-up window %d: the box's fake-snapshot guard refused to prune — nothing was deleted: %s", w.ID, r.Reason), details) case r.Outcome == "error": s.event(customerID, EventWindowFailed, "warning", fmt.Sprintf("Off-site clean-up window %d: the prune failed on the box: %s", w.ID, r.Reason), details) } return nil } // SweepExpiredWindows closes every window left open past its 20 minutes (a box that crashed or lied). func (s *Service) SweepExpiredWindows(ctx context.Context) { ws, err := s.Store.ExpiredOffsiteWindows() if err != nil { s.logf("[WARN] offsitekeys: window sweep: %v", err) return } for _, w := range ws { t, pw, err := s.TargetFor(w.CustomerID) if err == nil { err = s.Reg.CloseWindow(ctx, t, pw) } if err != nil { s.logf("[ERROR] offsitekeys: window %d for %s is past its time and could NOT be closed: %v (retrying next sweep)", w.ID, w.CustomerID, err) continue } _, _ = s.Store.CloseOffsiteWindowRow(w.ID, -1, "", "timeout") s.logf("[WARN] offsitekeys: clean-up window %d for %s was left open — closed by the hub", w.ID, w.CustomerID) s.event(w.CustomerID, EventWindowFailed, "warning", fmt.Sprintf("Off-site clean-up window %d was not closed by the box within %s; the hub closed it (the deleting key line is removed).", w.ID, windowLength), nil) } } // ── Decision 74 (R-823): the household's "delete my set-aside history" — done by the HUB, after a delay ── // // The box's key cannot delete (decision 69), so a due abandonment is a REQUEST to the hub. The hub waits // AbandonDelay (default 7 days) from the request — the household (via the box's recovery, which cancels) // and the operator can cancel in that time — then deletes ONLY `.orphaned-<…>`, never the live // repository. A broken-into box can therefore make a set-aside copy disappear only after a week of // operator mails. Every request, cancel and deletion is an operator event. const ( EventAbandonRequested = "offsite_abandon_requested" EventAbandonCancelled = "offsite_abandon_cancelled" EventAbandonDeleted = "offsite_abandon_deleted" EventAbandonFailed = "offsite_abandon_failed" DefaultAbandonDelay = 7 * 24 * time.Hour ) func (s *Service) abandonDelay() time.Duration { if s.AbandonDelay > 0 { return s.AbandonDelay } return DefaultAbandonDelay } // AbandonStatus is what the box (and the operator) sees. type AbandonStatus struct { State string `json:"state"` // none | pending | cancelled | deleted Path string `json:"path,omitempty"` DueAt time.Time `json:"due_at,omitempty"` } func statusOf(a *store.OffsiteAbandon) AbandonStatus { if a == nil { return AbandonStatus{State: "none"} } return AbandonStatus{State: a.State(), Path: a.Path, DueAt: a.DueAt.UTC()} } // RequestAbandon records (idempotently) the household's request to delete path, due after the delay. func (s *Service) RequestAbandon(ctx context.Context, customerID, path string) (AbandonStatus, error) { t, pw, err := s.TargetFor(customerID) if err != nil { return AbandonStatus{}, err } if !IsSetAsidePath(t.RepoPath, path) { return AbandonStatus{}, fmt.Errorf("offsitekeys: %q is not a set-aside copy of %s", path, t.RepoPath) } if cur, err := s.Store.LatestOffsiteAbandon(customerID, path); err == nil && cur != nil && cur.State() == "pending" { return statusOf(cur), nil // already requested — the clock is NOT restarted } // The copy must exist now (a request for nothing is refused, so a typo cannot sit armed for a week). sh, err := s.Reg.open(ctx, t, pw) if err != nil { return AbandonStatus{}, err } _, lerr := sh.Run(ctx, "ls -d "+path, nil) sh.Close() if lerr != nil { return AbandonStatus{}, fmt.Errorf("offsitekeys: set-aside copy %s not found", path) } due := s.now().Add(s.abandonDelay()) if _, err := s.Store.CreateOffsiteAbandon(customerID, path, due); err != nil { return AbandonStatus{}, err } s.logf("[WARN] offsitekeys: %s asked to DELETE its set-aside off-site copy %s — the hub deletes it at %s unless cancelled (delay %s)", customerID, path, due.UTC().Format(time.RFC3339), s.abandonDelay()) s.event(customerID, EventAbandonRequested, "warning", fmt.Sprintf("Off-site: the box asked to delete the set-aside copy %s (the household's choice). The hub deletes it on %s unless the household or the operator cancels.", path, due.UTC().Format("2006-01-02 15:04 UTC")), map[string]any{"path": path, "due_at": due.UTC()}) a, _ := s.Store.LatestOffsiteAbandon(customerID, path) return statusOf(a), nil } // CancelAbandon cancels every pending request of the customer (by = "box" | "operator"). func (s *Service) CancelAbandon(customerID, by string) (int, error) { n, err := s.Store.CancelOffsiteAbandon(customerID, by) if err != nil || n == 0 { return n, err } s.logf("[INFO] offsitekeys: %s's set-aside deletion CANCELLED by %s (%d request(s)) — nothing deleted", customerID, by, n) s.event(customerID, EventAbandonCancelled, "info", fmt.Sprintf("Off-site: the pending deletion of the set-aside copy was cancelled by the %s. Nothing was deleted.", by), nil) return n, nil } // AbandonStatusFor returns the latest request's state. func (s *Service) AbandonStatusFor(customerID string) (AbandonStatus, error) { a, err := s.Store.LatestOffsiteAbandon(customerID, "") if err != nil { return AbandonStatus{}, err } return statusOf(a), nil } // SweepAbandons deletes every request that is due, not cancelled and not yet deleted. func (s *Service) SweepAbandons(ctx context.Context) { due, err := s.Store.DueOffsiteAbandons() if err != nil { s.logf("[WARN] offsitekeys: abandon sweep: %v", err) return } for _, a := range due { t, pw, terr := s.TargetFor(a.CustomerID) if terr == nil { terr = s.Reg.DeleteSetAside(ctx, t, pw, a.Path) } if terr != nil { _ = s.Store.MarkOffsiteAbandonError(a.ID, terr.Error()) s.logf("[ERROR] offsitekeys: deleting %s's set-aside copy %s failed (retrying): %v", a.CustomerID, a.Path, terr) if a.LastError == "" { s.event(a.CustomerID, EventAbandonFailed, "warning", fmt.Sprintf("Off-site: deleting the set-aside copy %s failed and is retried: %v", a.Path, terr), nil) } continue } _ = s.Store.MarkOffsiteAbandonDeleted(a.ID) s.logf("[WARN] offsitekeys: DELETED %s's set-aside off-site copy %s (requested %s, due %s)", a.CustomerID, a.Path, a.RequestedAt.UTC().Format(time.RFC3339), a.DueAt.UTC().Format(time.RFC3339)) s.event(a.CustomerID, EventAbandonDeleted, "info", fmt.Sprintf("Off-site: the set-aside copy %s was deleted, as the household chose (requested %s).", a.Path, a.RequestedAt.UTC().Format("2006-01-02")), nil) } } // RemoveUnpinnedKeys is the operator's clean-up of a sub-account's key file (decision 72). func (s *Service) RemoveUnpinnedKeys(ctx context.Context, customerID string) (int, error) { t, pw, err := s.TargetFor(customerID) if err != nil { return 0, err } n, err := s.Reg.RemoveUnpinned(ctx, t, pw, s.Store.OffsiteWindowOpen(customerID)) if err != nil { return 0, err } s.logf("[INFO] offsitekeys: removed %d unpinned key line(s) from %s's sub-account (%s) on the operator's request", n, customerID, t.User) return n, nil }