package backup import ( "fmt" "sort" "strconv" "strings" "sync" "time" ) // ── The backup run digest (R-182) ──────────────────────────────────────────────────────────────── // // WHAT WAS WRONG, MEASURED. On 2026-08-03 nine per-app `recovery_unit_capture_failed` events reached // the hub and TWO operator e-mails went out. The hub's operator cooldown key is // `customerID + ":" + eventType + tier-suffix`, and that event carries `app` but no `tier`, so the // key held **no app identifier**: the first refused app took the hour's slot and every other app's // failure was discarded — *before* anything was written down, so it left no row on any channel and // could not be found afterwards. // // WHY NOT JUST PUT THE APP IN THE KEY. That was the obvious fix and the operator ruled against it: // on a full disk it produces one e-mail per app, which is the volume problem wearing the correctness // problem's clothes. **One digest per run instead**, listing every failure — and separately, every // failure recorded when it happens. // // THE RECORD AND THE NOTIFICATION ARE DIFFERENT THINGS, and that separation is the durable part: // // - the RECORD is the per-app `recovery_unit_capture_failed` event, emitted unconditionally, now // routed record-only by the hub so it never competes for an e-mail slot; // - the NOTIFICATION is this digest, emitted once per run and only when something failed. // // WHY A DIGEST IS SAFE HERE — the one thing that could have made it dangerous. A digest introduces a // silent-failure path if the ABSENCE of an e-mail could mean "the run never finished". It cannot: // the hub's daily deadline check raises `expected_backup_missed` / `expected_dbdump_missed` // (`hub/internal/monitor/deadline.go:396,417`) from the box's REPORT freshness and its stored // events, entirely independently of any mail the controller chooses to send. Silence therefore still // means "the run finished and found nothing wrong". If that check is ever weakened, this design // loses its footing — which is why it is named here and not only in a report. // runKind distinguishes the paths that can produce a digest. It is carried in the details and shown // in the subject, because "the nightly run failed" and "the run I just triggered failed" are read // differently at 07:00. const ( runKindNightly = "nightly" runKindManual = "manual" // runKindRefresh is the PERIODIC status sweep, which captures units outside any backup run. // // IT DELIBERATELY CARRIES NO run_id. A run digest gets a unique run id so the hub's 1-hour // cooldown can never collapse two real runs (the operator ruled on that explicitly). The sweep // is the opposite case: it can fire every time the status page is polled, so it MUST fall under // the ordinary cooldown or a full disk becomes a mail flood — the exact failure this whole // change exists to avoid, arriving from the other direction. // // Without this path the sweep's failures would be recorded and never notified, because the // per-app event is now record-only — a NEW silence introduced while fixing a silence. This is // what stops that. runKindRefresh = "refresh" ) // runFailure is one app's failed or refused leg within a run. type runFailure struct { App string Leg string Reason string } // runSummary is the per-RUN collector. Same lifetime as `admissionSet` and for the same reason: an // absent collector must mean "no run in flight", never "a stale answer from last night". type runSummary struct { mu sync.Mutex kind string runID string attempted map[string]bool // apps this run actually tried to back up failures []runFailure } // beginRunSummary opens the per-run digest scope and returns the closer, mirroring // beginAdmissionRun. A second call while one is live REPLACES it and the closer restores the // previous, so nesting cannot silently drop a caller's scope. func (m *Manager) beginRunSummary(kind, runID string) func() { m.summaryMu.Lock() prev := m.summary m.summary = &runSummary{kind: kind, runID: runID, attempted: map[string]bool{}} m.summaryMu.Unlock() return func() { m.summaryMu.Lock() m.summary = prev m.summaryMu.Unlock() } } // noteAttempted records that this run tried to back up an app. // // THE DENOMINATOR IS NOT DECORATION. "3 of 4 apps failed" is a catastrophe and "3 of 40" is a bad // night, and a list of names cannot tell them apart — the operator's first decision, get up now or // look after coffee, is made from exactly this ratio. func (m *Manager) noteAttempted(app string) { m.summaryMu.Lock() defer m.summaryMu.Unlock() if m.summary != nil { m.summary.mu.Lock() m.summary.attempted[app] = true m.summary.mu.Unlock() } } // noteFailure records one failed or refused leg. Deliberate skips must NOT come through here — // §8.1: a drive that is unplugged or decommissioned has its own alert, and putting it in the digest // turns a nightly e-mail into one the operator learns to ignore. func (m *Manager) noteFailure(app, leg, reason string) { m.summaryMu.Lock() defer m.summaryMu.Unlock() if m.summary == nil { return // no run in flight — nothing to summarise } m.summary.mu.Lock() m.summary.attempted[app] = true m.summary.failures = append(m.summary.failures, runFailure{App: app, Leg: leg, Reason: reason}) m.summary.mu.Unlock() } // emitRunSummary sends the digest, if and only if something failed. // // A CLEAN RUN EMITS NOTHING — not an empty digest. A nightly "0 failures" mail is an unread mail // within a week, and it would also destroy the property the whole design rests on: that silence // means the run finished and found nothing wrong. func (m *Manager) emitRunSummary() { m.summaryMu.Lock() s := m.summary m.summaryMu.Unlock() if s == nil { return } s.mu.Lock() failures := append([]runFailure(nil), s.failures...) attempted := len(s.attempted) kind, runID := s.kind, s.runID s.mu.Unlock() if len(failures) == 0 { return } // Stable order so two identical runs render identically and a diff of two mails is meaningful. sort.Slice(failures, func(i, j int) bool { if failures[i].App != failures[j].App { return failures[i].App < failures[j].App } return failures[i].Leg < failures[j].Leg }) if m.runSummaryNotify == nil { // Nil-safe, but say so: an unwired seam here means the digest exists and reaches nobody, // which is the built-but-never-wired failure this project has shipped four times. m.logger.Printf("[WARN] [backup] %d app(s) failed in this %s run but no run-summary notifier is wired — "+ "the failures are recorded per app and NOT summarised to the operator", len(failures), kind) return } apps := make([]RunFailureDetail, 0, len(failures)) names := make([]string, 0, len(failures)) for _, f := range failures { apps = append(apps, RunFailureDetail{App: f.App, Leg: f.Leg, Reason: f.Reason}) names = append(names, f.App) } msg := fmt.Sprintf("%d of %d apps failed to back up in this %s run: %s", len(failures), attempted, kind, strings.Join(dedupeStable(names), ", ")) m.logger.Printf("[INFO] [backup] Run summary: %d of %d apps failed (%s run) — notifying the operator once", len(failures), attempted, kind) m.runSummaryNotify(RunSummary{ RunID: runID, RunKind: kind, Failed: len(failures), Attempted: attempted, Apps: apps, Usage: m.summaryUsage(), Message: msg, }) } // summaryUsage reads the target filesystem once for the digest. It uses the same seam the reserve // does, so a test states occupancy as an input rather than manufacturing it. Nil when unreadable — // which the hub renders as "unavailable", never as zeros. func (m *Manager) summaryUsage() *UnitSpace { if m.stackProvider == nil { return nil } for _, st := range m.stackProvider.ListDeployedStacks() { if u := m.readUnitSpace(st.Name); u != nil { return u } } return nil } func dedupeStable(in []string) []string { seen := map[string]bool{} out := in[:0:0] for _, s := range in { if !seen[s] { seen[s] = true out = append(out, s) } } return out } // RunFailureDetail is one app's failure as the hub receives it. type RunFailureDetail struct { App string `json:"app"` Leg string `json:"leg"` Reason string `json:"reason"` } // RunSummary is the digest payload handed to the notifier seam. type RunSummary struct { RunID string RunKind string Failed int Attempted int Apps []RunFailureDetail Usage *UnitSpace Message string } // SetRunSummaryNotify wires the per-run digest. INIT-ONLY — call once at startup in main.go, // alongside SetUnitNotify. Nil-safe, but an unwired seam is logged loudly rather than being silently // the old behaviour. func (m *Manager) SetRunSummaryNotify(fn func(RunSummary)) { m.runSummaryNotify = fn } // admissionReason returns the human reason this run refused an app, taken from the verdict the // reserve already recorded. Reused rather than re-derived: the verdict carries the bound term, the // estimate and the disk figures, and re-deriving them here would be a second source of truth for a // sentence the operator reads. func (m *Manager) admissionReason(stackName string) string { m.admissionMu.Lock() defer m.admissionMu.Unlock() if set := m.admission; set != nil { if v, ok := set.v[stackName]; ok && v.err != nil { return v.err.Error() } } return "refused by the capture reserve" } // runKindFor reports whether this run was the scheduled one or one a person triggered. // // The distinction is the operator's ruling: someone pressing the button is actively trying to get a // backup, so a manual run must report even if the nightly one already wrote this hour. It is carried // into the subject line because "the nightly run failed" and "the run I just asked for failed" are // acted on differently. func (m *Manager) runKindFor() string { if m.manualRun.Load() { return runKindManual } return runKindNightly } // MarkManualRun tags the NEXT backup run as operator-triggered. Called by the API/debug handlers // that expose a "run backup now" control; the scheduled path leaves it alone. func (m *Manager) MarkManualRun() { m.manualRun.Store(true) } // newRunID returns a per-run identifier. It only has to be distinct between two runs on one box // within the hub's 1-hour cooldown window, which a nanosecond clock reading satisfies; it is never // persisted, compared across boxes, or used as a security value. func newRunID() string { return "run-" + strconv.FormatInt(time.Now().UnixNano(), 36) }