felhom-controller/controller/internal/backup/runsummary.go

package backup

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// ── The backup run digest (R-182) ────────────────────────────────────────────────────────────────
//
// WHAT WAS WRONG, MEASURED. On 2026-08-03 nine per-app `recovery_unit_capture_failed` events reached
// the hub and TWO operator e-mails went out. The hub's operator cooldown key is
// `customerID + ":" + eventType + tier-suffix`, and that event carries `app` but no `tier`, so the
// key held **no app identifier**: the first refused app took the hour's slot and every other app's
// failure was discarded — *before* anything was written down, so it left no row on any channel and
// could not be found afterwards.
//
// WHY NOT JUST PUT THE APP IN THE KEY. That was the obvious fix and the operator ruled against it:
// on a full disk it produces one e-mail per app, which is the volume problem wearing the correctness
// problem's clothes. **One digest per run instead**, listing every failure — and separately, every
// failure recorded when it happens.
//
// THE RECORD AND THE NOTIFICATION ARE DIFFERENT THINGS, and that separation is the durable part:
//
//   - the RECORD is the per-app `recovery_unit_capture_failed` event, emitted unconditionally, now
//     routed record-only by the hub so it never competes for an e-mail slot;
//   - the NOTIFICATION is this digest, emitted once per run and only when something failed.
//
// WHY A DIGEST IS SAFE HERE — the one thing that could have made it dangerous. A digest introduces a
// silent-failure path if the ABSENCE of an e-mail could mean "the run never finished". It cannot:
// the hub's daily deadline check raises `expected_backup_missed` / `expected_dbdump_missed`
// (`hub/internal/monitor/deadline.go:396,417`) from the box's REPORT freshness and its stored
// events, entirely independently of any mail the controller chooses to send. Silence therefore still
// means "the run finished and found nothing wrong". If that check is ever weakened, this design
// loses its footing — which is why it is named here and not only in a report.

// runKind distinguishes the paths that can produce a digest. It is carried in the details and shown
// in the subject, because "the nightly run failed" and "the run I just triggered failed" are read
// differently at 07:00.
const (
	runKindNightly = "nightly"
	runKindManual  = "manual"
	// runKindRefresh is the PERIODIC status sweep, which captures units outside any backup run.
	//
	// IT DELIBERATELY CARRIES NO run_id. A run digest gets a unique run id so the hub's 1-hour
	// cooldown can never collapse two real runs (the operator ruled on that explicitly). The sweep
	// is the opposite case: it can fire every time the status page is polled, so it MUST fall under
	// the ordinary cooldown or a full disk becomes a mail flood — the exact failure this whole
	// change exists to avoid, arriving from the other direction.
	//
	// Without this path the sweep's failures would be recorded and never notified, because the
	// per-app event is now record-only — a NEW silence introduced while fixing a silence. This is
	// what stops that.
	runKindRefresh = "refresh"
)

// runFailure is one app's failed or refused leg within a run.
type runFailure struct {
	App    string
	Leg    string
	Reason string
}

// runSummary is the per-RUN collector. Same lifetime as `admissionSet` and for the same reason: an
// absent collector must mean "no run in flight", never "a stale answer from last night".
type runSummary struct {
	mu        sync.Mutex
	kind      string
	runID     string
	attempted map[string]bool // apps this run actually tried to back up
	failures  []runFailure
}

// beginRunSummary opens the per-run digest scope and returns the closer, mirroring
// beginAdmissionRun. A second call while one is live REPLACES it and the closer restores the
// previous, so nesting cannot silently drop a caller's scope.
func (m *Manager) beginRunSummary(kind, runID string) func() {
	m.summaryMu.Lock()
	prev := m.summary
	m.summary = &runSummary{kind: kind, runID: runID, attempted: map[string]bool{}}
	m.summaryMu.Unlock()
	return func() {
		m.summaryMu.Lock()
		m.summary = prev
		m.summaryMu.Unlock()
	}
}

// noteAttempted records that this run tried to back up an app.
//
// THE DENOMINATOR IS NOT DECORATION. "3 of 4 apps failed" is a catastrophe and "3 of 40" is a bad
// night, and a list of names cannot tell them apart — the operator's first decision, get up now or
// look after coffee, is made from exactly this ratio.
func (m *Manager) noteAttempted(app string) {
	m.summaryMu.Lock()
	defer m.summaryMu.Unlock()
	if m.summary != nil {
		m.summary.mu.Lock()
		m.summary.attempted[app] = true
		m.summary.mu.Unlock()
	}
}

// noteFailure records one failed or refused leg. Deliberate skips must NOT come through here —
// §8.1: a drive that is unplugged or decommissioned has its own alert, and putting it in the digest
// turns a nightly e-mail into one the operator learns to ignore.
func (m *Manager) noteFailure(app, leg, reason string) {
	m.summaryMu.Lock()
	defer m.summaryMu.Unlock()
	if m.summary == nil {
		return // no run in flight — nothing to summarise
	}
	m.summary.mu.Lock()
	m.summary.attempted[app] = true
	m.summary.failures = append(m.summary.failures, runFailure{App: app, Leg: leg, Reason: reason})
	m.summary.mu.Unlock()
}

// emitRunSummary sends the digest, if and only if something failed.
//
// A CLEAN RUN EMITS NOTHING — not an empty digest. A nightly "0 failures" mail is an unread mail
// within a week, and it would also destroy the property the whole design rests on: that silence
// means the run finished and found nothing wrong.
func (m *Manager) emitRunSummary() {
	m.summaryMu.Lock()
	s := m.summary
	m.summaryMu.Unlock()
	if s == nil {
		return
	}
	s.mu.Lock()
	failures := append([]runFailure(nil), s.failures...)
	attempted := len(s.attempted)
	kind, runID := s.kind, s.runID
	s.mu.Unlock()

	if len(failures) == 0 {
		return
	}
	// Stable order so two identical runs render identically and a diff of two mails is meaningful.
	sort.Slice(failures, func(i, j int) bool {
		if failures[i].App != failures[j].App {
			return failures[i].App < failures[j].App
		}
		return failures[i].Leg < failures[j].Leg
	})

	if m.runSummaryNotify == nil {
		// Nil-safe, but say so: an unwired seam here means the digest exists and reaches nobody,
		// which is the built-but-never-wired failure this project has shipped four times.
		m.logger.Printf("[WARN] [backup] %d app(s) failed in this %s run but no run-summary notifier is wired — "+
			"the failures are recorded per app and NOT summarised to the operator", len(failures), kind)
		return
	}

	apps := make([]RunFailureDetail, 0, len(failures))
	names := make([]string, 0, len(failures))
	for _, f := range failures {
		apps = append(apps, RunFailureDetail{App: f.App, Leg: f.Leg, Reason: f.Reason})
		names = append(names, f.App)
	}
	msg := fmt.Sprintf("%d of %d apps failed to back up in this %s run: %s",
		len(failures), attempted, kind, strings.Join(dedupeStable(names), ", "))

	m.logger.Printf("[INFO] [backup] Run summary: %d of %d apps failed (%s run) — notifying the operator once",
		len(failures), attempted, kind)
	m.runSummaryNotify(RunSummary{
		RunID:     runID,
		RunKind:   kind,
		Failed:    len(failures),
		Attempted: attempted,
		Apps:      apps,
		Usage:     m.summaryUsage(),
		Message:   msg,
	})
}

// summaryUsage reads the target filesystem once for the digest. It uses the same seam the reserve
// does, so a test states occupancy as an input rather than manufacturing it. Nil when unreadable —
// which the hub renders as "unavailable", never as zeros.
func (m *Manager) summaryUsage() *UnitSpace {
	if m.stackProvider == nil {
		return nil
	}
	for _, st := range m.stackProvider.ListDeployedStacks() {
		if u := m.readUnitSpace(st.Name); u != nil {
			return u
		}
	}
	return nil
}

func dedupeStable(in []string) []string {
	seen := map[string]bool{}
	out := in[:0:0]
	for _, s := range in {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	return out
}

// RunFailureDetail is one app's failure as the hub receives it.
type RunFailureDetail struct {
	App    string `json:"app"`
	Leg    string `json:"leg"`
	Reason string `json:"reason"`
}

// RunSummary is the digest payload handed to the notifier seam.
type RunSummary struct {
	RunID     string
	RunKind   string
	Failed    int
	Attempted int
	Apps      []RunFailureDetail
	Usage     *UnitSpace
	Message   string
}

// SetRunSummaryNotify wires the per-run digest. INIT-ONLY — call once at startup in main.go,
// alongside SetUnitNotify. Nil-safe, but an unwired seam is logged loudly rather than being silently
// the old behaviour.
func (m *Manager) SetRunSummaryNotify(fn func(RunSummary)) {
	m.runSummaryNotify = fn
}

// admissionReason returns the human reason this run refused an app, taken from the verdict the
// reserve already recorded. Reused rather than re-derived: the verdict carries the bound term, the
// estimate and the disk figures, and re-deriving them here would be a second source of truth for a
// sentence the operator reads.
func (m *Manager) admissionReason(stackName string) string {
	m.admissionMu.Lock()
	defer m.admissionMu.Unlock()
	if set := m.admission; set != nil {
		if v, ok := set.v[stackName]; ok && v.err != nil {
			return v.err.Error()
		}
	}
	return "refused by the capture reserve"
}

// runKindFor reports whether this run was the scheduled one or one a person triggered.
//
// The distinction is the operator's ruling: someone pressing the button is actively trying to get a
// backup, so a manual run must report even if the nightly one already wrote this hour. It is carried
// into the subject line because "the nightly run failed" and "the run I just asked for failed" are
// acted on differently.
func (m *Manager) runKindFor() string {
	if m.manualRun.Load() {
		return runKindManual
	}
	return runKindNightly
}

// MarkManualRun tags the NEXT backup run as operator-triggered. Called by the API/debug handlers
// that expose a "run backup now" control; the scheduled path leaves it alone.
func (m *Manager) MarkManualRun() { m.manualRun.Store(true) }

// newRunID returns a per-run identifier. It only has to be distinct between two runs on one box
// within the hub's 1-hour cooldown window, which a nanosecond clock reading satisfies; it is never
// persisted, compared across boxes, or used as a security value.
func newRunID() string {
	return "run-" + strconv.FormatInt(time.Now().UnixNano(), 36)
}