felhom.eu/hub/internal/monitor/offsite_escrow_pending.go

package monitor

import (
	"encoding/json"
	"fmt"
	"time"
)

// ── R-243 (2026-10-08) — A BOX THAT NEVER BACKS UP OFF-SITE BECAUSE ITS ESCROW IS PENDING ─────────
//
// `offsite_stale` deliberately ignores a box whose escrow is not `escrowed`: pending is the designed
// onboarding state (`07` §6.1, Tier-3 PAUSED — a run without the household's recovery code would write a
// copy nobody could open). That exclusion is right, and it left one state unobserved: a household that
// never does the escrow step (or a box held in `awaiting_recovery_key`, the R-241 state) never backs up
// off-site, and no alarm of any kind fires — `offsite_stale` needs `escrowed`, the delivery checker skips
// the applied shape, and `backup_failed` needs a run that never starts.
//
// THE SIGNAL. `offsite_escrow_pending` (warning, OPERATOR-ONLY — the household already sees the reminder
// on every page; this is the operator's „they have not acted" line): off-site is ON, the escrow is not
// `escrowed`, and there has been no successful off-site run for escrowPendingAfter. The clock starts at
// the last successful run when there was one (a box that fell back to pending), otherwise at the first
// host report the hub received from the customer (when the box became observable). An unknown anchor
// never fires: a box that has never sent a host report has its own staleness alarms, and firing here on
// a guess is the 2026-07-23 cry-wolf.
//
// THE LINE: 7 days. `offsite_stale`'s 48 h assumes runs are EXPECTED; here they are not yet, because
// the household is in its onboarding step, so the line must be longer. 7 days is `08` §6.3's line for „a
// box needing an action nobody took" (`os_update_stale`, `agent_behind`) and the off-site whole-guest
// cadence, so a household that does the step in its first week is never reported.
//
// ONE MAIL PER BOX, THEN QUIET FOR A WEEK; it is re-sent weekly while still true (`08` §6.3's re-send
// rule) and CLEARS when the state ends (escrowed, off-site off, or a successful run after the anchor) —
// with one info line, recorded and never mailed, so the operator who was told it broke can see it healed.
// The raise time is persisted (`os_alarm:` setting), so a hub restart neither re-mails nor forgets.
//
// Pinned by TestR243_* (offsite_escrow_pending_test.go).
const escrowPendingAfter = 7 * 24 * time.Hour

const (
	eventEscrowPending        = "offsite_escrow_pending"
	eventEscrowPendingCleared = "offsite_escrow_pending_cleared"
)

func escrowPendingKey(customerID string) string { return "offsite_escrow_pending:" + customerID }

// escrowPendingAnchor returns when the no-off-site clock started, and whether it is known.
func (oc *OffsiteChecker) escrowPendingAnchor(customerID string, off *offsiteReport) (time.Time, bool) {
	if off.LastSuccess != "" {
		if t, err := time.Parse(time.RFC3339, off.LastSuccess); err == nil {
			return t, true
		}
	}
	first, err := oc.store.GetFirstHostReportAt(customerID)
	if err != nil || first.IsZero() {
		return time.Time{}, false
	}
	return first, true
}

// inEscrowPending is the state: off-site on, escrow not done.
func inEscrowPending(off *offsiteReport) bool {
	return off != nil && off.Enabled && off.EscrowState != "escrowed"
}

// checkEscrowPending raises, re-sends weekly, or clears the signal for one customer. Caller holds oc.mu.
func (oc *OffsiteChecker) checkEscrowPending(customerID string, off *offsiteReport) {
	key := escrowPendingKey(customerID)
	raised := oc.store.OSAlarmRaised(key)
	now := oc.now()

	clear := func(why string) {
		if raised.IsZero() {
			return
		}
		_ = oc.store.SetOSAlarmRaised(key, time.Time{})
		msg := fmt.Sprintf("Customer %s: off-site backup is no longer held by a pending escrow (%s).", customerID, why)
		details, _ := json.Marshal(map[string]any{"customer_id": customerID, "reason": why})
		oc.logger.Printf("[INFO] Offsite escrow pending CLEARED: %s (%s)", customerID, why)
		if _, err := oc.store.SaveEvent(customerID, eventEscrowPendingCleared, "info", msg, string(details), "hub"); err != nil {
			oc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventEscrowPendingCleared, customerID, err)
			return
		}
		if oc.onEvent != nil {
			oc.onEvent(customerID, eventEscrowPendingCleared, "info", msg, string(details), "hub")
		}
	}

	if !inEscrowPending(off) {
		why := "the escrow is done"
		if off == nil || !off.Enabled {
			why = "off-site backup is off"
		}
		clear(why)
		return
	}
	anchor, ok := oc.escrowPendingAnchor(customerID, off)
	if !ok {
		oc.logger.Printf("[DEBUG] Offsite escrow pending: %s — no anchor (no successful run, no host report) — not judged", customerID)
		return
	}
	if !raised.IsZero() && anchor.After(raised) {
		// A run succeeded after the alarm and the box fell back to pending again: a new episode.
		clear("a run succeeded after the alarm")
		raised = time.Time{}
	}
	age := now.Sub(anchor)
	if age <= escrowPendingAfter {
		return
	}
	if !raised.IsZero() && now.Sub(raised) < 7*24*time.Hour {
		return // quiet for a week
	}
	since := "the box first reported"
	if off.LastSuccess != "" {
		since = "its last successful off-site run"
	}
	state := off.EscrowState
	if state == "" {
		state = "not started"
	}
	extra := ""
	if off.State != "" {
		extra = fmt.Sprintf(" The box declares off-site state %q.", off.State)
	}
	msg := fmt.Sprintf("Customer %s: off-site backup is ON but has not run for %s since %s — the escrow step is %s, so no off-site copy is being made.%s "+
		"The household sees the reminder on every page; this mail is for you: they have not acted.",
		customerID, age.Round(time.Hour), since, state, extra)
	details, _ := json.Marshal(map[string]any{
		"customer_id": customerID, "escrow_state": off.EscrowState, "offsite_state": off.State,
		"last_success": off.LastSuccess, "anchor": anchor.UTC().Format(time.RFC3339),
		"after": escrowPendingAfter.String(),
	})
	if err := oc.store.SetOSAlarmRaised(key, now); err != nil {
		oc.logger.Printf("[WARN] Offsite escrow pending: could not record the raise for %s (%v) — not sending, so a restart cannot mail twice", customerID, err)
		return
	}
	oc.logger.Printf("[INFO] Offsite escrow pending: %s (%s since %s)", customerID, age.Round(time.Hour), since)
	if _, err := oc.store.SaveEvent(customerID, eventEscrowPending, "warning", msg, string(details), "hub"); err != nil {
		oc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventEscrowPending, customerID, err)
		return
	}
	if oc.onEvent != nil {
		oc.onEvent(customerID, eventEscrowPending, "warning", msg, string(details), "hub")
	}
}