package monitor import ( "encoding/json" "fmt" "time" ) // ── R-243 (2026-10-08) — A BOX THAT NEVER BACKS UP OFF-SITE BECAUSE ITS ESCROW IS PENDING ───────── // // `offsite_stale` deliberately ignores a box whose escrow is not `escrowed`: pending is the designed // onboarding state (`07` §6.1, Tier-3 PAUSED — a run without the household's recovery code would write a // copy nobody could open). That exclusion is right, and it left one state unobserved: a household that // never does the escrow step (or a box held in `awaiting_recovery_key`, the R-241 state) never backs up // off-site, and no alarm of any kind fires — `offsite_stale` needs `escrowed`, the delivery checker skips // the applied shape, and `backup_failed` needs a run that never starts. // // THE SIGNAL. `offsite_escrow_pending` (warning, OPERATOR-ONLY — the household already sees the reminder // on every page; this is the operator's „they have not acted" line): off-site is ON, the escrow is not // `escrowed`, and there has been no successful off-site run for escrowPendingAfter. The clock starts at // the last successful run when there was one (a box that fell back to pending), otherwise at the first // host report the hub received from the customer (when the box became observable). An unknown anchor // never fires: a box that has never sent a host report has its own staleness alarms, and firing here on // a guess is the 2026-07-23 cry-wolf. // // THE LINE: 7 days. `offsite_stale`'s 48 h assumes runs are EXPECTED; here they are not yet, because // the household is in its onboarding step, so the line must be longer. 7 days is `08` §6.3's line for „a // box needing an action nobody took" (`os_update_stale`, `agent_behind`) and the off-site whole-guest // cadence, so a household that does the step in its first week is never reported. // // ONE MAIL PER BOX, THEN QUIET FOR A WEEK; it is re-sent weekly while still true (`08` §6.3's re-send // rule) and CLEARS when the state ends (escrowed, off-site off, or a successful run after the anchor) — // with one info line, recorded and never mailed, so the operator who was told it broke can see it healed. // The raise time is persisted (`os_alarm:` setting), so a hub restart neither re-mails nor forgets. // // Pinned by TestR243_* (offsite_escrow_pending_test.go). const escrowPendingAfter = 7 * 24 * time.Hour const ( eventEscrowPending = "offsite_escrow_pending" eventEscrowPendingCleared = "offsite_escrow_pending_cleared" ) func escrowPendingKey(customerID string) string { return "offsite_escrow_pending:" + customerID } // escrowPendingAnchor returns when the no-off-site clock started, and whether it is known. func (oc *OffsiteChecker) escrowPendingAnchor(customerID string, off *offsiteReport) (time.Time, bool) { if off.LastSuccess != "" { if t, err := time.Parse(time.RFC3339, off.LastSuccess); err == nil { return t, true } } first, err := oc.store.GetFirstHostReportAt(customerID) if err != nil || first.IsZero() { return time.Time{}, false } return first, true } // inEscrowPending is the state: off-site on, escrow not done. func inEscrowPending(off *offsiteReport) bool { return off != nil && off.Enabled && off.EscrowState != "escrowed" } // checkEscrowPending raises, re-sends weekly, or clears the signal for one customer. Caller holds oc.mu. func (oc *OffsiteChecker) checkEscrowPending(customerID string, off *offsiteReport) { key := escrowPendingKey(customerID) raised := oc.store.OSAlarmRaised(key) now := oc.now() clear := func(why string) { if raised.IsZero() { return } _ = oc.store.SetOSAlarmRaised(key, time.Time{}) msg := fmt.Sprintf("Customer %s: off-site backup is no longer held by a pending escrow (%s).", customerID, why) details, _ := json.Marshal(map[string]any{"customer_id": customerID, "reason": why}) oc.logger.Printf("[INFO] Offsite escrow pending CLEARED: %s (%s)", customerID, why) if _, err := oc.store.SaveEvent(customerID, eventEscrowPendingCleared, "info", msg, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventEscrowPendingCleared, customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, eventEscrowPendingCleared, "info", msg, string(details), "hub") } } if !inEscrowPending(off) { why := "the escrow is done" if off == nil || !off.Enabled { why = "off-site backup is off" } clear(why) return } anchor, ok := oc.escrowPendingAnchor(customerID, off) if !ok { oc.logger.Printf("[DEBUG] Offsite escrow pending: %s — no anchor (no successful run, no host report) — not judged", customerID) return } if !raised.IsZero() && anchor.After(raised) { // A run succeeded after the alarm and the box fell back to pending again: a new episode. clear("a run succeeded after the alarm") raised = time.Time{} } age := now.Sub(anchor) if age <= escrowPendingAfter { return } if !raised.IsZero() && now.Sub(raised) < 7*24*time.Hour { return // quiet for a week } since := "the box first reported" if off.LastSuccess != "" { since = "its last successful off-site run" } state := off.EscrowState if state == "" { state = "not started" } extra := "" if off.State != "" { extra = fmt.Sprintf(" The box declares off-site state %q.", off.State) } msg := fmt.Sprintf("Customer %s: off-site backup is ON but has not run for %s since %s — the escrow step is %s, so no off-site copy is being made.%s "+ "The household sees the reminder on every page; this mail is for you: they have not acted.", customerID, age.Round(time.Hour), since, state, extra) details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "escrow_state": off.EscrowState, "offsite_state": off.State, "last_success": off.LastSuccess, "anchor": anchor.UTC().Format(time.RFC3339), "after": escrowPendingAfter.String(), }) if err := oc.store.SetOSAlarmRaised(key, now); err != nil { oc.logger.Printf("[WARN] Offsite escrow pending: could not record the raise for %s (%v) — not sending, so a restart cannot mail twice", customerID, err) return } oc.logger.Printf("[INFO] Offsite escrow pending: %s (%s since %s)", customerID, age.Round(time.Hour), since) if _, err := oc.store.SaveEvent(customerID, eventEscrowPending, "warning", msg, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventEscrowPending, customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, eventEscrowPending, "warning", msg, string(details), "hub") } }