package monitor import ( "encoding/json" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // ── D53 (`08` §10, R-1291) — A HELD APP REACHES THE OPERATOR ────────────────────────────────────── // // A held app is stopped on purpose and stays stopped until a person acts. demo-hp's Kimai sat held for 11 days // (2026-10-14 → 10-26) and nobody was told after the first minute. The box now reports its holds (app_held / // app_hold_cleared, severity info — nothing is mailed on receipt); this checker turns a hold that LASTS into ONE // operator mail after a grace that follows the reason, then one a week while it lasts. // // THE GRACE (`08` §10): putback_pending 6 h — it should have cleared itself, so it is stuck; restore_failed and // restore_mixed 1 day — only a person can act; update_failed and unhealthy_stop 3 days — the household can act first. // An unknown reason gets 1 day. The clock is the hold's own time (the box's), else when the hub first heard of it. // // OPERATOR ONLY (D53): app_held_reminder is in the dispatcher's operatorOnlyEvents, so no household is ever mailed. // THE HUB NEVER CLEARS A HOLD — a row goes only when the box sends app_hold_cleared. // // A ROW THE BOX STOPPED CONFIRMING is not reminded: the box repeats app_held daily, so a row older than heldAppStale // belongs to a box that is down or gone, which its own staleness alarms report. The list still shows it. // // The reminder is recorded (MarkHeldAppReminded) BEFORE it is sent, so a hub restart cannot mail twice. // Pinned by held_apps_test.go. const ( eventHeldAppReminder = "app_held_reminder" heldAppRepeat = 7 * 24 * time.Hour heldAppStale = 50 * time.Hour heldAppCheckEvery = 15 * time.Minute ) // HeldAppGrace returns the wait before the first reminder for a reason (`08` §10). func HeldAppGrace(reason string) time.Duration { switch reason { case "putback_pending": return 6 * time.Hour case "update_failed", "unhealthy_stop": return 72 * time.Hour default: // restore_failed, restore_mixed, and a reason this hub does not know yet return 24 * time.Hour } } // HeldAppWhoClears says who may lift a hold, in the operator's words. func HeldAppWhoClears(reason string) string { switch reason { case "update_failed": return "the household, by restoring the app from its backup" case "unhealthy_stop": return "the household's Start (one more try)" case "putback_pending": return "nobody — it clears itself when the drive with the pre-restore copy comes back" default: return "a person only (support: the controller's -clear-restore-hold)" } } type heldAppsStore interface { ListHeldApps(customerID string) ([]store.HeldApp, error) MarkHeldAppReminded(customerID, stack, reason string, since, at time.Time) error SaveEvent(customerID, eventType, severity, message, detailsJSON, source string) (int64, error) } // HeldAppsChecker sends the operator's reminders for held apps. type HeldAppsChecker struct { store heldAppsStore onEvent EventNotifyFunc logger *log.Logger now func() time.Time every time.Duration // graceOverride, when > 0, replaces every reason's grace. A TEST configuration only (the live proof shortens // the wait); production leaves it 0. Logged at start when set. graceOverride time.Duration mu sync.Mutex lastRun time.Time } // NewHeldAppsChecker builds the checker. graceOverride 0 = the per-reason grace. func NewHeldAppsChecker(st heldAppsStore, graceOverride time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *HeldAppsChecker { if graceOverride > 0 && logger != nil { logger.Printf("[WARN] Held apps: the reminder grace is OVERRIDDEN to %s for every reason — a test configuration", graceOverride) } return &HeldAppsChecker{store: st, onEvent: onEvent, logger: logger, now: time.Now, every: heldAppCheckEvery, graceOverride: graceOverride} } func (hc *HeldAppsChecker) grace(reason string) time.Duration { if hc.graceOverride > 0 { return hc.graceOverride } return HeldAppGrace(reason) } // Check runs on the hub's 60 s sweep and throttles itself to one pass per heldAppCheckEvery. func (hc *HeldAppsChecker) Check() { hc.mu.Lock() defer hc.mu.Unlock() now := hc.now() every := hc.every if hc.graceOverride > 0 && hc.graceOverride < every { every = hc.graceOverride // a test grace of minutes must not wait a 15-minute pass } if !hc.lastRun.IsZero() && now.Sub(hc.lastRun) < every { return } hc.lastRun = now held, err := hc.store.ListHeldApps("") if err != nil { hc.logger.Printf("[WARN] Held apps: list failed: %v — no reminder this pass", err) return } for _, h := range held { hc.checkOne(h, now) } } func (hc *HeldAppsChecker) checkOne(h store.HeldApp, now time.Time) { if now.Sub(h.LastSeen) > heldAppStale { hc.logger.Printf("[DEBUG] Held apps: %s/%s not confirmed since %s — no reminder (the box is not reporting)", h.CustomerID, h.Stack, h.LastSeen.Format(time.RFC3339)) return } anchor := h.Since if anchor.IsZero() { anchor = h.FirstSeen } age := now.Sub(anchor) if age < hc.grace(h.Reason) { return } if !h.RemindedAt.IsZero() && now.Sub(h.RemindedAt) < heldAppRepeat { return } if err := hc.store.MarkHeldAppReminded(h.CustomerID, h.Stack, h.Reason, h.Since, now); err != nil { hc.logger.Printf("[WARN] Held apps: could not record the reminder for %s/%s (%v) — not sending, so a restart cannot mail twice", h.CustomerID, h.Stack, err) return } repeat := !h.RemindedAt.IsZero() msg := fmt.Sprintf("Customer %s: the app %s has been held for %s (%s): %s. Who may clear it: %s. The hub never clears a hold.", h.CustomerID, h.Stack, roundAge(age), h.Reason, h.Why, HeldAppWhoClears(h.Reason)) if repeat { msg += " Weekly reminder: it is still held." } details, _ := json.Marshal(map[string]any{ "customer_id": h.CustomerID, "stack_name": h.Stack, "reason": h.Reason, "since": anchor.UTC().Format(time.RFC3339), "why": h.Why, "repeat": repeat, }) hc.logger.Printf("[INFO] Held apps: reminder for %s/%s (%s, held %s, repeat=%v)", h.CustomerID, h.Stack, h.Reason, roundAge(age), repeat) if _, err := hc.store.SaveEvent(h.CustomerID, eventHeldAppReminder, "warning", msg, string(details), "hub"); err != nil { hc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventHeldAppReminder, h.CustomerID, err) return } if hc.onEvent != nil { hc.onEvent(h.CustomerID, eventHeldAppReminder, "warning", msg, string(details), "hub") } } func roundAge(d time.Duration) string { if d >= 48*time.Hour { return fmt.Sprintf("%d days", int(d/(24*time.Hour))) } if d >= time.Hour { return d.Round(time.Hour).String() } return d.Round(time.Minute).String() }