felhom.eu/hub/internal/monitor/held_apps.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// ── D53 (`08` §10, R-1291) — A HELD APP REACHES THE OPERATOR ──────────────────────────────────────
//
// A held app is stopped on purpose and stays stopped until a person acts. demo-hp's Kimai sat held for 11 days
// (2026-10-14 → 10-26) and nobody was told after the first minute. The box now reports its holds (app_held /
// app_hold_cleared, severity info — nothing is mailed on receipt); this checker turns a hold that LASTS into ONE
// operator mail after a grace that follows the reason, then one a week while it lasts.
//
// THE GRACE (`08` §10): putback_pending 6 h — it should have cleared itself, so it is stuck; restore_failed and
// restore_mixed 1 day — only a person can act; update_failed and unhealthy_stop 3 days — the household can act first.
// An unknown reason gets 1 day. The clock is the hold's own time (the box's), else when the hub first heard of it.
//
// OPERATOR ONLY (D53): app_held_reminder is in the dispatcher's operatorOnlyEvents, so no household is ever mailed.
// THE HUB NEVER CLEARS A HOLD — a row goes only when the box sends app_hold_cleared.
//
// A ROW THE BOX STOPPED CONFIRMING is not reminded: the box repeats app_held daily, so a row older than heldAppStale
// belongs to a box that is down or gone, which its own staleness alarms report. The list still shows it.
//
// The reminder is recorded (MarkHeldAppReminded) BEFORE it is sent, so a hub restart cannot mail twice.
// Pinned by held_apps_test.go.
const (
	eventHeldAppReminder = "app_held_reminder"
	heldAppRepeat        = 7 * 24 * time.Hour
	heldAppStale         = 50 * time.Hour
	heldAppCheckEvery    = 15 * time.Minute
)

// HeldAppGrace returns the wait before the first reminder for a reason (`08` §10).
func HeldAppGrace(reason string) time.Duration {
	switch reason {
	case "putback_pending":
		return 6 * time.Hour
	case "update_failed", "unhealthy_stop":
		return 72 * time.Hour
	default: // restore_failed, restore_mixed, and a reason this hub does not know yet
		return 24 * time.Hour
	}
}

// HeldAppWhoClears says who may lift a hold, in the operator's words.
func HeldAppWhoClears(reason string) string {
	switch reason {
	case "update_failed":
		return "the household, by restoring the app from its backup"
	case "unhealthy_stop":
		return "the household's Start (one more try)"
	case "putback_pending":
		return "nobody — it clears itself when the drive with the pre-restore copy comes back"
	default:
		return "a person only (support: the controller's -clear-restore-hold)"
	}
}

type heldAppsStore interface {
	ListHeldApps(customerID string) ([]store.HeldApp, error)
	MarkHeldAppReminded(customerID, stack, reason string, since, at time.Time) error
	SaveEvent(customerID, eventType, severity, message, detailsJSON, source string) (int64, error)
}

// HeldAppsChecker sends the operator's reminders for held apps.
type HeldAppsChecker struct {
	store   heldAppsStore
	onEvent EventNotifyFunc
	logger  *log.Logger
	now     func() time.Time
	every   time.Duration
	// graceOverride, when > 0, replaces every reason's grace. A TEST configuration only (the live proof shortens
	// the wait); production leaves it 0. Logged at start when set.
	graceOverride time.Duration

	mu      sync.Mutex
	lastRun time.Time
}

// NewHeldAppsChecker builds the checker. graceOverride 0 = the per-reason grace.
func NewHeldAppsChecker(st heldAppsStore, graceOverride time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *HeldAppsChecker {
	if graceOverride > 0 && logger != nil {
		logger.Printf("[WARN] Held apps: the reminder grace is OVERRIDDEN to %s for every reason — a test configuration", graceOverride)
	}
	return &HeldAppsChecker{store: st, onEvent: onEvent, logger: logger, now: time.Now, every: heldAppCheckEvery, graceOverride: graceOverride}
}

func (hc *HeldAppsChecker) grace(reason string) time.Duration {
	if hc.graceOverride > 0 {
		return hc.graceOverride
	}
	return HeldAppGrace(reason)
}

// Check runs on the hub's 60 s sweep and throttles itself to one pass per heldAppCheckEvery.
func (hc *HeldAppsChecker) Check() {
	hc.mu.Lock()
	defer hc.mu.Unlock()
	now := hc.now()
	every := hc.every
	if hc.graceOverride > 0 && hc.graceOverride < every {
		every = hc.graceOverride // a test grace of minutes must not wait a 15-minute pass
	}
	if !hc.lastRun.IsZero() && now.Sub(hc.lastRun) < every {
		return
	}
	hc.lastRun = now
	held, err := hc.store.ListHeldApps("")
	if err != nil {
		hc.logger.Printf("[WARN] Held apps: list failed: %v — no reminder this pass", err)
		return
	}
	for _, h := range held {
		hc.checkOne(h, now)
	}
}

func (hc *HeldAppsChecker) checkOne(h store.HeldApp, now time.Time) {
	if now.Sub(h.LastSeen) > heldAppStale {
		hc.logger.Printf("[DEBUG] Held apps: %s/%s not confirmed since %s — no reminder (the box is not reporting)",
			h.CustomerID, h.Stack, h.LastSeen.Format(time.RFC3339))
		return
	}
	anchor := h.Since
	if anchor.IsZero() {
		anchor = h.FirstSeen
	}
	age := now.Sub(anchor)
	if age < hc.grace(h.Reason) {
		return
	}
	if !h.RemindedAt.IsZero() && now.Sub(h.RemindedAt) < heldAppRepeat {
		return
	}
	if err := hc.store.MarkHeldAppReminded(h.CustomerID, h.Stack, h.Reason, h.Since, now); err != nil {
		hc.logger.Printf("[WARN] Held apps: could not record the reminder for %s/%s (%v) — not sending, so a restart cannot mail twice",
			h.CustomerID, h.Stack, err)
		return
	}
	repeat := !h.RemindedAt.IsZero()
	msg := fmt.Sprintf("Customer %s: the app %s has been held for %s (%s): %s. Who may clear it: %s. The hub never clears a hold.",
		h.CustomerID, h.Stack, roundAge(age), h.Reason, h.Why, HeldAppWhoClears(h.Reason))
	if repeat {
		msg += " Weekly reminder: it is still held."
	}
	details, _ := json.Marshal(map[string]any{
		"customer_id": h.CustomerID, "stack_name": h.Stack, "reason": h.Reason,
		"since": anchor.UTC().Format(time.RFC3339), "why": h.Why, "repeat": repeat,
	})
	hc.logger.Printf("[INFO] Held apps: reminder for %s/%s (%s, held %s, repeat=%v)", h.CustomerID, h.Stack, h.Reason, roundAge(age), repeat)
	if _, err := hc.store.SaveEvent(h.CustomerID, eventHeldAppReminder, "warning", msg, string(details), "hub"); err != nil {
		hc.logger.Printf("[WARN] Failed to save %s for %s: %v", eventHeldAppReminder, h.CustomerID, err)
		return
	}
	if hc.onEvent != nil {
		hc.onEvent(h.CustomerID, eventHeldAppReminder, "warning", msg, string(details), "hub")
	}
}

func roundAge(d time.Duration) string {
	if d >= 48*time.Hour {
		return fmt.Sprintf("%d days", int(d/(24*time.Hour)))
	}
	if d >= time.Hour {
		return d.Round(time.Hour).String()
	}
	return d.Round(time.Minute).String()
}