felhom.eu/hub/internal/monitor/host_capability.go

package monitor

import (
	"encoding/json"
	"log"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// HostCapabilityChecker is the operator-facing alert for the agent's privileged-capability
// self-check (felhom-agent v0.44.0). It is a deliberate SIBLING of HostStalenessChecker: same
// state-transition shape (per-host ok/degraded map, seed-without-event, emit only on change), same
// dispatcher/event plumbing — but it keys on the capability snapshot the agent rides in its host
// report (whether each required `sudo -n` grant is usable) rather than report recency.
//
// A host is "degraded" iff its latest report has any CRITICAL capability with status "degraded"
// (a missing user-facing grant — the multi-drive-flapping class). Non-critical degradations ride
// the report + the agent's own logs but do NOT alert the operator (avoid noise). The event is
// attributed to the host's customer, so the existing operator notification UX picks it up
// unchanged. Customer is NEVER notified — internal capability health is operator-only: both event
// types sit in notify.operatorOnlyEvents (R-1150, hub v0.159.0; before that only configuration kept
// them from a household). Pinned by notify/r1150_operator_only_test.go.
//
// R-1150 (hub v0.159.0) — ONE capability is not a grant. The agent's `localapi:backup-cadence-cap`
// (felhom-agent cmd/felhom-agent/main.go backupCapStatusName, R-973) is CRITICAL and "degraded" for
// 24 h after the agent REFUSED a whole-guest backup a guest asked for beyond its schedule. Reported
// as „agent privileged capability degraded" it read as a broken sudo grant. It is therefore a SECOND,
// independent state per host with its own event pair:
//
//   - the grants (every other critical capability): ok↔degraded → agent_capability_degraded /
//     agent_capability_recovered, exactly as before, the cadence cap never named in them;
//   - the backup refusal (the cadence cap alone): ok→degraded → whole_guest_backup_refused (warning),
//     degraded→ok → whole_guest_backup_refusal_cleared (info: recorded, never mailed).
//
// Two states rather than one verdict with a special case, because the two can change on the SAME
// report independently (a refusal while a grant is missing, a grant restored while the refusal
// stands): each fires on its own edge, so one report flip is at most one event per state — never
// two events for one state, and never a "grants restored" line when only the refusal window ended.
// Pinned by host_capability_test.go TestR1150_*.
//
// R-1157 (hub v0.160.0) — ONE EPISODE, COUNTED. The agent holds the cap degraded for 24 h after its NEWEST refusal,
// so a guest refused every day reads as one long degraded state. The agent's reason names the newest refusal's
// time, so a CHANGED reason while degraded is a further refusal: it adds to the episode's count, and the alarm
// repeats (whole_guest_backup_refused again, the count and the first refusal in its text and details) at most
// once per refusalRepeatEvery. The cleared record carries the episode's total. Pinned by TestR1157_*.
type HostCapabilityChecker struct {
	store   *store.Store
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu         sync.Mutex
	states     map[string]string // hostID → "ok" | "degraded" (the grants)
	refusals   map[string]string // hostID → "ok" | "degraded" (the backup cadence cap, R-1150)
	episodes   map[string]*refusalEpisode
	customerOf map[string]string // hostID → customerID
	now        func() time.Time
}

// refusalEpisode is one degraded stretch of the cadence cap on a host (R-1157).
type refusalEpisode struct {
	firstReason, lastReason string
	count                   int
	lastAlarm               time.Time
}

// refusalRepeatEvery bounds the repeat of whole_guest_backup_refused inside one episode (R-1157).
const refusalRepeatEvery = 24 * time.Hour

// backupCadenceCapName is the agent's R-973 cadence-cap status name. It must equal felhom-agent's
// backupCapStatusName (cmd/felhom-agent/main.go) — the agent's status name stays (R-1150).
const backupCadenceCapName = "localapi:backup-cadence-cap"

// R-1150 event types — hub-origin ONLY (deliberately not in api.allowedEventTypes: the guest's
// controller is the party this alarm suspects, and a box able to POST the type could fake it or,
// by spending the operator cooldown first, drown the real one) and operator-only.
const (
	EventWholeGuestBackupRefused        = "whole_guest_backup_refused"
	EventWholeGuestBackupRefusalCleared = "whole_guest_backup_refusal_cleared"
)

// NewHostCapabilityChecker creates the checker and seeds state from the current snapshots. No
// events are generated during initialization (mirrors the staleness checker).
func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostCapabilityChecker {
	cc := &HostCapabilityChecker{
		store:      s,
		logger:     logger,
		onEvent:    onEvent,
		states:     make(map[string]string),
		refusals:   make(map[string]string),
		episodes:   make(map[string]*refusalEpisode),
		customerOf: make(map[string]string),
		now:        time.Now,
	}
	rows, err := s.GetHostCapabilities()
	if err != nil {
		logger.Printf("[WARN] Host capability checker: failed to seed states: %v", err)
		return cc
	}
	var okCount, degCount, refusedCount int
	for _, row := range rows {
		if s.IsCustomerBlocked(row.CustomerID) {
			continue
		}
		cc.customerOf[row.HostID] = row.CustomerID
		st, _, _ := capabilityState(row.Capabilities)
		// F2: seed only HEALTHY hosts. A host already degraded at (re)start is left UNSEEDED so the
		// first Check() observes oldState=="" and emits once — otherwise a box already broken when the
		// hub restarts would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit.
		// The same rule, separately, for the backup refusal (R-1150).
		if st == "degraded" {
			degCount++
		} else {
			cc.states[row.HostID] = st
			okCount++
		}
		if rst, _ := backupRefusalState(row.Capabilities); rst == "degraded" {
			refusedCount++
		} else {
			cc.refusals[row.HostID] = rst
		}
	}
	logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded, %d with a refused whole-guest backup (degraded/refused left unseeded → first Check emits)", okCount, degCount, refusedCount)
	return cc
}

// Check evaluates all hosts and emits an operator event on each ok↔degraded transition. Call on the
// same sweep as the staleness checker (every 60s).
func (cc *HostCapabilityChecker) Check() {
	rows, err := cc.store.GetHostCapabilities()
	if err != nil {
		cc.logger.Printf("[WARN] Host capability check failed: %v", err)
		return
	}
	cc.mu.Lock()
	defer cc.mu.Unlock()

	seen := make(map[string]bool, len(rows))
	for _, row := range rows {
		seen[row.HostID] = true
		if cc.store.IsCustomerBlocked(row.CustomerID) {
			delete(cc.states, row.HostID)
			delete(cc.refusals, row.HostID)
			delete(cc.episodes, row.HostID)
			continue
		}
		cc.customerOf[row.HostID] = row.CustomerID

		newState, names, features := capabilityState(row.Capabilities)
		if old, changed := step(cc.states, row.HostID, newState); changed {
			cc.emitTransition(row.HostID, row.CustomerID, old, newState, names, features)
		}
		refState, reason := backupRefusalState(row.Capabilities)
		old, changed := step(cc.refusals, row.HostID, refState)
		ep := cc.episodes[row.HostID]
		switch {
		case changed && refState == "degraded":
			ep = &refusalEpisode{firstReason: reason, lastReason: reason, count: 1, lastAlarm: cc.now()}
			cc.episodes[row.HostID] = ep
			cc.emitRefusal(row.HostID, row.CustomerID, old, refState, reason, ep)
		case changed:
			cc.emitRefusal(row.HostID, row.CustomerID, old, refState, reason, ep)
			delete(cc.episodes, row.HostID)
		case refState == "degraded" && ep != nil && reason != ep.lastReason:
			// A further refusal inside the episode (R-1157): counted always, alarmed at most once a day.
			ep.count++
			ep.lastReason = reason
			if cc.now().Sub(ep.lastAlarm) >= refusalRepeatEvery {
				ep.lastAlarm = cc.now()
				cc.emitRefusal(row.HostID, row.CustomerID, refState, refState, reason, ep)
			} else {
				cc.logger.Printf("[INFO] Host capability: %s refused another whole-guest backup (%d in this episode); the alarm repeats at most once a day", row.HostID, ep.count)
			}
		}
	}

	for _, m := range []map[string]string{cc.states, cc.refusals} {
		for id := range m {
			if !seen[id] {
				delete(m, id)
			}
		}
	}
	for id := range cc.episodes {
		if !seen[id] {
			delete(cc.episodes, id)
		}
	}
}

// step records newState for hostID in m and reports whether an event is due, with the state it left
// ("unknown" on a first observation). F2: a first observation (incl. a hub restart) seeds a healthy
// state silently, and a state already DEGRADED at first sight is due once (else a host that was already
// broken when the hub (re)started would never alert). The dispatcher's 1h operator cooldown dedups a
// re-emit across a hub restart. A steady state is never due.
func step(m map[string]string, hostID, newState string) (oldState string, due bool) {
	oldState = m[hostID]
	m[hostID] = newState
	if oldState == "" {
		return "unknown", newState == "degraded"
	}
	return oldState, oldState != newState
}

// GetState returns the current capability state for a host ("unknown" if unseen).
func (cc *HostCapabilityChecker) GetState(hostID string) string {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	s := cc.states[hostID]
	if s == "" {
		return "unknown"
	}
	return s
}

// GetBackupRefusalState returns the host's whole-guest backup refusal state (R-1150): "ok", "degraded"
// (the agent refused one inside its window), or "unknown" if unseen.
func (cc *HostCapabilityChecker) GetBackupRefusalState(hostID string) string {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	s := cc.refusals[hostID]
	if s == "" {
		return "unknown"
	}
	return s
}

// capabilityState reduces a snapshot to "ok"/"degraded" plus the degraded CRITICAL capability names
// + features (for the event detail). An empty/absent snapshot (pre-v0.44.0 agent) is "ok" — an old
// agent can't trip a false alert. The backup cadence cap is NOT a grant and is left to
// backupRefusalState (R-1150).
func capabilityState(caps []store.CapabilityStatus) (state string, names, features []string) {
	for _, c := range caps {
		if c.Name == backupCadenceCapName {
			continue
		}
		if c.Critical && c.Status == "degraded" {
			names = append(names, c.Name)
			features = append(features, c.Feature)
		}
	}
	sort.Strings(names)
	if len(names) > 0 {
		return "degraded", names, features
	}
	return "ok", nil, nil
}

func (cc *HostCapabilityChecker) emitTransition(hostID, customerID, oldState, newState string, names, features []string) {
	var eventType, severity, message string
	switch {
	case newState == "degraded":
		eventType = "agent_capability_degraded"
		severity = "warning"
		message = "Host " + hostID + ": agent privileged capability degraded — " +
			strings.Join(names, ", ") + " (impairs: " + strings.Join(dedup(features), "; ") + ")"
	case newState == "ok" && oldState == "degraded":
		eventType = "agent_capability_recovered"
		severity = "info"
		message = "Host " + hostID + ": agent privileged capabilities recovered (all required grants restored)"
	default:
		return
	}

	details, _ := json.Marshal(map[string]any{
		"host_id":               hostID,
		"degraded_capabilities": names,
		"gated_features":        dedup(features),
	})

	cc.logger.Printf("[INFO] Host capability: %s %s → %s (%s)", hostID, oldState, newState, eventType)

	if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
		cc.logger.Printf("[WARN] Failed to save host capability event for %s: %v", hostID, err)
		return
	}
	if cc.onEvent != nil {
		cc.onEvent(customerID, eventType, severity, message, string(details), "hub")
	}
}

// backupRefusalState reads the agent's cadence-cap status (R-1150): "degraded" with the agent's own
// reason while it reports a refused whole-guest backup, else "ok" (absent on an agent without the cap).
// The agent always reports the cap as critical; Critical is required here as it is for every alarm.
func backupRefusalState(caps []store.CapabilityStatus) (state, reason string) {
	for _, c := range caps {
		if c.Name == backupCadenceCapName && c.Critical && c.Status == "degraded" {
			return "degraded", c.Reason
		}
	}
	return "ok", ""
}

// emitRefusal raises the operator's alarm for a refused whole-guest backup (R-1150) — the agent turned
// down a backup the guest asked for beyond its schedule — and records its end.
func (cc *HostCapabilityChecker) emitRefusal(hostID, customerID, oldState, newState, reason string, ep *refusalEpisode) {
	var eventType, severity, message string
	switch {
	case newState == "degraded" && oldState == "degraded" && ep != nil:
		eventType = EventWholeGuestBackupRefused
		severity = "warning"
		message = "Host " + hostID + ": whole-guest backups are still being refused — " + strconv.Itoa(ep.count) +
			" refusals since the first one in this episode. The guest keeps asking for more whole-guest backups than " +
			"its schedule allows. Its controller may be misbehaving or compromised: check the guest."
	case newState == "degraded":
		eventType = EventWholeGuestBackupRefused
		severity = "warning"
		message = "Host " + hostID + ": a whole-guest backup was refused — the guest asked for more whole-guest " +
			"backups than its schedule allows. Its controller may be misbehaving or compromised: check the guest."
	case newState == "ok" && oldState == "degraded":
		eventType = EventWholeGuestBackupRefusalCleared
		severity = "info"
		message = "Host " + hostID + ": the agent no longer reports a refused whole-guest backup (its refusal window has ended)"
	default:
		return
	}
	d := map[string]any{
		"host_id":      hostID,
		"capability":   backupCadenceCapName,
		"agent_reason": reason,
	}
	if ep != nil {
		d["refusals_in_episode"] = ep.count
		d["first_refusal"] = ep.firstReason
	}
	details, _ := json.Marshal(d)
	cc.logger.Printf("[INFO] Host capability: %s backup refusal %s → %s (%s)", hostID, oldState, newState, eventType)
	if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
		cc.logger.Printf("[WARN] Failed to save backup refusal event for %s: %v", hostID, err)
		return
	}
	if cc.onEvent != nil {
		cc.onEvent(customerID, eventType, severity, message, string(details), "hub")
	}
}

func dedup(in []string) []string {
	seen := make(map[string]bool, len(in))
	var out []string
	for _, s := range in {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	return out
}