package monitor import ( "encoding/json" "log" "sort" "strconv" "strings" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // HostCapabilityChecker is the operator-facing alert for the agent's privileged-capability // self-check (felhom-agent v0.44.0). It is a deliberate SIBLING of HostStalenessChecker: same // state-transition shape (per-host ok/degraded map, seed-without-event, emit only on change), same // dispatcher/event plumbing — but it keys on the capability snapshot the agent rides in its host // report (whether each required `sudo -n` grant is usable) rather than report recency. // // A host is "degraded" iff its latest report has any CRITICAL capability with status "degraded" // (a missing user-facing grant — the multi-drive-flapping class). Non-critical degradations ride // the report + the agent's own logs but do NOT alert the operator (avoid noise). The event is // attributed to the host's customer, so the existing operator notification UX picks it up // unchanged. Customer is NEVER notified — internal capability health is operator-only: both event // types sit in notify.operatorOnlyEvents (R-1150, hub v0.159.0; before that only configuration kept // them from a household). Pinned by notify/r1150_operator_only_test.go. // // R-1150 (hub v0.159.0) — ONE capability is not a grant. The agent's `localapi:backup-cadence-cap` // (felhom-agent cmd/felhom-agent/main.go backupCapStatusName, R-973) is CRITICAL and "degraded" for // 24 h after the agent REFUSED a whole-guest backup a guest asked for beyond its schedule. Reported // as „agent privileged capability degraded" it read as a broken sudo grant. It is therefore a SECOND, // independent state per host with its own event pair: // // - the grants (every other critical capability): ok↔degraded → agent_capability_degraded / // agent_capability_recovered, exactly as before, the cadence cap never named in them; // - the backup refusal (the cadence cap alone): ok→degraded → whole_guest_backup_refused (warning), // degraded→ok → whole_guest_backup_refusal_cleared (info: recorded, never mailed). // // Two states rather than one verdict with a special case, because the two can change on the SAME // report independently (a refusal while a grant is missing, a grant restored while the refusal // stands): each fires on its own edge, so one report flip is at most one event per state — never // two events for one state, and never a "grants restored" line when only the refusal window ended. // Pinned by host_capability_test.go TestR1150_*. // // R-1157 (hub v0.160.0) — ONE EPISODE, COUNTED. The agent holds the cap degraded for 24 h after its NEWEST refusal, // so a guest refused every day reads as one long degraded state. The agent's reason names the newest refusal's // time, so a CHANGED reason while degraded is a further refusal: it adds to the episode's count, and the alarm // repeats (whole_guest_backup_refused again, the count and the first refusal in its text and details) at most // once per refusalRepeatEvery. The cleared record carries the episode's total. Pinned by TestR1157_*. type HostCapabilityChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex states map[string]string // hostID → "ok" | "degraded" (the grants) refusals map[string]string // hostID → "ok" | "degraded" (the backup cadence cap, R-1150) episodes map[string]*refusalEpisode customerOf map[string]string // hostID → customerID now func() time.Time } // refusalEpisode is one degraded stretch of the cadence cap on a host (R-1157). type refusalEpisode struct { firstReason, lastReason string count int lastAlarm time.Time } // refusalRepeatEvery bounds the repeat of whole_guest_backup_refused inside one episode (R-1157). const refusalRepeatEvery = 24 * time.Hour // backupCadenceCapName is the agent's R-973 cadence-cap status name. It must equal felhom-agent's // backupCapStatusName (cmd/felhom-agent/main.go) — the agent's status name stays (R-1150). const backupCadenceCapName = "localapi:backup-cadence-cap" // R-1150 event types — hub-origin ONLY (deliberately not in api.allowedEventTypes: the guest's // controller is the party this alarm suspects, and a box able to POST the type could fake it or, // by spending the operator cooldown first, drown the real one) and operator-only. const ( EventWholeGuestBackupRefused = "whole_guest_backup_refused" EventWholeGuestBackupRefusalCleared = "whole_guest_backup_refusal_cleared" ) // NewHostCapabilityChecker creates the checker and seeds state from the current snapshots. No // events are generated during initialization (mirrors the staleness checker). func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostCapabilityChecker { cc := &HostCapabilityChecker{ store: s, logger: logger, onEvent: onEvent, states: make(map[string]string), refusals: make(map[string]string), episodes: make(map[string]*refusalEpisode), customerOf: make(map[string]string), now: time.Now, } rows, err := s.GetHostCapabilities() if err != nil { logger.Printf("[WARN] Host capability checker: failed to seed states: %v", err) return cc } var okCount, degCount, refusedCount int for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) { continue } cc.customerOf[row.HostID] = row.CustomerID st, _, _ := capabilityState(row.Capabilities) // F2: seed only HEALTHY hosts. A host already degraded at (re)start is left UNSEEDED so the // first Check() observes oldState=="" and emits once — otherwise a box already broken when the // hub restarts would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit. // The same rule, separately, for the backup refusal (R-1150). if st == "degraded" { degCount++ } else { cc.states[row.HostID] = st okCount++ } if rst, _ := backupRefusalState(row.Capabilities); rst == "degraded" { refusedCount++ } else { cc.refusals[row.HostID] = rst } } logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded, %d with a refused whole-guest backup (degraded/refused left unseeded → first Check emits)", okCount, degCount, refusedCount) return cc } // Check evaluates all hosts and emits an operator event on each ok↔degraded transition. Call on the // same sweep as the staleness checker (every 60s). func (cc *HostCapabilityChecker) Check() { rows, err := cc.store.GetHostCapabilities() if err != nil { cc.logger.Printf("[WARN] Host capability check failed: %v", err) return } cc.mu.Lock() defer cc.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { seen[row.HostID] = true if cc.store.IsCustomerBlocked(row.CustomerID) { delete(cc.states, row.HostID) delete(cc.refusals, row.HostID) delete(cc.episodes, row.HostID) continue } cc.customerOf[row.HostID] = row.CustomerID newState, names, features := capabilityState(row.Capabilities) if old, changed := step(cc.states, row.HostID, newState); changed { cc.emitTransition(row.HostID, row.CustomerID, old, newState, names, features) } refState, reason := backupRefusalState(row.Capabilities) old, changed := step(cc.refusals, row.HostID, refState) ep := cc.episodes[row.HostID] switch { case changed && refState == "degraded": ep = &refusalEpisode{firstReason: reason, lastReason: reason, count: 1, lastAlarm: cc.now()} cc.episodes[row.HostID] = ep cc.emitRefusal(row.HostID, row.CustomerID, old, refState, reason, ep) case changed: cc.emitRefusal(row.HostID, row.CustomerID, old, refState, reason, ep) delete(cc.episodes, row.HostID) case refState == "degraded" && ep != nil && reason != ep.lastReason: // A further refusal inside the episode (R-1157): counted always, alarmed at most once a day. ep.count++ ep.lastReason = reason if cc.now().Sub(ep.lastAlarm) >= refusalRepeatEvery { ep.lastAlarm = cc.now() cc.emitRefusal(row.HostID, row.CustomerID, refState, refState, reason, ep) } else { cc.logger.Printf("[INFO] Host capability: %s refused another whole-guest backup (%d in this episode); the alarm repeats at most once a day", row.HostID, ep.count) } } } for _, m := range []map[string]string{cc.states, cc.refusals} { for id := range m { if !seen[id] { delete(m, id) } } } for id := range cc.episodes { if !seen[id] { delete(cc.episodes, id) } } } // step records newState for hostID in m and reports whether an event is due, with the state it left // ("unknown" on a first observation). F2: a first observation (incl. a hub restart) seeds a healthy // state silently, and a state already DEGRADED at first sight is due once (else a host that was already // broken when the hub (re)started would never alert). The dispatcher's 1h operator cooldown dedups a // re-emit across a hub restart. A steady state is never due. func step(m map[string]string, hostID, newState string) (oldState string, due bool) { oldState = m[hostID] m[hostID] = newState if oldState == "" { return "unknown", newState == "degraded" } return oldState, oldState != newState } // GetState returns the current capability state for a host ("unknown" if unseen). func (cc *HostCapabilityChecker) GetState(hostID string) string { cc.mu.Lock() defer cc.mu.Unlock() s := cc.states[hostID] if s == "" { return "unknown" } return s } // GetBackupRefusalState returns the host's whole-guest backup refusal state (R-1150): "ok", "degraded" // (the agent refused one inside its window), or "unknown" if unseen. func (cc *HostCapabilityChecker) GetBackupRefusalState(hostID string) string { cc.mu.Lock() defer cc.mu.Unlock() s := cc.refusals[hostID] if s == "" { return "unknown" } return s } // capabilityState reduces a snapshot to "ok"/"degraded" plus the degraded CRITICAL capability names // + features (for the event detail). An empty/absent snapshot (pre-v0.44.0 agent) is "ok" — an old // agent can't trip a false alert. The backup cadence cap is NOT a grant and is left to // backupRefusalState (R-1150). func capabilityState(caps []store.CapabilityStatus) (state string, names, features []string) { for _, c := range caps { if c.Name == backupCadenceCapName { continue } if c.Critical && c.Status == "degraded" { names = append(names, c.Name) features = append(features, c.Feature) } } sort.Strings(names) if len(names) > 0 { return "degraded", names, features } return "ok", nil, nil } func (cc *HostCapabilityChecker) emitTransition(hostID, customerID, oldState, newState string, names, features []string) { var eventType, severity, message string switch { case newState == "degraded": eventType = "agent_capability_degraded" severity = "warning" message = "Host " + hostID + ": agent privileged capability degraded — " + strings.Join(names, ", ") + " (impairs: " + strings.Join(dedup(features), "; ") + ")" case newState == "ok" && oldState == "degraded": eventType = "agent_capability_recovered" severity = "info" message = "Host " + hostID + ": agent privileged capabilities recovered (all required grants restored)" default: return } details, _ := json.Marshal(map[string]any{ "host_id": hostID, "degraded_capabilities": names, "gated_features": dedup(features), }) cc.logger.Printf("[INFO] Host capability: %s %s → %s (%s)", hostID, oldState, newState, eventType) if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { cc.logger.Printf("[WARN] Failed to save host capability event for %s: %v", hostID, err) return } if cc.onEvent != nil { cc.onEvent(customerID, eventType, severity, message, string(details), "hub") } } // backupRefusalState reads the agent's cadence-cap status (R-1150): "degraded" with the agent's own // reason while it reports a refused whole-guest backup, else "ok" (absent on an agent without the cap). // The agent always reports the cap as critical; Critical is required here as it is for every alarm. func backupRefusalState(caps []store.CapabilityStatus) (state, reason string) { for _, c := range caps { if c.Name == backupCadenceCapName && c.Critical && c.Status == "degraded" { return "degraded", c.Reason } } return "ok", "" } // emitRefusal raises the operator's alarm for a refused whole-guest backup (R-1150) — the agent turned // down a backup the guest asked for beyond its schedule — and records its end. func (cc *HostCapabilityChecker) emitRefusal(hostID, customerID, oldState, newState, reason string, ep *refusalEpisode) { var eventType, severity, message string switch { case newState == "degraded" && oldState == "degraded" && ep != nil: eventType = EventWholeGuestBackupRefused severity = "warning" message = "Host " + hostID + ": whole-guest backups are still being refused — " + strconv.Itoa(ep.count) + " refusals since the first one in this episode. The guest keeps asking for more whole-guest backups than " + "its schedule allows. Its controller may be misbehaving or compromised: check the guest." case newState == "degraded": eventType = EventWholeGuestBackupRefused severity = "warning" message = "Host " + hostID + ": a whole-guest backup was refused — the guest asked for more whole-guest " + "backups than its schedule allows. Its controller may be misbehaving or compromised: check the guest." case newState == "ok" && oldState == "degraded": eventType = EventWholeGuestBackupRefusalCleared severity = "info" message = "Host " + hostID + ": the agent no longer reports a refused whole-guest backup (its refusal window has ended)" default: return } d := map[string]any{ "host_id": hostID, "capability": backupCadenceCapName, "agent_reason": reason, } if ep != nil { d["refusals_in_episode"] = ep.count d["first_refusal"] = ep.firstReason } details, _ := json.Marshal(d) cc.logger.Printf("[INFO] Host capability: %s backup refusal %s → %s (%s)", hostID, oldState, newState, eventType) if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { cc.logger.Printf("[WARN] Failed to save backup refusal event for %s: %v", hostID, err) return } if cc.onEvent != nil { cc.onEvent(customerID, eventType, severity, message, string(details), "hub") } } func dedup(in []string) []string { seen := make(map[string]bool, len(in)) var out []string for _, s := range in { if !seen[s] { seen[s] = true out = append(out, s) } } return out }