hub v0.159.0: a refused whole-guest backup has its own alarm (R-1150); a reinstall re-issues the claim code and the off-site login when the old host is stale or off (R-1154)
R-1150: the agent's localapi:backup-cadence-cap is a separate per-host state in HostCapabilityChecker: alone it raises whole_guest_backup_refused (warning, plain text, agent reason in details) and whole_guest_backup_refusal_cleared (info); other critical caps keep agent_capability_degraded/_recovered unchanged. New types operator-only and hub-origin only (not in allowedEventTypes: the suspected controller must not post them). agent_capability_degraded/_recovered join operatorOnlyEvents (the checker's "never the customer" claim was configuration only). R-1154: handleHostEnroll's reuse path re-issues (claim code + off-site login) when no controller wait is open and either deliberate_off_at is held or both the host report and the controller report are older than alerting.stale_threshold (wired from main); once per silence (host_reenroll_reissued record, also written at mint); a live box gets nothing re-issued and the skip is recorded (host_reenroll_not_reissued). 05 section 8 [DESIGN]. Tests red-proved; go build/vet/test green.
This commit is contained in:
admin committed
1 parent
e4f022a0d9
commit
5121536f10
14 files changed
+995
-33
No files matched your search
@@ -20,17 +20,49 @@ import (
|
||||
// (a missing user-facing grant — the multi-drive-flapping class). Non-critical degradations ride
|
||||
// the report + the agent's own logs but do NOT alert the operator (avoid noise). The event is
|
||||
// attributed to the host's customer, so the existing operator notification UX picks it up
|
||||
// unchanged. Customer is NEVER notified — internal capability health is operator-only.
|
||||
// unchanged. Customer is NEVER notified — internal capability health is operator-only: both event
|
||||
// types sit in notify.operatorOnlyEvents (R-1150, hub v0.159.0; before that only configuration kept
|
||||
// them from a household). Pinned by notify/r1150_operator_only_test.go.
|
||||
//
|
||||
// R-1150 (hub v0.159.0) — ONE capability is not a grant. The agent's `localapi:backup-cadence-cap`
|
||||
// (felhom-agent cmd/felhom-agent/main.go backupCapStatusName, R-973) is CRITICAL and "degraded" for
|
||||
// 24 h after the agent REFUSED a whole-guest backup a guest asked for beyond its schedule. Reported
|
||||
// as „agent privileged capability degraded" it read as a broken sudo grant. It is therefore a SECOND,
|
||||
// independent state per host with its own event pair:
|
||||
//
|
||||
// - the grants (every other critical capability): ok↔degraded → agent_capability_degraded /
|
||||
// agent_capability_recovered, exactly as before, the cadence cap never named in them;
|
||||
// - the backup refusal (the cadence cap alone): ok→degraded → whole_guest_backup_refused (warning),
|
||||
// degraded→ok → whole_guest_backup_refusal_cleared (info: recorded, never mailed).
|
||||
//
|
||||
// Two states rather than one verdict with a special case, because the two can change on the SAME
|
||||
// report independently (a refusal while a grant is missing, a grant restored while the refusal
|
||||
// stands): each fires on its own edge, so one report flip is at most one event per state — never
|
||||
// two events for one state, and never a "grants restored" line when only the refusal window ended.
|
||||
// Pinned by host_capability_test.go TestR1150_*.
|
||||
type HostCapabilityChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
onEvent EventNotifyFunc
|
||||
|
||||
mu sync.Mutex
|
||||
states map[string]string // hostID → "ok" | "degraded"
|
||||
states map[string]string // hostID → "ok" | "degraded" (the grants)
|
||||
refusals map[string]string // hostID → "ok" | "degraded" (the backup cadence cap, R-1150)
|
||||
customerOf map[string]string // hostID → customerID
|
||||
}
|
||||
|
||||
// backupCadenceCapName is the agent's R-973 cadence-cap status name. It must equal felhom-agent's
|
||||
// backupCapStatusName (cmd/felhom-agent/main.go) — the agent's status name stays (R-1150).
|
||||
const backupCadenceCapName = "localapi:backup-cadence-cap"
|
||||
|
||||
// R-1150 event types — hub-origin ONLY (deliberately not in api.allowedEventTypes: the guest's
|
||||
// controller is the party this alarm suspects, and a box able to POST the type could fake it or,
|
||||
// by spending the operator cooldown first, drown the real one) and operator-only.
|
||||
const (
|
||||
EventWholeGuestBackupRefused = "whole_guest_backup_refused"
|
||||
EventWholeGuestBackupRefusalCleared = "whole_guest_backup_refusal_cleared"
|
||||
)
|
||||
|
||||
// NewHostCapabilityChecker creates the checker and seeds state from the current snapshots. No
|
||||
// events are generated during initialization (mirrors the staleness checker).
|
||||
func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostCapabilityChecker {
|
||||
@@ -39,6 +71,7 @@ func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *l
|
||||
logger: logger,
|
||||
onEvent: onEvent,
|
||||
states: make(map[string]string),
|
||||
refusals: make(map[string]string),
|
||||
customerOf: make(map[string]string),
|
||||
}
|
||||
rows, err := s.GetHostCapabilities()
|
||||
@@ -46,7 +79,7 @@ func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *l
|
||||
logger.Printf("[WARN] Host capability checker: failed to seed states: %v", err)
|
||||
return cc
|
||||
}
|
||||
var okCount, degCount int
|
||||
var okCount, degCount, refusedCount int
|
||||
for _, row := range rows {
|
||||
if s.IsCustomerBlocked(row.CustomerID) {
|
||||
continue
|
||||
@@ -56,14 +89,20 @@ func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *l
|
||||
// F2: seed only HEALTHY hosts. A host already degraded at (re)start is left UNSEEDED so the
|
||||
// first Check() observes oldState=="" and emits once — otherwise a box already broken when the
|
||||
// hub restarts would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit.
|
||||
// The same rule, separately, for the backup refusal (R-1150).
|
||||
if st == "degraded" {
|
||||
degCount++
|
||||
continue
|
||||
} else {
|
||||
cc.states[row.HostID] = st
|
||||
okCount++
|
||||
}
|
||||
if rst, _ := backupRefusalState(row.Capabilities); rst == "degraded" {
|
||||
refusedCount++
|
||||
} else {
|
||||
cc.refusals[row.HostID] = rst
|
||||
}
|
||||
cc.states[row.HostID] = st
|
||||
okCount++
|
||||
}
|
||||
logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded (degraded left unseeded → first Check emits)", okCount, degCount)
|
||||
logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded, %d with a refused whole-guest backup (degraded/refused left unseeded → first Check emits)", okCount, degCount, refusedCount)
|
||||
return cc
|
||||
}
|
||||
|
||||
@@ -83,37 +122,44 @@ func (cc *HostCapabilityChecker) Check() {
|
||||
seen[row.HostID] = true
|
||||
if cc.store.IsCustomerBlocked(row.CustomerID) {
|
||||
delete(cc.states, row.HostID)
|
||||
delete(cc.refusals, row.HostID)
|
||||
continue
|
||||
}
|
||||
cc.customerOf[row.HostID] = row.CustomerID
|
||||
|
||||
newState, names, features := capabilityState(row.Capabilities)
|
||||
oldState := cc.states[row.HostID]
|
||||
if oldState == "" {
|
||||
// F2: first observation (incl. a hub restart). A healthy first-obs seeds silently; a host
|
||||
// already DEGRADED at first sight emits once (else a host that was already broken when the
|
||||
// hub (re)started would never alert). The dispatcher's 1h operator cooldown dedups a re-emit
|
||||
// across a hub restart.
|
||||
cc.states[row.HostID] = newState
|
||||
if newState == "degraded" {
|
||||
cc.emitTransition(row.HostID, row.CustomerID, "unknown", newState, names, features)
|
||||
}
|
||||
continue
|
||||
if old, changed := step(cc.states, row.HostID, newState); changed {
|
||||
cc.emitTransition(row.HostID, row.CustomerID, old, newState, names, features)
|
||||
}
|
||||
if oldState == newState {
|
||||
continue
|
||||
refState, reason := backupRefusalState(row.Capabilities)
|
||||
if old, changed := step(cc.refusals, row.HostID, refState); changed {
|
||||
cc.emitRefusal(row.HostID, row.CustomerID, old, refState, reason)
|
||||
}
|
||||
cc.states[row.HostID] = newState
|
||||
cc.emitTransition(row.HostID, row.CustomerID, oldState, newState, names, features)
|
||||
}
|
||||
|
||||
for id := range cc.states {
|
||||
if !seen[id] {
|
||||
delete(cc.states, id)
|
||||
for _, m := range []map[string]string{cc.states, cc.refusals} {
|
||||
for id := range m {
|
||||
if !seen[id] {
|
||||
delete(m, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// step records newState for hostID in m and reports whether an event is due, with the state it left
|
||||
// ("unknown" on a first observation). F2: a first observation (incl. a hub restart) seeds a healthy
|
||||
// state silently, and a state already DEGRADED at first sight is due once (else a host that was already
|
||||
// broken when the hub (re)started would never alert). The dispatcher's 1h operator cooldown dedups a
|
||||
// re-emit across a hub restart. A steady state is never due.
|
||||
func step(m map[string]string, hostID, newState string) (oldState string, due bool) {
|
||||
oldState = m[hostID]
|
||||
m[hostID] = newState
|
||||
if oldState == "" {
|
||||
return "unknown", newState == "degraded"
|
||||
}
|
||||
return oldState, oldState != newState
|
||||
}
|
||||
|
||||
// GetState returns the current capability state for a host ("unknown" if unseen).
|
||||
func (cc *HostCapabilityChecker) GetState(hostID string) string {
|
||||
cc.mu.Lock()
|
||||
@@ -125,11 +171,27 @@ func (cc *HostCapabilityChecker) GetState(hostID string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// GetBackupRefusalState returns the host's whole-guest backup refusal state (R-1150): "ok", "degraded"
|
||||
// (the agent refused one inside its window), or "unknown" if unseen.
|
||||
func (cc *HostCapabilityChecker) GetBackupRefusalState(hostID string) string {
|
||||
cc.mu.Lock()
|
||||
defer cc.mu.Unlock()
|
||||
s := cc.refusals[hostID]
|
||||
if s == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// capabilityState reduces a snapshot to "ok"/"degraded" plus the degraded CRITICAL capability names
|
||||
// + features (for the event detail). An empty/absent snapshot (pre-v0.44.0 agent) is "ok" — an old
|
||||
// agent can't trip a false alert.
|
||||
// agent can't trip a false alert. The backup cadence cap is NOT a grant and is left to
|
||||
// backupRefusalState (R-1150).
|
||||
func capabilityState(caps []store.CapabilityStatus) (state string, names, features []string) {
|
||||
for _, c := range caps {
|
||||
if c.Name == backupCadenceCapName {
|
||||
continue
|
||||
}
|
||||
if c.Critical && c.Status == "degraded" {
|
||||
names = append(names, c.Name)
|
||||
features = append(features, c.Feature)
|
||||
@@ -175,6 +237,50 @@ func (cc *HostCapabilityChecker) emitTransition(hostID, customerID, oldState, ne
|
||||
}
|
||||
}
|
||||
|
||||
// backupRefusalState reads the agent's cadence-cap status (R-1150): "degraded" with the agent's own
|
||||
// reason while it reports a refused whole-guest backup, else "ok" (absent on an agent without the cap).
|
||||
// The agent always reports the cap as critical; Critical is required here as it is for every alarm.
|
||||
func backupRefusalState(caps []store.CapabilityStatus) (state, reason string) {
|
||||
for _, c := range caps {
|
||||
if c.Name == backupCadenceCapName && c.Critical && c.Status == "degraded" {
|
||||
return "degraded", c.Reason
|
||||
}
|
||||
}
|
||||
return "ok", ""
|
||||
}
|
||||
|
||||
// emitRefusal raises the operator's alarm for a refused whole-guest backup (R-1150) — the agent turned
|
||||
// down a backup the guest asked for beyond its schedule — and records its end.
|
||||
func (cc *HostCapabilityChecker) emitRefusal(hostID, customerID, oldState, newState, reason string) {
|
||||
var eventType, severity, message string
|
||||
switch {
|
||||
case newState == "degraded":
|
||||
eventType = EventWholeGuestBackupRefused
|
||||
severity = "warning"
|
||||
message = "Host " + hostID + ": a whole-guest backup was refused — the guest asked for more whole-guest " +
|
||||
"backups than its schedule allows. Its controller may be misbehaving or compromised: check the guest."
|
||||
case newState == "ok" && oldState == "degraded":
|
||||
eventType = EventWholeGuestBackupRefusalCleared
|
||||
severity = "info"
|
||||
message = "Host " + hostID + ": the agent no longer reports a refused whole-guest backup (its refusal window has ended)"
|
||||
default:
|
||||
return
|
||||
}
|
||||
details, _ := json.Marshal(map[string]any{
|
||||
"host_id": hostID,
|
||||
"capability": backupCadenceCapName,
|
||||
"agent_reason": reason,
|
||||
})
|
||||
cc.logger.Printf("[INFO] Host capability: %s backup refusal %s → %s (%s)", hostID, oldState, newState, eventType)
|
||||
if _, err := cc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
|
||||
cc.logger.Printf("[WARN] Failed to save backup refusal event for %s: %v", hostID, err)
|
||||
return
|
||||
}
|
||||
if cc.onEvent != nil {
|
||||
cc.onEvent(customerID, eventType, severity, message, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
func dedup(in []string) []string {
|
||||
seen := make(map[string]bool, len(in))
|
||||
var out []string
|
||||
|
||||
Reference in new issue
Block a user