admin/felhom.eu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hub: F2 — alert on host already degraded/stale at (re)start (seed only healthy) v0.21.0

Browse Source 
Constructors seed only healthy hosts; an already-degraded/stale host is left unseeded so the first
Check() emits once (cooldown dedups on hub restart). Born-degraded red-proof + staleness test updated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
This commit is contained in:
authentik Default Admin
2026-06-29 21:46:25 +02:00
parent a297639c58
commit 12f038618e
5 changed files with 79 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hub/internal/monitor/host_capability.go
+15 -5
View File
@@ -53,14 +53,17 @@ func NewHostCapabilityChecker(s *store.Store, onEvent EventNotifyFunc, logger *l
}
cc.customerOf[row.HostID] = row.CustomerID
st, _, _ := capabilityState(row.Capabilities)
cc.states[row.HostID] = st
// F2: seed only HEALTHY hosts. A host already degraded at (re)start is left UNSEEDED so the
// first Check() observes oldState=="" and emits once — otherwise a box already broken when the
// hub restarts would stay silent forever. The dispatcher's 1h cooldown dedups the re-emit.
if st == "degraded" {
degCount++
} else {
okCount++
continue
}
cc.states[row.HostID] = st
okCount++
}
logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded", okCount, degCount)
logger.Printf("[INFO] Host capability checker initialized: %d ok, %d degraded (degraded left unseeded → first Check emits)", okCount, degCount)
return cc
}
@@ -87,7 +90,14 @@ func (cc *HostCapabilityChecker) Check() {
newState, names, features := capabilityState(row.Capabilities)
oldState := cc.states[row.HostID]
if oldState == "" {
cc.states[row.HostID] = newState // first observation — no event
// F2: first observation (incl. a hub restart). A healthy first-obs seeds silently; a host
// already DEGRADED at first sight emits once (else a host that was already broken when the
// hub (re)started would never alert). The dispatcher's 1h operator cooldown dedups a re-emit
// across a hub restart.
cc.states[row.HostID] = newState
if newState == "degraded" {
cc.emitTransition(row.HostID, row.CustomerID, "unknown", newState, names, features)
}
continue
}
if oldState == newState {
hub/internal/monitor/host_capability_test.go
+25
View File
@@ -105,3 +105,28 @@ func TestHostCapabilityChecker_OldAgentNoCaps(t *testing.T) {
t.Fatalf("old agent (no caps) state = %s, want ok", cc.GetState("h1"))
}
}
// F2 RED-PROOF: a host already DEGRADED at hub (re)start is left UNSEEDED by the constructor and the
// FIRST Check emits once — so a box that was already broken when the hub restarted alerts (vs the
// pre-fix path which seeded it silently and never alerted). Companion: pre-fix the constructor seeded
// "degraded" and Check saw oldState==newState → no event.
func TestHostCapabilityChecker_F2_BornDegradedAlerts(t *testing.T) {
st := newCapStore(t)
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
var events []string
cc := NewHostCapabilityChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
if len(events) != 0 {
t.Fatalf("construction must not emit, got %v", events)
}
if cc.GetState("h1") != "unknown" {
t.Fatalf("born-degraded must be left unseeded, GetState = %s want unknown", cc.GetState("h1"))
}
cc.Check()
if len(events) != 1 || events[0] != "agent_capability_degraded" {
t.Fatalf("born-degraded first Check → one agent_capability_degraded, got %v", events)
}
cc.Check() // steady degraded → no duplicate
if len(events) != 1 {
t.Fatalf("steady degraded must not re-emit, got %v", events)
}
}
hub/internal/monitor/host_staleness.go
+11 -4
View File
@@ -56,19 +56,20 @@ func NewHostStalenessChecker(s *store.Store, threshold time.Duration, onEvent Ev
}
sc.customerOf[row.HostID] = row.CustomerID
age := time.Since(row.LastReportAt)
// F2: seed only HEALTHY hosts. A host already stale/down at (re)start is left UNSEEDED so the
// first Check() observes oldState=="" and emits once (a box that went silent while the hub was
// down would otherwise stay silent). The dispatcher's 1h cooldown dedups the re-emit.
switch {
case age > sc.downAfter:
sc.states[row.HostID] = "down"
downCount++
case age > sc.threshold:
sc.states[row.HostID] = "stale"
staleCount++
default:
sc.states[row.HostID] = "ok"
okCount++
}
}
logger.Printf("[INFO] Host staleness checker initialized: %d ok, %d stale, %d down", okCount, staleCount, downCount)
logger.Printf("[INFO] Host staleness checker initialized: %d ok, %d stale, %d down (stale/down left unseeded → first Check emits)", okCount, staleCount, downCount)
return sc
}
@@ -105,7 +106,13 @@ func (sc *HostStalenessChecker) Check() {
oldState := sc.states[row.HostID]
if oldState == "" {
sc.states[row.HostID] = newState // first observation — no event
// F2: first observation (incl. a hub restart). Healthy seeds silently; a host already
// stale/down at first sight emits once (else a box that went silent while the hub was
// down would never alert). The dispatcher's 1h cooldown dedups a re-emit on hub restart.
sc.states[row.HostID] = newState
if newState != "ok" {
sc.emitTransition(row.HostID, row.CustomerID, "unknown", newState, age)
}
continue
}
if oldState == newState {
hub/internal/monitor/host_staleness_test.go
+12 -7
View File
@@ -42,20 +42,25 @@ func TestHostStalenessChecker(t *testing.T) {
events = append(events, eventType)
}
// Seed already-stale (40m) → state stale, but NO event on init.
// F2: already-stale (40m) at construction → left UNSEEDED (not seeded "stale"), NO event on init.
backdate(t, db, "h1", 40)
sc := NewHostStalenessChecker(st, 30*time.Minute, onEvent, log.New(io.Discard, "", 0))
if len(events) != 0 {
t.Fatalf("seed must not emit events, got %v", events)
t.Fatalf("construction must not emit, got %v", events)
}
if sc.GetState("h1") != "stale" {
t.Fatalf("seeded state = %q, want stale", sc.GetState("h1"))
if sc.GetState("h1") != "unknown" {
t.Fatalf("born-stale must be left unseeded, GetState = %q want unknown", sc.GetState("h1"))
}
// Same age → no transition.
// F2: the FIRST Check on a born-stale host emits host_stale once (persistent-down alerting).
sc.Check()
if len(events) != 0 {
t.Fatalf("no transition expected, got %v", events)
if len(events) != 1 || last(events) != "host_stale" {
t.Fatalf("born-stale first Check → one host_stale, got %v", events)
}
// Steady stale → no duplicate.
sc.Check()
if len(events) != 1 {
t.Fatalf("steady stale must not re-emit, got %v", events)
}
// Fresh report → host_recovered.