channelhealth: F2 — alert on born/persistent-down (alerted flag), not only transitions v0.91.0

A channel broken at startup/reseed (e.g. controller boots into pin_mismatch) was dashboard-only,
no operator email ever. New 'alerted' flag drives alerting instead of prev=='': born-down
non-transient alerts cycle 1; transient still N>=2; healthy first-obs silent; recovery re-arms.
Red-proof + companion included.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
This commit is contained in:
2026-06-29 21:45:57 +02:00
parent 0f7ba7b665
commit b2ad871720
3 changed files with 109 additions and 24 deletions
+28 -14
View File
@@ -116,6 +116,9 @@ type Checker struct {
mu sync.Mutex
state string // "" (unseeded) | "up" | "down:<reason>"
consecutiveDown int
alerted bool // have we emitted a down alert for the CURRENT down-spell? (F2: drives
// alerting instead of `prev==""`, so a BORN-down — broken at startup/reseed — alerts too, not
// just a live up→down transition; re-armed on recovery / reason-change.)
}
// New builds a checker over the probe + sink seams.
@@ -140,20 +143,21 @@ func (c *Checker) Check(ctx context.Context) error {
c.sink.SetDashboard(false, "", "")
prev := c.state
c.state = "up"
if prev == "" || prev == "up" {
return nil // seed, or steady-up → no notify
c.alerted = false // re-arm for the next down-spell
if prev != "" && prev != "up" {
c.logger.Printf("[INFO] [channel] agent channel recovered (was %s)", prev)
c.sink.NotifyRecovered()
}
c.logger.Printf("[INFO] [channel] agent channel recovered (was %s)", prev)
c.sink.NotifyRecovered()
return nil
return nil // healthy first-obs / steady-up → no notify
}
// Channel DOWN — classify + debounce transient reasons.
cls := classify(constructionErr, perr)
c.consecutiveDown++
if cls.debounce && c.consecutiveDown < debounceThreshold {
// A transient blip (e.g. the ~1s agent-restart socket gap). Hold the previous state — do NOT
// flip the dashboard or notify. If we've never seen anything yet, assume up until confirmed.
// A transient blip (e.g. the ~1s agent-restart socket gap, or the agent not yet up on a cold
// boot). Hold the previous state — do NOT flip the dashboard or notify. Unseeded → assume up
// until confirmed (so a transient born-down still needs N>=2 before it alerts).
if c.state == "" {
c.state = "up"
}
@@ -162,22 +166,32 @@ func (c *Checker) Check(ctx context.Context) error {
return nil
}
// Confirmed down. F2: a NEW down-spell — coming from up/unseeded OR a reason change — re-arms the
// alert, so a BORN-down (broken at startup/reseed) alerts on cycle 1 for non-transient reasons,
// not only a live up->down transition. A steady down that already alerted does not re-fire.
newState := "down:" + string(cls.reason)
c.sink.SetDashboard(true, cls.reason, cls.hungarian) // dashboard reflects current state always
prev := c.state
if prev == "" || prev == "up" || prev != newState {
c.alerted = false
}
c.state = newState
if prev == "" {
c.logger.Printf("[INFO] [channel] agent channel down at startup (%s) — seeded, no alert: %v", cls.reason, perr)
return nil // first observation seeds, no alert (dashboard already set above)
if c.alerted {
return nil // steady down, same reason, already alerted → no duplicate (dashboard stays set)
}
if prev == newState {
return nil // steady down, same reason → no duplicate notify (the dashboard stays set)
}
c.logger.Printf("[WARN] [channel] agent channel DOWN (%s→%s): %v", prev, newState, perr)
c.logger.Printf("[WARN] [channel] agent channel DOWN (%s->%s): %v", orUnseeded(prev), newState, perr)
c.sink.NotifyDown(cls.reason, cls.eventType, cls.severity, cls.english)
c.alerted = true
return nil
}
func orUnseeded(s string) string {
if s == "" {
return "unseeded"
}
return s
}
// State returns the current channel state (for tests/diagnostics).
func (c *Checker) State() string {
c.mu.Lock()