channelhealth: controller->agent channel health-check (periodic probe + classified operator alert) v0.90.0
New internal/channelhealth Checker: ~60s probe via the PRODUCTION memoized client (Server.ProbeAgentChannel, GET /storage), classifies failures (spike Q1 map), debounces transient reasons (N>=2; construction error latches distinctly), seeds first obs, alerts operator+dashboard on transition. Notifier.NotifyAgentChannelDown/Recovered (English, operator-only), AlertManager dashboard banner (Hungarian). No agent/hub change. Spike-proven. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
package channelhealth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type downCall struct {
|
||||
reason Reason
|
||||
eventType string
|
||||
severity string
|
||||
}
|
||||
|
||||
type fakeSink struct {
|
||||
downs []downCall
|
||||
recovered int
|
||||
dashDown bool
|
||||
dashReason Reason
|
||||
}
|
||||
|
||||
func (f *fakeSink) SetDashboard(down bool, reason Reason, _ string) {
|
||||
f.dashDown = down
|
||||
f.dashReason = reason
|
||||
}
|
||||
func (f *fakeSink) NotifyDown(reason Reason, eventType, severity, _ string) {
|
||||
f.downs = append(f.downs, downCall{reason, eventType, severity})
|
||||
}
|
||||
func (f *fakeSink) NotifyRecovered() { f.recovered++ }
|
||||
|
||||
// scriptedProbe returns the next (constructionErr, err) on each call; a fresh agentapi.New is NEVER
|
||||
// built here — the checker only calls this seam (proving it reuses the production client).
|
||||
type scriptedProbe struct {
|
||||
steps []struct {
|
||||
cons bool
|
||||
err error
|
||||
}
|
||||
i int
|
||||
calls int
|
||||
}
|
||||
|
||||
func (p *scriptedProbe) fn(_ context.Context) (bool, error) {
|
||||
p.calls++
|
||||
s := p.steps[p.i]
|
||||
if p.i < len(p.steps)-1 {
|
||||
p.i++
|
||||
}
|
||||
return s.cons, s.err
|
||||
}
|
||||
|
||||
func newChecker(t *testing.T, sink Sink) *Checker {
|
||||
t.Helper()
|
||||
return New(nil, sink, log.New(io.Discard, "", 0))
|
||||
}
|
||||
|
||||
func step(cons bool, err error) struct {
|
||||
cons bool
|
||||
err error
|
||||
} {
|
||||
return struct {
|
||||
cons bool
|
||||
err error
|
||||
}{cons, err}
|
||||
}
|
||||
|
||||
// run feeds the checker a sequence of probe results.
|
||||
func run(c *Checker, p *scriptedProbe) {
|
||||
c.probe = p.fn
|
||||
for range p.steps {
|
||||
_ = c.Check(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
// §8 classification: each error → correct reason/event/severity. Seed up first so the down is a
|
||||
// transition (non-debounce reasons fire on the first down observation).
|
||||
func TestClassify_PerReason(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
cons bool
|
||||
err error
|
||||
reason Reason
|
||||
eventType string
|
||||
severity string
|
||||
}{
|
||||
{"pin", false, errors.New(`agentapi: GET /storage: Get "https://x": agentapi: TLS pin mismatch: ...`), ReasonPinMismatch, "agent_channel_pin_mismatch", "error"},
|
||||
{"401", false, errors.New("agentapi: GET /storage: HTTP 401"), ReasonUnauthorized, "agent_channel_unauthorized", "error"},
|
||||
{"dns", false, errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: lookup x: no such host`), ReasonMisconfigured, "agent_channel_misconfigured", "error"},
|
||||
{"construction", true, errors.New("agentapi: fingerprint must be a SHA-256 (64 hex chars), got 10"), ReasonConstruction, "agent_channel_construction_error", "error"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
p := &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil), step(tc.cons, tc.err)}} // seed up → down
|
||||
run(c, p)
|
||||
if len(sink.downs) != 1 {
|
||||
t.Fatalf("want 1 down alert, got %d", len(sink.downs))
|
||||
}
|
||||
d := sink.downs[0]
|
||||
if d.reason != tc.reason || d.eventType != tc.eventType || d.severity != tc.severity {
|
||||
t.Fatalf("got %+v, want reason=%s event=%s sev=%s", d, tc.reason, tc.eventType, tc.severity)
|
||||
}
|
||||
if !sink.dashDown {
|
||||
t.Errorf("dashboard should be down")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// §7-C debounce RED-PROOF: a single connection-refused (the ~1s agent-restart blip) must NOT alert;
|
||||
// TWO consecutive must alert exactly once. A no-debounce impl fires on the single blip → fails this.
|
||||
func TestDebounce_TransientBlipSuppressed(t *testing.T) {
|
||||
refused := errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: connect: connection refused`)
|
||||
|
||||
// One refused sandwiched by up: NO alert.
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
run(c, &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil), step(false, refused), step(false, nil)}})
|
||||
if len(sink.downs) != 0 {
|
||||
t.Fatalf("single transient blip must NOT alert, got %d", len(sink.downs))
|
||||
}
|
||||
if sink.dashDown {
|
||||
t.Errorf("single blip must not flip the dashboard down")
|
||||
}
|
||||
|
||||
// Two consecutive refused: exactly ONE alert.
|
||||
sink2 := &fakeSink{}
|
||||
c2 := newChecker(t, sink2)
|
||||
run(c2, &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil), step(false, refused), step(false, refused)}})
|
||||
if len(sink2.downs) != 1 || sink2.downs[0].reason != ReasonUnreachable {
|
||||
t.Fatalf("two consecutive refused → want 1 unreachable alert, got %+v", sink2.downs)
|
||||
}
|
||||
}
|
||||
|
||||
// §7-A/B: up→down alerts once (not per cycle); §7: down→up recovers.
|
||||
func TestTransitions_NoDuplicate_AndRecovery(t *testing.T) {
|
||||
pin := errors.New("agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...")
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
// seed up, then 3× down (same reason), then up.
|
||||
run(c, &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil), step(false, pin), step(false, pin), step(false, pin), step(false, nil)}})
|
||||
if len(sink.downs) != 1 {
|
||||
t.Fatalf("steady down must alert once, got %d", len(sink.downs))
|
||||
}
|
||||
if sink.recovered != 1 {
|
||||
t.Fatalf("want 1 recovered, got %d", sink.recovered)
|
||||
}
|
||||
if sink.dashDown {
|
||||
t.Errorf("dashboard should be cleared after recovery")
|
||||
}
|
||||
}
|
||||
|
||||
// §7-E: a reason change (unreachable→pin_mismatch) re-alerts.
|
||||
func TestReasonChange_ReAlerts(t *testing.T) {
|
||||
refused := errors.New("...: connect: connection refused")
|
||||
pin := errors.New("...: agentapi: TLS pin mismatch: ...")
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
// seed up; 2× refused (confirm unreachable → alert 1); then pin (reason change → alert 2).
|
||||
run(c, &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil), step(false, refused), step(false, refused), step(false, pin)}})
|
||||
if len(sink.downs) != 2 {
|
||||
t.Fatalf("reason change should re-alert (want 2), got %d: %+v", len(sink.downs), sink.downs)
|
||||
}
|
||||
if sink.downs[0].reason != ReasonUnreachable || sink.downs[1].reason != ReasonPinMismatch {
|
||||
t.Fatalf("want unreachable then pin_mismatch, got %+v", sink.downs)
|
||||
}
|
||||
}
|
||||
|
||||
// §9.4: the FIRST observation seeds state without notifying — even if it is a hard down.
|
||||
func TestFirstObservation_SeedsNoAlert(t *testing.T) {
|
||||
pin := errors.New("...: agentapi: TLS pin mismatch: ...")
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
run(c, &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, pin)}}) // first ever = down
|
||||
if len(sink.downs) != 0 {
|
||||
t.Fatalf("first observation must not notify, got %d", len(sink.downs))
|
||||
}
|
||||
if !sink.dashDown {
|
||||
t.Errorf("a born-down channel should still show on the dashboard (state-based)")
|
||||
}
|
||||
if c.State() != "down:pin_mismatch" {
|
||||
t.Errorf("state = %s, want down:pin_mismatch", c.State())
|
||||
}
|
||||
}
|
||||
|
||||
// The checker only uses the injected probe seam (never a fresh agentapi.New).
|
||||
func TestProbe_SeamOnly(t *testing.T) {
|
||||
sink := &fakeSink{}
|
||||
c := newChecker(t, sink)
|
||||
p := &scriptedProbe{steps: []struct {
|
||||
cons bool
|
||||
err error
|
||||
}{step(false, nil)}}
|
||||
run(c, p)
|
||||
if p.calls != 1 {
|
||||
t.Fatalf("checker should call the probe seam exactly once, got %d", p.calls)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user