Files
felhom-controller/controller/internal/channelhealth/checker_test.go
T
admin a277b18981 channelhealth: controller->agent channel health-check (periodic probe + classified operator alert) v0.90.0
New internal/channelhealth Checker: ~60s probe via the PRODUCTION memoized client
(Server.ProbeAgentChannel, GET /storage), classifies failures (spike Q1 map), debounces transient
reasons (N>=2; construction error latches distinctly), seeds first obs, alerts operator+dashboard on
transition. Notifier.NotifyAgentChannelDown/Recovered (English, operator-only), AlertManager dashboard
banner (Hungarian). No agent/hub change. Spike-proven.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt
2026-06-29 20:28:02 +02:00

219 lines
6.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package channelhealth
import (
"context"
"errors"
"io"
"log"
"testing"
)
type downCall struct {
reason Reason
eventType string
severity string
}
type fakeSink struct {
downs []downCall
recovered int
dashDown bool
dashReason Reason
}
func (f *fakeSink) SetDashboard(down bool, reason Reason, _ string) {
f.dashDown = down
f.dashReason = reason
}
func (f *fakeSink) NotifyDown(reason Reason, eventType, severity, _ string) {
f.downs = append(f.downs, downCall{reason, eventType, severity})
}
func (f *fakeSink) NotifyRecovered() { f.recovered++ }
// scriptedProbe returns the next (constructionErr, err) on each call; a fresh agentapi.New is NEVER
// built here — the checker only calls this seam (proving it reuses the production client).
type scriptedProbe struct {
steps []struct {
cons bool
err error
}
i int
calls int
}
func (p *scriptedProbe) fn(_ context.Context) (bool, error) {
p.calls++
s := p.steps[p.i]
if p.i < len(p.steps)-1 {
p.i++
}
return s.cons, s.err
}
func newChecker(t *testing.T, sink Sink) *Checker {
t.Helper()
return New(nil, sink, log.New(io.Discard, "", 0))
}
func step(cons bool, err error) struct {
cons bool
err error
} {
return struct {
cons bool
err error
}{cons, err}
}
// run feeds the checker a sequence of probe results.
func run(c *Checker, p *scriptedProbe) {
c.probe = p.fn
for range p.steps {
_ = c.Check(context.Background())
}
}
// §8 classification: each error → correct reason/event/severity. Seed up first so the down is a
// transition (non-debounce reasons fire on the first down observation).
func TestClassify_PerReason(t *testing.T) {
cases := []struct {
name string
cons bool
err error
reason Reason
eventType string
severity string
}{
{"pin", false, errors.New(`agentapi: GET /storage: Get "https://x": agentapi: TLS pin mismatch: ...`), ReasonPinMismatch, "agent_channel_pin_mismatch", "error"},
{"401", false, errors.New("agentapi: GET /storage: HTTP 401"), ReasonUnauthorized, "agent_channel_unauthorized", "error"},
{"dns", false, errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: lookup x: no such host`), ReasonMisconfigured, "agent_channel_misconfigured", "error"},
{"construction", true, errors.New("agentapi: fingerprint must be a SHA-256 (64 hex chars), got 10"), ReasonConstruction, "agent_channel_construction_error", "error"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
sink := &fakeSink{}
c := newChecker(t, sink)
p := &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil), step(tc.cons, tc.err)}} // seed up → down
run(c, p)
if len(sink.downs) != 1 {
t.Fatalf("want 1 down alert, got %d", len(sink.downs))
}
d := sink.downs[0]
if d.reason != tc.reason || d.eventType != tc.eventType || d.severity != tc.severity {
t.Fatalf("got %+v, want reason=%s event=%s sev=%s", d, tc.reason, tc.eventType, tc.severity)
}
if !sink.dashDown {
t.Errorf("dashboard should be down")
}
})
}
}
// §7-C debounce RED-PROOF: a single connection-refused (the ~1s agent-restart blip) must NOT alert;
// TWO consecutive must alert exactly once. A no-debounce impl fires on the single blip → fails this.
func TestDebounce_TransientBlipSuppressed(t *testing.T) {
refused := errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: connect: connection refused`)
// One refused sandwiched by up: NO alert.
sink := &fakeSink{}
c := newChecker(t, sink)
run(c, &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil), step(false, refused), step(false, nil)}})
if len(sink.downs) != 0 {
t.Fatalf("single transient blip must NOT alert, got %d", len(sink.downs))
}
if sink.dashDown {
t.Errorf("single blip must not flip the dashboard down")
}
// Two consecutive refused: exactly ONE alert.
sink2 := &fakeSink{}
c2 := newChecker(t, sink2)
run(c2, &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil), step(false, refused), step(false, refused)}})
if len(sink2.downs) != 1 || sink2.downs[0].reason != ReasonUnreachable {
t.Fatalf("two consecutive refused → want 1 unreachable alert, got %+v", sink2.downs)
}
}
// §7-A/B: up→down alerts once (not per cycle); §7: down→up recovers.
func TestTransitions_NoDuplicate_AndRecovery(t *testing.T) {
pin := errors.New("agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...")
sink := &fakeSink{}
c := newChecker(t, sink)
// seed up, then 3× down (same reason), then up.
run(c, &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil), step(false, pin), step(false, pin), step(false, pin), step(false, nil)}})
if len(sink.downs) != 1 {
t.Fatalf("steady down must alert once, got %d", len(sink.downs))
}
if sink.recovered != 1 {
t.Fatalf("want 1 recovered, got %d", sink.recovered)
}
if sink.dashDown {
t.Errorf("dashboard should be cleared after recovery")
}
}
// §7-E: a reason change (unreachable→pin_mismatch) re-alerts.
func TestReasonChange_ReAlerts(t *testing.T) {
refused := errors.New("...: connect: connection refused")
pin := errors.New("...: agentapi: TLS pin mismatch: ...")
sink := &fakeSink{}
c := newChecker(t, sink)
// seed up; 2× refused (confirm unreachable → alert 1); then pin (reason change → alert 2).
run(c, &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil), step(false, refused), step(false, refused), step(false, pin)}})
if len(sink.downs) != 2 {
t.Fatalf("reason change should re-alert (want 2), got %d: %+v", len(sink.downs), sink.downs)
}
if sink.downs[0].reason != ReasonUnreachable || sink.downs[1].reason != ReasonPinMismatch {
t.Fatalf("want unreachable then pin_mismatch, got %+v", sink.downs)
}
}
// §9.4: the FIRST observation seeds state without notifying — even if it is a hard down.
func TestFirstObservation_SeedsNoAlert(t *testing.T) {
pin := errors.New("...: agentapi: TLS pin mismatch: ...")
sink := &fakeSink{}
c := newChecker(t, sink)
run(c, &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, pin)}}) // first ever = down
if len(sink.downs) != 0 {
t.Fatalf("first observation must not notify, got %d", len(sink.downs))
}
if !sink.dashDown {
t.Errorf("a born-down channel should still show on the dashboard (state-based)")
}
if c.State() != "down:pin_mismatch" {
t.Errorf("state = %s, want down:pin_mismatch", c.State())
}
}
// The checker only uses the injected probe seam (never a fresh agentapi.New).
func TestProbe_SeamOnly(t *testing.T) {
sink := &fakeSink{}
c := newChecker(t, sink)
p := &scriptedProbe{steps: []struct {
cons bool
err error
}{step(false, nil)}}
run(c, p)
if p.calls != 1 {
t.Fatalf("checker should call the probe seam exactly once, got %d", p.calls)
}
}