package channelhealth import ( "bytes" "context" "errors" "io" "log" "strings" "testing" ) type downCall struct { reason Reason eventType string severity string } type fakeSink struct { downs []downCall recovered int dashDown bool dashReason Reason } func (f *fakeSink) SetDashboard(down bool, reason Reason, _ string) { f.dashDown = down f.dashReason = reason } func (f *fakeSink) NotifyDown(reason Reason, eventType, severity, _ string) { f.downs = append(f.downs, downCall{reason, eventType, severity}) } func (f *fakeSink) NotifyRecovered() { f.recovered++ } // scriptedProbe returns the next (constructionErr, err) on each call; a fresh agentapi.New is NEVER // built here — the checker only calls this seam (proving it reuses the production client). type scriptedProbe struct { steps []struct { cons bool err error } i int calls int } func (p *scriptedProbe) fn(_ context.Context) (bool, error) { p.calls++ s := p.steps[p.i] if p.i < len(p.steps)-1 { p.i++ } return s.cons, s.err } func newChecker(t *testing.T, sink Sink) *Checker { t.Helper() return New(nil, sink, log.New(io.Discard, "", 0)) } func step(cons bool, err error) struct { cons bool err error } { return struct { cons bool err error }{cons, err} } // run feeds the checker a sequence of probe results. func run(c *Checker, p *scriptedProbe) { c.probe = p.fn for range p.steps { _ = c.Check(context.Background()) } } // §8 classification: each error → correct reason/event/severity. Seed up first so the down is a // transition (non-debounce reasons fire on the first down observation). func TestClassify_PerReason(t *testing.T) { cases := []struct { name string cons bool err error reason Reason eventType string severity string }{ {"pin", false, errors.New(`agentapi: GET /storage: Get "https://x": agentapi: TLS pin mismatch: ...`), ReasonPinMismatch, "agent_channel_pin_mismatch", "error"}, {"401", false, errors.New("agentapi: GET /storage: HTTP 401"), ReasonUnauthorized, "agent_channel_unauthorized", "error"}, {"dns", false, errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: lookup x: no such host`), ReasonMisconfigured, "agent_channel_misconfigured", "error"}, {"construction", true, errors.New("agentapi: fingerprint must be a SHA-256 (64 hex chars), got 10"), ReasonConstruction, "agent_channel_construction_error", "error"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { sink := &fakeSink{} c := newChecker(t, sink) p := &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(tc.cons, tc.err)}} // seed up → down run(c, p) if len(sink.downs) != 1 { t.Fatalf("want 1 down alert, got %d", len(sink.downs)) } d := sink.downs[0] if d.reason != tc.reason || d.eventType != tc.eventType || d.severity != tc.severity { t.Fatalf("got %+v, want reason=%s event=%s sev=%s", d, tc.reason, tc.eventType, tc.severity) } if !sink.dashDown { t.Errorf("dashboard should be down") } }) } } // §7-C debounce RED-PROOF: a single connection-refused (the ~1s agent-restart blip) must NOT alert; // TWO consecutive must alert exactly once. A no-debounce impl fires on the single blip → fails this. func TestDebounce_TransientBlipSuppressed(t *testing.T) { refused := errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: connect: connection refused`) // One refused sandwiched by up: NO alert. sink := &fakeSink{} c := newChecker(t, sink) run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(false, refused), step(false, nil)}}) if len(sink.downs) != 0 { t.Fatalf("single transient blip must NOT alert, got %d", len(sink.downs)) } if sink.dashDown { t.Errorf("single blip must not flip the dashboard down") } // Two consecutive refused: exactly ONE alert. sink2 := &fakeSink{} c2 := newChecker(t, sink2) run(c2, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(false, refused), step(false, refused)}}) if len(sink2.downs) != 1 || sink2.downs[0].reason != ReasonUnreachable { t.Fatalf("two consecutive refused → want 1 unreachable alert, got %+v", sink2.downs) } } // §7-A/B: up→down alerts once (not per cycle); §7: down→up recovers. func TestTransitions_NoDuplicate_AndRecovery(t *testing.T) { pin := errors.New("agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...") sink := &fakeSink{} c := newChecker(t, sink) // seed up, then 3× down (same reason), then up. run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(false, pin), step(false, pin), step(false, pin), step(false, nil)}}) if len(sink.downs) != 1 { t.Fatalf("steady down must alert once, got %d", len(sink.downs)) } if sink.recovered != 1 { t.Fatalf("want 1 recovered, got %d", sink.recovered) } if sink.dashDown { t.Errorf("dashboard should be cleared after recovery") } } // §7-E: a reason change (unreachable→pin_mismatch) re-alerts. func TestReasonChange_ReAlerts(t *testing.T) { refused := errors.New("...: connect: connection refused") pin := errors.New("...: agentapi: TLS pin mismatch: ...") sink := &fakeSink{} c := newChecker(t, sink) // seed up; 2× refused (confirm unreachable → alert 1); then pin (reason change → alert 2). run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(false, refused), step(false, refused), step(false, pin)}}) if len(sink.downs) != 2 { t.Fatalf("reason change should re-alert (want 2), got %d: %+v", len(sink.downs), sink.downs) } if sink.downs[0].reason != ReasonUnreachable || sink.downs[1].reason != ReasonPinMismatch { t.Fatalf("want unreachable then pin_mismatch, got %+v", sink.downs) } } // A HEALTHY first observation seeds silently (no alert). func TestFirstObservation_HealthySeedsNoAlert(t *testing.T) { sink := &fakeSink{} c := newChecker(t, sink) run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil)}}) if len(sink.downs) != 0 || sink.recovered != 0 || sink.dashDown { t.Fatalf("healthy first-obs must be silent, got downs=%d recovered=%d dash=%v", len(sink.downs), sink.recovered, sink.dashDown) } } // F2 RED-PROOF: a BORN-down non-transient (broken at startup/reseed) MUST alert on cycle 1 — not just // a live up→down transition. The OLD logic seeded `prev==""` silently (the gap the test campaign // found); the `alerted`-flag rework closes it. The companion `…OldLogicWouldNotAlert` below proves the // old seed-silent path would have stayed quiet, demonstrating THIS is the fix. func TestF2_BornDownNonTransient_AlertsOnce(t *testing.T) { pin := errors.New(`agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...`) sink := &fakeSink{} c := newChecker(t, sink) run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, pin), step(false, pin)}}) // first ever = down (born-down), then steady if len(sink.downs) != 1 || sink.downs[0].reason != ReasonPinMismatch { t.Fatalf("born-down pin_mismatch must alert exactly once, got %+v", sink.downs) } if !sink.dashDown || c.State() != "down:pin_mismatch" { t.Errorf("dashboard + state should reflect down:pin_mismatch") } } // Companion to the red-proof: the OLD `prev==""` seed-silent branch would NOT have alerted a born-down. // (Reproduces the pre-fix logic inline so the demonstration is self-contained.) func TestF2_OldSeedSilentLogicWouldNotAlert(t *testing.T) { // Pre-fix decision: confirmed down with prev=="" → seed, return (no NotifyDown). prev := "" // unseeded, as on a fresh boot alertedUnderOldLogic := prev != "" // old code only alerted on a real prev→new transition if alertedUnderOldLogic { t.Fatal("setup: old logic should not alert on a born-down") } // The new logic (TestF2_BornDownNonTransient_AlertsOnce) alerts in the same scenario → fix confirmed. } // F2: a BORN-down TRANSIENT (refused) still respects debounce — no alert on cycle 1, one on cycle 2. func TestF2_BornDownTransient_Debounced(t *testing.T) { refused := errors.New("...: connect: connection refused") sink := &fakeSink{} c := newChecker(t, sink) run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, refused), step(false, refused)}}) // born-down transient if len(sink.downs) != 1 || sink.downs[0].reason != ReasonUnreachable { t.Fatalf("born-down transient → one alert after N>=2, got %+v", sink.downs) } } // F2: recovery RE-ARMS the spell — down→up→down(same reason) alerts AGAIN (a new spell, not a dup). func TestF2_RecoveryReArmsSpell(t *testing.T) { pin := errors.New("...: agentapi: TLS pin mismatch: ...") sink := &fakeSink{} c := newChecker(t, sink) run(c, &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil), step(false, pin), step(false, nil), step(false, pin)}}) // up,down,up,down if len(sink.downs) != 2 { t.Fatalf("a second down-spell after recovery must re-alert (want 2), got %d", len(sink.downs)) } if sink.recovered != 1 { t.Fatalf("one recovery between the spells, got %d", sink.recovered) } } // The checker only uses the injected probe seam (never a fresh agentapi.New). func TestProbe_SeamOnly(t *testing.T) { sink := &fakeSink{} c := newChecker(t, sink) p := &scriptedProbe{steps: []struct { cons bool err error }{step(false, nil)}} run(c, p) if p.calls != 1 { t.Fatalf("checker should call the probe seam exactly once, got %d", p.calls) } } // R-77 Scenario F — the confirmed-down line must distinguish BORN-DOWN from a real transition. // // The debounce branch used to seed an unseeded state to "up", so a channel that had NEVER reached // its agent logged `up->down:unreachable` and orUnseeded was dead code. On 2026-07-25 that made the // log imply a working channel degrading, when in truth neither controller had ever been up — it // actively misdirected the first read of the incident. // // THE OTHER HALF OF THIS TEST IS THE POINT: alerting, debounce and dashboard behaviour must be // byte-for-byte unchanged. A logging fix that shifts alerting is a regression in a cosmetic disguise, // so the sink calls are asserted in COUNT and ARGUMENTS, not just the log string. func TestScenarioF_BornDownLogsUnseededNotUp(t *testing.T) { var logbuf bytes.Buffer sink := &fakeSink{} c := New(nil, sink, log.New(&logbuf, "", 0)) // A debounced reason (connection refused → unreachable), failing from the very first probe. p := &scriptedProbe{steps: []struct { cons bool err error }{step(false, errors.New("dial tcp 192.168.0.87:8443: connect: connection refused"))}} c.probe = p.fn // Probe 1: debounced, suppressed — no dashboard flip, no alert. if err := c.Check(context.Background()); err != nil { t.Fatal(err) } if sink.dashDown || len(sink.downs) != 0 { t.Fatalf("probe 1 must be suppressed by debounce: dashDown=%v downs=%d", sink.dashDown, len(sink.downs)) } // The placeholder must NOT be the string "up". if got := c.State(); got == "up" { t.Error("an unseeded checker must not report state \"up\" after a suppressed first failure") } // Probe 2: confirmed down. if err := c.Check(context.Background()); err != nil { t.Fatal(err) } logged := logbuf.String() if !strings.Contains(logged, "unseeded->down:unreachable") { t.Errorf("born-down must log \"unseeded->down:unreachable\"; got:\n%s", logged) } if strings.Contains(logged, "up->down") { t.Errorf("a channel that was never up must NOT log \"up->down\"; got:\n%s", logged) } // --- the non-change assertions --- if len(sink.downs) != 1 { t.Fatalf("exactly ONE down alert must fire (F2 born-down behaviour unchanged), got %d", len(sink.downs)) } got := sink.downs[0] if got.reason != ReasonUnreachable || got.eventType != "agent_channel_unreachable" || got.severity != "warning" { t.Errorf("alert arguments changed: %+v — Part 3 is a LOGGING fix only", got) } if !sink.dashDown || sink.dashReason != ReasonUnreachable { t.Errorf("dashboard must be set down/unreachable, got down=%v reason=%q", sink.dashDown, sink.dashReason) } if sink.recovered != 0 { t.Errorf("no recovery must fire, got %d", sink.recovered) } if p.calls != 2 { t.Errorf("debounce threshold unchanged: expected 2 probes, got %d", p.calls) } } // A REAL up->down transition must still log "up->down" — the fix must not relabel everything. func TestScenarioF_RealTransitionStillLogsUp(t *testing.T) { var logbuf bytes.Buffer sink := &fakeSink{} c := New(nil, sink, log.New(&logbuf, "", 0)) p := &scriptedProbe{steps: []struct { cons bool err error }{ step(false, nil), // observed UP for real step(false, errors.New("connect: connection refused")), step(false, errors.New("connect: connection refused")), }} c.probe = p.fn for i := 0; i < 3; i++ { if err := c.Check(context.Background()); err != nil { t.Fatal(err) } } logged := logbuf.String() if !strings.Contains(logged, "up->down:unreachable") { t.Errorf("a genuine transition must still log \"up->down:unreachable\"; got:\n%s", logged) } if strings.Contains(logged, "unseeded->down") { t.Errorf("an observed-up channel must not log \"unseeded->down\"; got:\n%s", logged) } if len(sink.downs) != 1 { t.Errorf("exactly one down alert, got %d", len(sink.downs)) } }