felhom-controller/controller/internal/channelhealth/checker_test.go

package channelhealth

import (
	"bytes"
	"context"
	"errors"
	"io"
	"log"
	"strings"
	"testing"
)

type downCall struct {
	reason    Reason
	eventType string
	severity  string
}

type fakeSink struct {
	downs      []downCall
	recovered  int
	dashDown   bool
	dashReason Reason
}

func (f *fakeSink) SetDashboard(down bool, reason Reason, _ string) {
	f.dashDown = down
	f.dashReason = reason
}
func (f *fakeSink) NotifyDown(reason Reason, eventType, severity, _ string) {
	f.downs = append(f.downs, downCall{reason, eventType, severity})
}
func (f *fakeSink) NotifyRecovered() { f.recovered++ }

// scriptedProbe returns the next (constructionErr, err) on each call; a fresh agentapi.New is NEVER
// built here — the checker only calls this seam (proving it reuses the production client).
type scriptedProbe struct {
	steps []struct {
		cons bool
		err  error
	}
	i     int
	calls int
}

func (p *scriptedProbe) fn(_ context.Context) (bool, error) {
	p.calls++
	s := p.steps[p.i]
	if p.i < len(p.steps)-1 {
		p.i++
	}
	return s.cons, s.err
}

func newChecker(t *testing.T, sink Sink) *Checker {
	t.Helper()
	return New(nil, sink, log.New(io.Discard, "", 0))
}

func step(cons bool, err error) struct {
	cons bool
	err  error
} {
	return struct {
		cons bool
		err  error
	}{cons, err}
}

// run feeds the checker a sequence of probe results.
func run(c *Checker, p *scriptedProbe) {
	c.probe = p.fn
	for range p.steps {
		_ = c.Check(context.Background())
	}
}

// §8 classification: each error → correct reason/event/severity. Seed up first so the down is a
// transition (non-debounce reasons fire on the first down observation).
func TestClassify_PerReason(t *testing.T) {
	cases := []struct {
		name      string
		cons      bool
		err       error
		reason    Reason
		eventType string
		severity  string
	}{
		{"pin", false, errors.New(`agentapi: GET /storage: Get "https://x": agentapi: TLS pin mismatch: ...`), ReasonPinMismatch, "agent_channel_pin_mismatch", "error"},
		{"401", false, errors.New("agentapi: GET /storage: HTTP 401"), ReasonUnauthorized, "agent_channel_unauthorized", "error"},
		{"dns", false, errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: lookup x: no such host`), ReasonMisconfigured, "agent_channel_misconfigured", "error"},
		{"construction", true, errors.New("agentapi: fingerprint must be a SHA-256 (64 hex chars), got 10"), ReasonConstruction, "agent_channel_construction_error", "error"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			sink := &fakeSink{}
			c := newChecker(t, sink)
			p := &scriptedProbe{steps: []struct {
				cons bool
				err  error
			}{step(false, nil), step(tc.cons, tc.err)}} // seed up → down
			run(c, p)
			if len(sink.downs) != 1 {
				t.Fatalf("want 1 down alert, got %d", len(sink.downs))
			}
			d := sink.downs[0]
			if d.reason != tc.reason || d.eventType != tc.eventType || d.severity != tc.severity {
				t.Fatalf("got %+v, want reason=%s event=%s sev=%s", d, tc.reason, tc.eventType, tc.severity)
			}
			if !sink.dashDown {
				t.Errorf("dashboard should be down")
			}
		})
	}
}

// §7-C debounce RED-PROOF: a single connection-refused (the ~1s agent-restart blip) must NOT alert;
// TWO consecutive must alert exactly once. A no-debounce impl fires on the single blip → fails this.
func TestDebounce_TransientBlipSuppressed(t *testing.T) {
	refused := errors.New(`agentapi: GET /storage: Get "https://x": dial tcp: connect: connection refused`)

	// One refused sandwiched by up: NO alert.
	sink := &fakeSink{}
	c := newChecker(t, sink)
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil), step(false, refused), step(false, nil)}})
	if len(sink.downs) != 0 {
		t.Fatalf("single transient blip must NOT alert, got %d", len(sink.downs))
	}
	if sink.dashDown {
		t.Errorf("single blip must not flip the dashboard down")
	}

	// Two consecutive refused: exactly ONE alert.
	sink2 := &fakeSink{}
	c2 := newChecker(t, sink2)
	run(c2, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil), step(false, refused), step(false, refused)}})
	if len(sink2.downs) != 1 || sink2.downs[0].reason != ReasonUnreachable {
		t.Fatalf("two consecutive refused → want 1 unreachable alert, got %+v", sink2.downs)
	}
}

// §7-A/B: up→down alerts once (not per cycle); §7: down→up recovers.
func TestTransitions_NoDuplicate_AndRecovery(t *testing.T) {
	pin := errors.New("agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...")
	sink := &fakeSink{}
	c := newChecker(t, sink)
	// seed up, then 3× down (same reason), then up.
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil), step(false, pin), step(false, pin), step(false, pin), step(false, nil)}})
	if len(sink.downs) != 1 {
		t.Fatalf("steady down must alert once, got %d", len(sink.downs))
	}
	if sink.recovered != 1 {
		t.Fatalf("want 1 recovered, got %d", sink.recovered)
	}
	if sink.dashDown {
		t.Errorf("dashboard should be cleared after recovery")
	}
}

// §7-E: a reason change (unreachable→pin_mismatch) re-alerts.
func TestReasonChange_ReAlerts(t *testing.T) {
	refused := errors.New("...: connect: connection refused")
	pin := errors.New("...: agentapi: TLS pin mismatch: ...")
	sink := &fakeSink{}
	c := newChecker(t, sink)
	// seed up; 2× refused (confirm unreachable → alert 1); then pin (reason change → alert 2).
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil), step(false, refused), step(false, refused), step(false, pin)}})
	if len(sink.downs) != 2 {
		t.Fatalf("reason change should re-alert (want 2), got %d: %+v", len(sink.downs), sink.downs)
	}
	if sink.downs[0].reason != ReasonUnreachable || sink.downs[1].reason != ReasonPinMismatch {
		t.Fatalf("want unreachable then pin_mismatch, got %+v", sink.downs)
	}
}

// A HEALTHY first observation seeds silently (no alert).
func TestFirstObservation_HealthySeedsNoAlert(t *testing.T) {
	sink := &fakeSink{}
	c := newChecker(t, sink)
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil)}})
	if len(sink.downs) != 0 || sink.recovered != 0 || sink.dashDown {
		t.Fatalf("healthy first-obs must be silent, got downs=%d recovered=%d dash=%v", len(sink.downs), sink.recovered, sink.dashDown)
	}
}

// F2 RED-PROOF: a BORN-down non-transient (broken at startup/reseed) MUST alert on cycle 1 — not just
// a live up→down transition. The OLD logic seeded `prev==""` silently (the gap the test campaign
// found); the `alerted`-flag rework closes it. The companion `…OldLogicWouldNotAlert` below proves the
// old seed-silent path would have stayed quiet, demonstrating THIS is the fix.
func TestF2_BornDownNonTransient_AlertsOnce(t *testing.T) {
	pin := errors.New(`agentapi: GET /storage: ...: agentapi: TLS pin mismatch: ...`)
	sink := &fakeSink{}
	c := newChecker(t, sink)
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, pin), step(false, pin)}}) // first ever = down (born-down), then steady
	if len(sink.downs) != 1 || sink.downs[0].reason != ReasonPinMismatch {
		t.Fatalf("born-down pin_mismatch must alert exactly once, got %+v", sink.downs)
	}
	if !sink.dashDown || c.State() != "down:pin_mismatch" {
		t.Errorf("dashboard + state should reflect down:pin_mismatch")
	}
}

// Companion to the red-proof: the OLD `prev==""` seed-silent branch would NOT have alerted a born-down.
// (Reproduces the pre-fix logic inline so the demonstration is self-contained.)
func TestF2_OldSeedSilentLogicWouldNotAlert(t *testing.T) {
	// Pre-fix decision: confirmed down with prev=="" → seed, return (no NotifyDown).
	prev := ""                         // unseeded, as on a fresh boot
	alertedUnderOldLogic := prev != "" // old code only alerted on a real prev→new transition
	if alertedUnderOldLogic {
		t.Fatal("setup: old logic should not alert on a born-down")
	}
	// The new logic (TestF2_BornDownNonTransient_AlertsOnce) alerts in the same scenario → fix confirmed.
}

// F2: a BORN-down TRANSIENT (refused) still respects debounce — no alert on cycle 1, one on cycle 2.
func TestF2_BornDownTransient_Debounced(t *testing.T) {
	refused := errors.New("...: connect: connection refused")
	sink := &fakeSink{}
	c := newChecker(t, sink)
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, refused), step(false, refused)}}) // born-down transient
	if len(sink.downs) != 1 || sink.downs[0].reason != ReasonUnreachable {
		t.Fatalf("born-down transient → one alert after N>=2, got %+v", sink.downs)
	}
}

// F2: recovery RE-ARMS the spell — down→up→down(same reason) alerts AGAIN (a new spell, not a dup).
func TestF2_RecoveryReArmsSpell(t *testing.T) {
	pin := errors.New("...: agentapi: TLS pin mismatch: ...")
	sink := &fakeSink{}
	c := newChecker(t, sink)
	run(c, &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil), step(false, pin), step(false, nil), step(false, pin)}}) // up,down,up,down
	if len(sink.downs) != 2 {
		t.Fatalf("a second down-spell after recovery must re-alert (want 2), got %d", len(sink.downs))
	}
	if sink.recovered != 1 {
		t.Fatalf("one recovery between the spells, got %d", sink.recovered)
	}
}

// The checker only uses the injected probe seam (never a fresh agentapi.New).
func TestProbe_SeamOnly(t *testing.T) {
	sink := &fakeSink{}
	c := newChecker(t, sink)
	p := &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, nil)}}
	run(c, p)
	if p.calls != 1 {
		t.Fatalf("checker should call the probe seam exactly once, got %d", p.calls)
	}
}

// R-77 Scenario F — the confirmed-down line must distinguish BORN-DOWN from a real transition.
//
// The debounce branch used to seed an unseeded state to "up", so a channel that had NEVER reached
// its agent logged `up->down:unreachable` and orUnseeded was dead code. On 2026-07-25 that made the
// log imply a working channel degrading, when in truth neither controller had ever been up — it
// actively misdirected the first read of the incident.
//
// THE OTHER HALF OF THIS TEST IS THE POINT: alerting, debounce and dashboard behaviour must be
// byte-for-byte unchanged. A logging fix that shifts alerting is a regression in a cosmetic disguise,
// so the sink calls are asserted in COUNT and ARGUMENTS, not just the log string.
func TestScenarioF_BornDownLogsUnseededNotUp(t *testing.T) {
	var logbuf bytes.Buffer
	sink := &fakeSink{}
	c := New(nil, sink, log.New(&logbuf, "", 0))
	// A debounced reason (connection refused → unreachable), failing from the very first probe.
	p := &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{step(false, errors.New("dial tcp 192.168.0.87:8443: connect: connection refused"))}}
	c.probe = p.fn

	// Probe 1: debounced, suppressed — no dashboard flip, no alert.
	if err := c.Check(context.Background()); err != nil {
		t.Fatal(err)
	}
	if sink.dashDown || len(sink.downs) != 0 {
		t.Fatalf("probe 1 must be suppressed by debounce: dashDown=%v downs=%d", sink.dashDown, len(sink.downs))
	}
	// The placeholder must NOT be the string "up".
	if got := c.State(); got == "up" {
		t.Error("an unseeded checker must not report state \"up\" after a suppressed first failure")
	}

	// Probe 2: confirmed down.
	if err := c.Check(context.Background()); err != nil {
		t.Fatal(err)
	}

	logged := logbuf.String()
	if !strings.Contains(logged, "unseeded->down:unreachable") {
		t.Errorf("born-down must log \"unseeded->down:unreachable\"; got:\n%s", logged)
	}
	if strings.Contains(logged, "up->down") {
		t.Errorf("a channel that was never up must NOT log \"up->down\"; got:\n%s", logged)
	}

	// --- the non-change assertions ---
	if len(sink.downs) != 1 {
		t.Fatalf("exactly ONE down alert must fire (F2 born-down behaviour unchanged), got %d", len(sink.downs))
	}
	got := sink.downs[0]
	if got.reason != ReasonUnreachable || got.eventType != "agent_channel_unreachable" || got.severity != "warning" {
		t.Errorf("alert arguments changed: %+v — Part 3 is a LOGGING fix only", got)
	}
	if !sink.dashDown || sink.dashReason != ReasonUnreachable {
		t.Errorf("dashboard must be set down/unreachable, got down=%v reason=%q", sink.dashDown, sink.dashReason)
	}
	if sink.recovered != 0 {
		t.Errorf("no recovery must fire, got %d", sink.recovered)
	}
	if p.calls != 2 {
		t.Errorf("debounce threshold unchanged: expected 2 probes, got %d", p.calls)
	}
}

// A REAL up->down transition must still log "up->down" — the fix must not relabel everything.
func TestScenarioF_RealTransitionStillLogsUp(t *testing.T) {
	var logbuf bytes.Buffer
	sink := &fakeSink{}
	c := New(nil, sink, log.New(&logbuf, "", 0))
	p := &scriptedProbe{steps: []struct {
		cons bool
		err  error
	}{
		step(false, nil), // observed UP for real
		step(false, errors.New("connect: connection refused")),
		step(false, errors.New("connect: connection refused")),
	}}
	c.probe = p.fn
	for i := 0; i < 3; i++ {
		if err := c.Check(context.Background()); err != nil {
			t.Fatal(err)
		}
	}
	logged := logbuf.String()
	if !strings.Contains(logged, "up->down:unreachable") {
		t.Errorf("a genuine transition must still log \"up->down:unreachable\"; got:\n%s", logged)
	}
	if strings.Contains(logged, "unseeded->down") {
		t.Errorf("an observed-up channel must not log \"unseeded->down\"; got:\n%s", logged)
	}
	if len(sink.downs) != 1 {
		t.Errorf("exactly one down alert, got %d", len(sink.downs))
	}
}