felhom-agent/internal/guestnet/watchdog_test.go

package guestnet

import (
	"bytes"
	"context"
	"errors"
	"io"
	"log/slog"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// --- fixtures captured LIVE from guest 9201 on 2026-07-21 (probe P3, via `ssh felhom-pve`) -------
//
// These are the byte shapes the parser must survive; note the literal backslash `ip -o` emits and
// the trailing space on the route line.

const (
	fxAddr       = "2: eth0    inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\\       valid_lft 4916sec preferred_lft 4916sec\n"
	fxRoute      = "default via 192.168.0.1 dev eth0 \n"
	fxPgrep      = "235839\n"
	fxIfacesDHCP = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n"
	fxIfacesStat = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet static\n\taddress 192.168.0.162/24\n\tgateway 192.168.0.1\n"
	// pct exec against a guest that does not exist / is not running (rc=2, message on stderr).
	fxNoGuestErr = "Configuration file 'nodes/demo-felhom/lxc/9999.conf' does not exist\n"
)

// scriptedRunner answers per probe kind and records EVERY argv. The counts are the assertions that
// matter: a watchdog that heals when it must not is worse than one that never heals.
type scriptedRunner struct {
	mu   sync.Mutex
	out  map[string]string // kind → stdout
	fail map[string]error  // kind → error
	errs map[string]string // kind → stderr
	call [][]string
	// healFixes models what a successful dhclient actually does: the client is running again and
	// the lease is renewed. Set false to model a guest whose network is broken beyond dhclient.
	healFixes bool
}

func newRunner() *scriptedRunner {
	return &scriptedRunner{
		out: map[string]string{
			"addr": fxAddr, "route": fxRoute, "iface": fxIfacesDHCP, "pgrep": fxPgrep, "dhclient": "",
		},
		fail:      map[string]error{},
		errs:      map[string]string{},
		healFixes: true,
	}
}

// kind classifies a `pct exec <vmid> -- <cmd> ...` argv.
func kind(args []string) string {
	if len(args) < 4 || args[0] != "exec" {
		return "other"
	}
	rest := args[3:]
	switch rest[0] {
	case "ip":
		if len(rest) > 1 && rest[1] == "route" {
			return "route"
		}
		return "addr"
	case "cat":
		return "iface"
	case "pgrep":
		return "pgrep"
	case "dhclient":
		return "dhclient"
	}
	return "other"
}

func (r *scriptedRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.call = append(r.call, append([]string{name}, args...))
	k := kind(args)
	if k == "dhclient" && r.fail[k] == nil && r.healFixes {
		// a real dhclient re-acquires the lease and stays resident
		r.out["pgrep"], r.out["addr"], r.out["route"] = fxPgrep, fxAddr, fxRoute
		delete(r.fail, "pgrep")
	}
	return []byte(r.out[k]), []byte(r.errs[k]), r.fail[k]
}

func (r *scriptedRunner) countOf(k string) int {
	r.mu.Lock()
	defer r.mu.Unlock()
	n := 0
	for _, c := range r.call {
		if len(c) > 1 && kind(c[1:]) == k {
			n++
		}
	}
	return n
}

func (r *scriptedRunner) lastOf(k string) []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	for i := len(r.call) - 1; i >= 0; i-- {
		if len(r.call[i]) > 1 && kind(r.call[i][1:]) == k {
			return r.call[i]
		}
	}
	return nil
}

// killDHClient models the exact 2026-07-20 state: lease still valid (address AND route present),
// dhclient gone. This is the fixture the whole feature exists for.
func (r *scriptedRunner) killDHClient() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.out["pgrep"] = ""
	r.fail["pgrep"] = errors.New("exit status 1") // pgrep: no match, EMPTY stderr
}

func (r *scriptedRunner) reviveDHClient() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.out["pgrep"] = fxPgrep
	delete(r.fail, "pgrep")
}

// expireLease models the state 80 minutes later: address and route gone too.
func (r *scriptedRunner) expireLease() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.out["addr"] = ""
	r.out["route"] = ""
}

type fakeGuests struct {
	guests []proxmox.Guest
	err    error
}

func (f *fakeGuests) Guests(context.Context) ([]proxmox.Guest, error) { return f.guests, f.err }

func running9201() *fakeGuests {
	return &fakeGuests{guests: []proxmox.Guest{
		{VMID: 9201, Name: "felhom-demo", Status: "running", Type: "lxc", Uptime: 7200},
	}}
}

// newTestWatchdog wires a watchdog with a manual clock the test advances, and captures the log so
// the "healthy cycles are observable" contract can be asserted rather than assumed.
func newTestWatchdog(r Runner, g GuestSource) (*Watchdog, *time.Time, *bytes.Buffer) {
	clock := time.Unix(1_784_000_000, 0).UTC()
	buf := &bytes.Buffer{}
	logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
	w := New(r, g, time.Minute, logger)
	w.now = func() time.Time { return clock }
	// The agent's own settle window is measured from startedAt, which New stamped with the REAL
	// clock; restamp it against the fake one, well in the past.
	w.startedAt = clock.Add(-time.Hour)
	return w, &clock, buf
}

// --- Scenario E: the TIMED failure, detected instantly ------------------------------------------
//
// RED-PROOF (recorded in REPORT.md): reverting classify()'s dhcp arm to IP-presence-only —
//
//	case ModeDHCP:
//	    if p.IP == "" { return StateUnhealthy, ... }
//	    return StateHealthy, ...
//
// makes TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy report "healthy" for the July-20 fixture, and
// TestWatchdog_HealsTheIncidentState records ZERO heals. That is the 80-minute silent window, exactly
// as it happened.

func TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy(t *testing.T) {
	r := newRunner()
	r.killDHClient()
	w, _, _ := newTestWatchdog(r, running9201())

	p := w.probe(context.Background(), 9201)
	if !p.Reachable {
		t.Fatalf("guest must read as reachable: %+v", p)
	}
	if p.IP != "192.168.0.104" || !p.HasRoute {
		t.Fatalf("the lease is still live in this fixture — IP/route must be present: %+v", p)
	}
	if p.DHCPAlive {
		t.Fatalf("dhclient must read as dead: %+v", p)
	}

	state, detail := classify(p)
	if state != StateUnhealthy {
		t.Fatalf("classify = %q, want %q — waiting for the IP to vanish is the 80-minute silent "+
			"window the incident proved (detail: %s)", state, StateUnhealthy, detail)
	}
	if !strings.Contains(detail, "dhclient") {
		t.Fatalf("the reason must name the dead client, got %q", detail)
	}
}

func TestWatchdog_HealsTheIncidentState(t *testing.T) {
	r := newRunner()
	r.killDHClient()
	w, clock, logBuf := newTestWatchdog(r, running9201())
	ctx := context.Background()

	// Cycle 1: unhealthy, but one bad probe is not a diagnosis.
	w.Tick(ctx)
	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("healed after ONE bad probe (%d heals) — a single blip must never trigger a heal", n)
	}

	// Cycle 2: second consecutive bad probe → heal. The heal makes the client live again.
	*clock = clock.Add(time.Minute)
	w.Tick(ctx)

	if n := r.countOf("dhclient"); n != 1 {
		t.Fatalf("heal ran %d times, want exactly 1", n)
	}
	// The invocation must be the incident's, verbatim.
	want := []string{"pct", "exec", "9201", "--", "dhclient",
		"-pf", "/run/dhclient.eth0.pid", "-lf", "/var/lib/dhcp/dhclient.eth0.leases", "eth0"}
	got := r.lastOf("dhclient")
	if len(got) != len(want) {
		t.Fatalf("heal argv = %v, want %v", got, want)
	}
	for i := range want {
		if got[i] != want[i] {
			t.Fatalf("heal argv[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)
		}
	}

	// The report must show the heal AND the verified-healthy re-probe.
	snap := w.Snapshot()
	if len(snap) != 1 {
		t.Fatalf("snapshot = %+v, want one guest", snap)
	}
	g := snap[0]
	if !g.Healed || !g.HealSucceeded {
		t.Fatalf("report must record a successful heal: %+v", g)
	}
	if g.State != string(StateHealthy) || g.LastHealAt == "" || g.HealsLastHour != 1 {
		t.Fatalf("post-heal report is wrong: %+v", g)
	}
	if !strings.Contains(logBuf.String(), "guestnet: guest network healed") {
		t.Fatalf("the heal was not logged: %s", logBuf.String())
	}
}

// A healthy box must be silent about alarms but NOT silent about having looked.
func TestWatchdog_HealthyCycleProbesAndNeverHeals(t *testing.T) {
	r := newRunner()
	w, _, logBuf := newTestWatchdog(r, running9201())

	w.Tick(context.Background())

	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("a healthy guest was healed %d times, want 0", n)
	}
	if r.countOf("pgrep") != 1 || r.countOf("addr") != 1 {
		t.Fatalf("the healthy path must still probe: %v", r.call)
	}
	if !strings.Contains(logBuf.String(), "guest network healthy") {
		t.Fatalf("a healthy cycle must be observable — otherwise 'no alarms' and 'never probed' "+
			"are the same evidence (v0.91.2's lesson). Log: %s", logBuf.String())
	}
	snap := w.Snapshot()
	if len(snap) != 1 || snap[0].State != string(StateHealthy) || !snap[0].DHClientAlive {
		t.Fatalf("healthy snapshot wrong: %+v", snap)
	}
}

// --- Scenario F: configuration and damping guards ------------------------------------------------

func TestWatchdog_StaticGuestIsNeverHealedWithDHClient(t *testing.T) {
	r := newRunner()
	r.out["iface"] = fxIfacesStat
	r.killDHClient() // on a static guest this is NORMAL
	w, clock, _ := newTestWatchdog(r, running9201())

	for i := 0; i < 5; i++ {
		w.Tick(context.Background())
		*clock = clock.Add(time.Minute)
	}
	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("a static guest was healed with dhclient %d times, want 0", n)
	}
	if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateHealthy) {
		t.Fatalf("a static guest with address+route is healthy, got %+v", s)
	}
}

func TestWatchdog_StaticGuestMissingAddressReportsButNeverHeals(t *testing.T) {
	r := newRunner()
	r.out["iface"] = fxIfacesStat
	r.expireLease() // no address, no route on a STATIC guest → R-50 territory, not ours
	w, clock, logBuf := newTestWatchdog(r, running9201())

	for i := 0; i < 5; i++ {
		w.Tick(context.Background())
		*clock = clock.Add(time.Minute)
	}
	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("healed a static guest %d times, want 0 — dhclient must never fight a static config", n)
	}
	s := w.Snapshot()
	if len(s) != 1 || s[0].State != string(StateStaticFault) {
		t.Fatalf("state = %+v, want static_fault", s)
	}
	if !strings.Contains(logBuf.String(), "not actionable") {
		t.Fatalf("a static fault must be reported loudly: %s", logBuf.String())
	}
	// Loud ONCE per transition, not once per cycle.
	if n := strings.Count(logBuf.String(), "not actionable"); n != 1 {
		t.Fatalf("static fault logged %d times over 5 cycles, want 1 (per transition)", n)
	}
}

func TestWatchdog_UnreachableGuestIsUnknownAndNeverHealed(t *testing.T) {
	r := newRunner()
	r.fail["addr"] = errors.New("exit status 2")
	r.errs["addr"] = fxNoGuestErr
	w, clock, _ := newTestWatchdog(r, running9201())

	for i := 0; i < 4; i++ {
		w.Tick(context.Background())
		*clock = clock.Add(time.Minute)
	}
	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("healed a guest we could not probe %d times, want 0 — acting blind is how the "+
			"incident happened", n)
	}
	if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) {
		t.Fatalf("state = %+v, want unknown", s)
	}
}

// A missing pgrep (or any probe tool) must read as unknown, never as a dead client — otherwise a
// broken probe would heal forever.
func TestWatchdog_ProbeToolFailureIsUnknownNotDead(t *testing.T) {
	r := newRunner()
	r.fail["pgrep"] = errors.New("exit status 127")
	r.errs["pgrep"] = "pgrep: command not found\n"
	w, clock, _ := newTestWatchdog(r, running9201())

	for i := 0; i < 4; i++ {
		w.Tick(context.Background())
		*clock = clock.Add(time.Minute)
	}
	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("a failed liveness probe caused %d heals, want 0", n)
	}
	if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) {
		t.Fatalf("state = %+v, want unknown", s)
	}
}

func TestWatchdog_DampingCeilings(t *testing.T) {
	r := newRunner()
	r.killDHClient()
	r.healFixes = false // the heal "works" but the client dies again immediately
	w, clock, _ := newTestWatchdog(r, running9201())
	ctx := context.Background()

	// 10 hours of one-minute cycles against a permanently broken guest.
	for i := 0; i < 600; i++ {
		w.Tick(ctx)
		*clock = clock.Add(time.Minute)
	}

	heals := r.countOf("dhclient")
	// Ceiling: 3 per hour AND ≥10 min apart ⇒ at most 3 in any rolling hour. Over 10 h the
	// min-interval rule dominates: 6 slots/hour capped to 3/hour ⇒ ≤ 30.
	if heals > 30 {
		t.Fatalf("heals = %d over 10 h, want ≤ 30 (≤3/hour) — the damper is not holding", heals)
	}
	if heals == 0 {
		t.Fatalf("heals = 0 — the damper has become a mute")
	}
}

func TestWatchdog_MinimumIntervalBetweenHeals(t *testing.T) {
	r := newRunner()
	r.killDHClient()
	r.healFixes = false
	w, clock, _ := newTestWatchdog(r, running9201())
	ctx := context.Background()

	w.Tick(ctx) // bad probe 1
	*clock = clock.Add(time.Minute)
	w.Tick(ctx) // bad probe 2 → heal #1
	if r.countOf("dhclient") != 1 {
		t.Fatalf("expected exactly one heal by now, got %d", r.countOf("dhclient"))
	}
	// Nine more minutes of failure: still inside the 10-minute cool-off.
	for i := 0; i < 9; i++ {
		*clock = clock.Add(time.Minute)
		w.Tick(ctx)
	}
	if n := r.countOf("dhclient"); n != 1 {
		t.Fatalf("heals = %d within the 10-minute cool-off, want 1", n)
	}
	if s := w.Snapshot(); len(s) != 1 || !s[0].Damped {
		t.Fatalf("a damped cycle must say so in the report: %+v", s)
	}
	// Past the cool-off, one more heal is allowed.
	*clock = clock.Add(2 * time.Minute)
	w.Tick(ctx)
	if n := r.countOf("dhclient"); n != 2 {
		t.Fatalf("heals = %d after the cool-off expired, want 2", n)
	}
}

func TestWatchdog_BootRacesObserveOnly(t *testing.T) {
	t.Run("young guest", func(t *testing.T) {
		r := newRunner()
		r.killDHClient()
		g := running9201()
		g.guests[0].Uptime = 40 // seconds
		w, clock, _ := newTestWatchdog(r, g)
		for i := 0; i < 4; i++ {
			w.Tick(context.Background())
			*clock = clock.Add(time.Minute)
		}
		if n := r.countOf("dhclient"); n != 0 {
			t.Fatalf("healed a guest that booted 40 s ago %d times, want 0 — it has no lease YET", n)
		}
	})

	t.Run("young agent", func(t *testing.T) {
		r := newRunner()
		r.killDHClient()
		w, clock, _ := newTestWatchdog(r, running9201())
		w.startedAt = *clock // the agent just started
		for i := 0; i < 2; i++ {
			w.Tick(context.Background())
			*clock = clock.Add(time.Minute)
		}
		if n := r.countOf("dhclient"); n != 0 {
			t.Fatalf("healed %d times within the agent's own settle window, want 0", n)
		}
	})
}

func TestWatchdog_StoppedGuestIsNotProbed(t *testing.T) {
	r := newRunner()
	g := running9201()
	g.guests[0].Status = "stopped"
	w, _, _ := newTestWatchdog(r, g)

	w.Tick(context.Background())

	if len(r.call) != 0 {
		t.Fatalf("a stopped guest was probed: %v", r.call)
	}
}

func TestWatchdog_GuestListFailureSkipsTheSweep(t *testing.T) {
	r := newRunner()
	w, _, logBuf := newTestWatchdog(r, &fakeGuests{err: errors.New("pool membership read: 403")})

	w.Tick(context.Background())

	if len(r.call) != 0 {
		t.Fatalf("acted with unproven ownership: %v", r.call)
	}
	if !strings.Contains(logBuf.String(), "ownership unproven") {
		t.Fatalf("the skip must be logged: %s", logBuf.String())
	}
}

// A transient bad probe followed by recovery must never heal, and must log the recovery.
func TestWatchdog_SingleBlipNeverHeals(t *testing.T) {
	r := newRunner()
	w, clock, logBuf := newTestWatchdog(r, running9201())
	ctx := context.Background()

	w.Tick(ctx) // healthy
	r.killDHClient()
	*clock = clock.Add(time.Minute)
	w.Tick(ctx) // bad probe 1
	r.reviveDHClient()
	*clock = clock.Add(time.Minute)
	w.Tick(ctx) // healthy again

	if n := r.countOf("dhclient"); n != 0 {
		t.Fatalf("a single blip caused %d heals, want 0", n)
	}
	if !strings.Contains(logBuf.String(), "guest network recovered") {
		t.Fatalf("the recovery must be visible: %s", logBuf.String())
	}
}

// --- parsers over the live P3 fixtures ------------------------------------------------------------

func TestParsers_OverLiveFixtures(t *testing.T) {
	if got := parseInet(fxAddr); got != "192.168.0.104" {
		t.Fatalf("parseInet = %q, want 192.168.0.104", got)
	}
	if got := parseInet(""); got != "" {
		t.Fatalf("parseInet(empty) = %q, want empty (the post-expiry state prints nothing)", got)
	}
	if !hasDefaultRoute(fxRoute) {
		t.Fatalf("hasDefaultRoute(%q) = false", fxRoute)
	}
	if hasDefaultRoute("") || hasDefaultRoute("172.17.0.0/16 dev docker0 proto kernel scope link\n") {
		t.Fatal("docker bridge routes must not read as a default route (the incident's exact leftovers)")
	}
	if got := parseMode(fxIfacesDHCP, "eth0"); got != ModeDHCP {
		t.Fatalf("parseMode(dhcp) = %q", got)
	}
	if got := parseMode(fxIfacesStat, "eth0"); got != ModeStatic {
		t.Fatalf("parseMode(static) = %q", got)
	}
	if got := parseMode("auto lo\niface lo inet loopback\n", "eth0"); got != ModeUnknown {
		t.Fatalf("parseMode(no eth0 stanza) = %q, want unknown", got)
	}
	if got := parseMode("iface eth0 inet manual\n", "eth0"); got != ModeUnknown {
		t.Fatalf("parseMode(manual) = %q, want unknown", got)
	}
}

func TestClassify_Table(t *testing.T) {
	cases := []struct {
		name string
		p    Probe
		want State
	}{
		{"dhcp all green", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true, DHCPAlive: true}, StateHealthy},
		{"dhcp dead client, live lease", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true}, StateUnhealthy},
		{"dhcp no address", Probe{Reachable: true, Mode: ModeDHCP, DHCPAlive: true}, StateUnhealthy},
		{"dhcp no route", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", DHCPAlive: true}, StateUnhealthy},
		{"static green", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy},
		{"static no client is normal", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy},
		{"static broken", Probe{Reachable: true, Mode: ModeStatic}, StateStaticFault},
		{"unreachable", Probe{Mode: ModeDHCP}, StateUnknown},
		{"unknown mode", Probe{Reachable: true, Mode: ModeUnknown, IP: "1.2.3.4", HasRoute: true}, StateUnknown},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got, _ := classify(tc.p); got != tc.want {
				t.Fatalf("classify = %q, want %q", got, tc.want)
			}
		})
	}
}

var _ io.Writer = (*bytes.Buffer)(nil)

// A3 (R-50): the guestnet healer is eth0-only and MUST stay blind to the island NIC. A guest on an
// island host presents eth0 DHCP (the LAN leg the healer owns) PLUS eth1 static (the island). Because
// parseMode is interface-scoped, adding eth1 static cannot flip eth0's detected mode — so the healer
// keeps treating eth0 as DHCP and never runs dhclient against the static island NIC (which would
// sabotage it). This is the verify-only guarantee that let R-50 ship the island NIC without a healer
// change. Red-proof: make parseMode scan globally instead of per-dev and the eth0 assertion fails.
func TestParseMode_IslandStaticNICDoesNotConfuseEth0(t *testing.T) {
	interfaces := "auto lo\niface lo inet loopback\n\n" +
		"auto eth0\niface eth0 inet dhcp\n\n" +
		"auto eth1\niface eth1 inet static\n    address 169.254.253.2/30\n"
	if got := parseMode(interfaces, "eth0"); got != ModeDHCP {
		t.Errorf("eth0 must classify DHCP even with an island eth1 static present, got %q", got)
	}
	if got := parseMode(interfaces, "eth1"); got != ModeStatic {
		t.Errorf("eth1 (island) must classify static when asked directly (dev-scoped), got %q", got)
	}
}