felhom-agent/internal/guestnet/probe.go

// Package guestnet implements R-54: the host-tier watchdog for each customer guest's own network.
//
// Origin — INCIDENT-guest-dhclient-killed-2026-07-20 §5 "OPEN RISK". The guest's DHCP client is
// started once by ifupdown at boot and NOTHING supervises it. When it was killed on 2026-07-20 the
// guest kept working for another ~80 minutes on its unexpired lease; only when the lease expired did
// the address and default route vanish, taking the Cloudflare tunnel, the hub reports, the catalog
// sync and the controller→agent channel with them. Total outage ~1h15m, and for the first 80 minutes
// every observable signal said healthy.
//
// The design consequence is the whole point of this package: **liveness of the DHCP client process
// is itself a probe**, not a detail. Waiting for the IP to disappear is waiting out the exact silent
// window the incident proved exists. See TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy.
//
// The agent is the right tier for this: it lives on the host, keeps its own line to the hub, and can
// still see and repair a guest that has gone completely mute. The controller cannot fix its own
// missing default route.
package guestnet

import (
	"context"
	"strings"
)

// Runner is the privileged-exec seam (satisfied by *proxmox.ExecRunner). Declared consumer-side so
// tests inject a scripted runner and no unit test goes near pct.
type Runner interface {
	Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error)
}

// Mode is how the guest is configured to get its address.
type Mode string

const (
	ModeDHCP    Mode = "dhcp"
	ModeStatic  Mode = "static"
	ModeUnknown Mode = "unknown"
)

// State is a guest's verdict for one cycle.
type State string

const (
	StateHealthy State = "healthy"
	// StateUnhealthy: DHCP-configured and something is wrong that dhclient can fix.
	StateUnhealthy State = "unhealthy"
	// StateStaticFault: a static guest missing its address/route. Reported loudly, NEVER healed —
	// re-running dhclient on a statically-configured guest would fight its own configuration, and
	// the durable answer is R-50 (island bridge), not a point fix here.
	StateStaticFault State = "static_fault"
	// StateUnknown: the guest could not be probed at all (pct exec failed, an interface file we
	// cannot read, a probe tool missing). Never healed — acting blind is how the incident happened.
	StateUnknown State = "unknown"
)

// Probe is one guest's observed network facts.
type Probe struct {
	VMID      int
	Mode      Mode
	IP        string // empty when absent
	HasRoute  bool
	DHCPAlive bool
	Reachable bool   // pct exec worked at all
	Detail    string // human-readable reason, operator-tier English
}

// eth0 is the guest interface every felhom guest uses (the LXC veth peer inside the guest).
const eth0 = "eth0"

// probe runs the four fixed-shape reads. Every argv is a constant plus the vmid — no guest-supplied
// data is ever interpolated into a command, and there is no shell anywhere in this path.
func (w *Watchdog) probe(ctx context.Context, vmid int) Probe {
	p := Probe{VMID: vmid, Mode: ModeUnknown}
	id := itoa(vmid)

	// 1. Address. This also settles reachability: if pct exec cannot run here, nothing else is
	//    worth attempting.
	out, errOut, err := w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "-4", "-o", "addr", "show", "dev", eth0)
	if err != nil {
		p.Detail = "address probe failed: " + firstLine(string(errOut))
		return p // Reachable stays false → StateUnknown
	}
	p.Reachable = true
	p.IP = parseInet(string(out))

	// 2. Default route.
	out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "route", "show", "default")
	if err == nil {
		p.HasRoute = hasDefaultRoute(string(out))
	}

	// 3. Configured mode. An unreadable interfaces file leaves ModeUnknown, which never heals.
	out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "cat", "/etc/network/interfaces")
	if err == nil {
		p.Mode = parseMode(string(out), eth0)
	}

	// 4. DHCP client liveness — the probe the incident was invisible to.
	//    pgrep exits 1 with EMPTY stderr when there is no match; anything on stderr means the probe
	//    itself failed (pgrep absent, guest wedged), which must read as unknown rather than as a
	//    dead client, or a missing tool would trigger heals forever.
	out, errOut, err = w.runner.Run(ctx, "pct", "exec", id, "--", "pgrep", "-x", "dhclient")
	switch {
	case err == nil && strings.TrimSpace(string(out)) != "":
		p.DHCPAlive = true
	case err != nil && strings.TrimSpace(string(errOut)) != "":
		p.Reachable = false
		p.Detail = "dhclient liveness probe failed: " + firstLine(string(errOut))
	default:
		p.DHCPAlive = false
	}
	return p
}

// classify turns observed facts into the verdict. Pure — table-tested.
func classify(p Probe) (State, string) {
	if !p.Reachable {
		d := p.Detail
		if d == "" {
			d = "guest not reachable via pct exec"
		}
		return StateUnknown, d
	}
	switch p.Mode {
	case ModeStatic:
		if p.IP != "" && p.HasRoute {
			return StateHealthy, "static address and default route present"
		}
		return StateStaticFault, "statically configured guest is missing its address or default route — reported only; dhclient must never be run against a static configuration (R-50 owns the durable fix)"
	case ModeDHCP:
		switch {
		case p.IP == "":
			return StateUnhealthy, "no IPv4 address on " + eth0
		case !p.HasRoute:
			return StateUnhealthy, "no default route"
		case !p.DHCPAlive:
			// THE incident state: address and route still present on an unexpired lease, with
			// nothing left to renew them. Damage is ~1-2 h in the future and invisible today.
			return StateUnhealthy, "dhclient is not running — the lease will not be renewed (the 2026-07-20 failure mode; address still present, renewal already dead)"
		default:
			return StateHealthy, "address, default route and dhclient all present"
		}
	default:
		return StateUnknown, "interface configuration mode could not be determined — not healing"
	}
}

// --- parsing (fixtures captured live from guest 9201 on 2026-07-21, probe P3) -------------------

// parseInet extracts the address from `ip -4 -o addr show dev eth0` output, e.g.
//
//	2: eth0    inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\       valid_lft ...
//
// Returns "" when there is no inet line at all (the post-lease-expiry state: the command succeeds
// and prints NOTHING).
func parseInet(out string) string {
	for _, line := range strings.Split(out, "\n") {
		fields := strings.Fields(line)
		for i, f := range fields {
			if f == "inet" && i+1 < len(fields) {
				addr := fields[i+1]
				if idx := strings.IndexByte(addr, '/'); idx > 0 {
					addr = addr[:idx]
				}
				return addr
			}
		}
	}
	return ""
}

// hasDefaultRoute parses `ip route show default`, e.g. "default via 192.168.0.1 dev eth0 ".
// Empty output = no default route (the incident state).
func hasDefaultRoute(out string) bool {
	for _, line := range strings.Split(out, "\n") {
		if strings.HasPrefix(strings.TrimSpace(line), "default ") {
			return true
		}
	}
	return false
}

// parseMode reads the iface stanza for dev out of /etc/network/interfaces:
//
//	auto eth0
//	iface eth0 inet dhcp
//
// Anything else (no stanza, a manual/loopback mode, a file we could not read) is ModeUnknown, and
// unknown never heals.
func parseMode(out, dev string) Mode {
	for _, line := range strings.Split(out, "\n") {
		f := strings.Fields(strings.TrimSpace(line))
		// iface <dev> inet <mode>
		if len(f) >= 4 && f[0] == "iface" && f[1] == dev && f[2] == "inet" {
			switch f[3] {
			case "dhcp":
				return ModeDHCP
			case "static":
				return ModeStatic
			default:
				return ModeUnknown
			}
		}
	}
	return ModeUnknown
}

func firstLine(s string) string {
	s = strings.TrimSpace(s)
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		s = s[:i]
	}
	if len(s) > 200 {
		s = s[:200]
	}
	return s
}

// itoa avoids pulling strconv into every call site's readability.
func itoa(i int) string {
	if i == 0 {
		return "0"
	}
	neg := i < 0
	if neg {
		i = -i
	}
	var b [20]byte
	pos := len(b)
	for i > 0 {
		pos--
		b[pos] = byte('0' + i%10)
		i /= 10
	}
	if neg {
		pos--
		b[pos] = '-'
	}
	return string(b[pos:])
}