// Package guestnet implements R-54: the host-tier watchdog for each customer guest's own network. // // Origin — INCIDENT-guest-dhclient-killed-2026-07-20 §5 "OPEN RISK". The guest's DHCP client is // started once by ifupdown at boot and NOTHING supervises it. When it was killed on 2026-07-20 the // guest kept working for another ~80 minutes on its unexpired lease; only when the lease expired did // the address and default route vanish, taking the Cloudflare tunnel, the hub reports, the catalog // sync and the controller→agent channel with them. Total outage ~1h15m, and for the first 80 minutes // every observable signal said healthy. // // The design consequence is the whole point of this package: **liveness of the DHCP client process // is itself a probe**, not a detail. Waiting for the IP to disappear is waiting out the exact silent // window the incident proved exists. See TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy. // // The agent is the right tier for this: it lives on the host, keeps its own line to the hub, and can // still see and repair a guest that has gone completely mute. The controller cannot fix its own // missing default route. package guestnet import ( "context" "strings" ) // Runner is the privileged-exec seam (satisfied by *proxmox.ExecRunner). Declared consumer-side so // tests inject a scripted runner and no unit test goes near pct. type Runner interface { Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error) } // Mode is how the guest is configured to get its address. type Mode string const ( ModeDHCP Mode = "dhcp" ModeStatic Mode = "static" ModeUnknown Mode = "unknown" ) // State is a guest's verdict for one cycle. type State string const ( StateHealthy State = "healthy" // StateUnhealthy: DHCP-configured and something is wrong that dhclient can fix. StateUnhealthy State = "unhealthy" // StateStaticFault: a static guest missing its address/route. Reported loudly, NEVER healed — // re-running dhclient on a statically-configured guest would fight its own configuration, and // the durable answer is R-50 (island bridge), not a point fix here. StateStaticFault State = "static_fault" // StateUnknown: the guest could not be probed at all (pct exec failed, an interface file we // cannot read, a probe tool missing). Never healed — acting blind is how the incident happened. StateUnknown State = "unknown" ) // Probe is one guest's observed network facts. type Probe struct { VMID int Mode Mode IP string // empty when absent HasRoute bool DHCPAlive bool Reachable bool // pct exec worked at all Detail string // human-readable reason, operator-tier English } // eth0 is the guest interface every felhom guest uses (the LXC veth peer inside the guest). const eth0 = "eth0" // probe runs the four fixed-shape reads. Every argv is a constant plus the vmid — no guest-supplied // data is ever interpolated into a command, and there is no shell anywhere in this path. func (w *Watchdog) probe(ctx context.Context, vmid int) Probe { p := Probe{VMID: vmid, Mode: ModeUnknown} id := itoa(vmid) // 1. Address. This also settles reachability: if pct exec cannot run here, nothing else is // worth attempting. out, errOut, err := w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "-4", "-o", "addr", "show", "dev", eth0) if err != nil { p.Detail = "address probe failed: " + firstLine(string(errOut)) return p // Reachable stays false → StateUnknown } p.Reachable = true p.IP = parseInet(string(out)) // 2. Default route. out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "route", "show", "default") if err == nil { p.HasRoute = hasDefaultRoute(string(out)) } // 3. Configured mode. An unreadable interfaces file leaves ModeUnknown, which never heals. out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "cat", "/etc/network/interfaces") if err == nil { p.Mode = parseMode(string(out), eth0) } // 4. DHCP client liveness — the probe the incident was invisible to. // pgrep exits 1 with EMPTY stderr when there is no match; anything on stderr means the probe // itself failed (pgrep absent, guest wedged), which must read as unknown rather than as a // dead client, or a missing tool would trigger heals forever. out, errOut, err = w.runner.Run(ctx, "pct", "exec", id, "--", "pgrep", "-x", "dhclient") switch { case err == nil && strings.TrimSpace(string(out)) != "": p.DHCPAlive = true case err != nil && strings.TrimSpace(string(errOut)) != "": p.Reachable = false p.Detail = "dhclient liveness probe failed: " + firstLine(string(errOut)) default: p.DHCPAlive = false } return p } // classify turns observed facts into the verdict. Pure — table-tested. func classify(p Probe) (State, string) { if !p.Reachable { d := p.Detail if d == "" { d = "guest not reachable via pct exec" } return StateUnknown, d } switch p.Mode { case ModeStatic: if p.IP != "" && p.HasRoute { return StateHealthy, "static address and default route present" } return StateStaticFault, "statically configured guest is missing its address or default route — reported only; dhclient must never be run against a static configuration (R-50 owns the durable fix)" case ModeDHCP: switch { case p.IP == "": return StateUnhealthy, "no IPv4 address on " + eth0 case !p.HasRoute: return StateUnhealthy, "no default route" case !p.DHCPAlive: // THE incident state: address and route still present on an unexpired lease, with // nothing left to renew them. Damage is ~1-2 h in the future and invisible today. return StateUnhealthy, "dhclient is not running — the lease will not be renewed (the 2026-07-20 failure mode; address still present, renewal already dead)" default: return StateHealthy, "address, default route and dhclient all present" } default: return StateUnknown, "interface configuration mode could not be determined — not healing" } } // --- parsing (fixtures captured live from guest 9201 on 2026-07-21, probe P3) ------------------- // parseInet extracts the address from `ip -4 -o addr show dev eth0` output, e.g. // // 2: eth0 inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\ valid_lft ... // // Returns "" when there is no inet line at all (the post-lease-expiry state: the command succeeds // and prints NOTHING). func parseInet(out string) string { for _, line := range strings.Split(out, "\n") { fields := strings.Fields(line) for i, f := range fields { if f == "inet" && i+1 < len(fields) { addr := fields[i+1] if idx := strings.IndexByte(addr, '/'); idx > 0 { addr = addr[:idx] } return addr } } } return "" } // hasDefaultRoute parses `ip route show default`, e.g. "default via 192.168.0.1 dev eth0 ". // Empty output = no default route (the incident state). func hasDefaultRoute(out string) bool { for _, line := range strings.Split(out, "\n") { if strings.HasPrefix(strings.TrimSpace(line), "default ") { return true } } return false } // parseMode reads the iface stanza for dev out of /etc/network/interfaces: // // auto eth0 // iface eth0 inet dhcp // // Anything else (no stanza, a manual/loopback mode, a file we could not read) is ModeUnknown, and // unknown never heals. func parseMode(out, dev string) Mode { for _, line := range strings.Split(out, "\n") { f := strings.Fields(strings.TrimSpace(line)) // iface inet if len(f) >= 4 && f[0] == "iface" && f[1] == dev && f[2] == "inet" { switch f[3] { case "dhcp": return ModeDHCP case "static": return ModeStatic default: return ModeUnknown } } } return ModeUnknown } func firstLine(s string) string { s = strings.TrimSpace(s) if i := strings.IndexByte(s, '\n'); i >= 0 { s = s[:i] } if len(s) > 200 { s = s[:200] } return s } // itoa avoids pulling strconv into every call site's readability. func itoa(i int) string { if i == 0 { return "0" } neg := i < 0 if neg { i = -i } var b [20]byte pos := len(b) for i > 0 { pos-- b[pos] = byte('0' + i%10) i /= 10 } if neg { pos-- b[pos] = '-' } return string(b[pos:]) }