feat(v0.92.0): guest-network watchdog (R-54) — supervise the guest's DHCP client
Closes the OPEN RISK in INCIDENT-guest-dhclient-killed-2026-07-20 §5. The guest's dhclient
is started once by ifupdown at boot and nothing supervises it; when it died on 2026-07-20
the guest ran another ~80 minutes on its unexpired lease, then lost its address and default
route and took the tunnel, hub reports, catalog sync and the controller->agent channel with
it (1h15m outage, healthy-looking for the first 80 minutes).
So liveness of the DHCP client is itself a probe: a DHCP guest is unhealthy the moment
`pgrep -x dhclient` comes back empty, while the lease is still live. Waiting for the address
to vanish is waiting out the silent window.
internal/guestnet: four fixed-shape pct exec probes (address, default route, interfaces
mode, dhclient liveness — parsers pinned to output captured live from 9201), the incident's
heal invocation verbatim, and dampers throughout: two consecutive bad probes, >=10 min
between heals, <=3/hour, observe-only while guest or agent uptime < 3 min. Refuses to act on
a static guest, an unknown mode, an unprobeable guest, or an unproven guest list (the source
is the pool-verified ListLXC ∩ felhom pool, never a bare ListLXC). A failed probe reads as
unknown, never as a dead client. Healthy cycles log a Debug line so "no alarms" and "never
probed" stay distinguishable. Not in the errc fan-out — a guest watchdog must never be able
to kill the agent.
guest_net is the repo's first default-ON gate (opt-out is `{"disable": true}`): it looks only
inward at guests we already own, and the failure exists on every box today.
Report block ships as GuestNetStatus, not the spec's WireGuestNet: Wire* is the DOWN
direction in this repo, report stanzas are *Status.
Red-proofs: classify reverted to IP-presence-only -> the July-20 fixture reports "healthy"
with zero heals; un-wiring the reporter and the goroutine fails the AST wiring test.
Also: `var version` was stale at 0.89.0 (ldflags hid it; `go run` did not).
This commit is contained in:
@@ -0,0 +1,239 @@
|
||||
// Package guestnet implements R-54: the host-tier watchdog for each customer guest's own network.
|
||||
//
|
||||
// Origin — INCIDENT-guest-dhclient-killed-2026-07-20 §5 "OPEN RISK". The guest's DHCP client is
|
||||
// started once by ifupdown at boot and NOTHING supervises it. When it was killed on 2026-07-20 the
|
||||
// guest kept working for another ~80 minutes on its unexpired lease; only when the lease expired did
|
||||
// the address and default route vanish, taking the Cloudflare tunnel, the hub reports, the catalog
|
||||
// sync and the controller→agent channel with them. Total outage ~1h15m, and for the first 80 minutes
|
||||
// every observable signal said healthy.
|
||||
//
|
||||
// The design consequence is the whole point of this package: **liveness of the DHCP client process
|
||||
// is itself a probe**, not a detail. Waiting for the IP to disappear is waiting out the exact silent
|
||||
// window the incident proved exists. See TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy.
|
||||
//
|
||||
// The agent is the right tier for this: it lives on the host, keeps its own line to the hub, and can
|
||||
// still see and repair a guest that has gone completely mute. The controller cannot fix its own
|
||||
// missing default route.
|
||||
package guestnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Runner is the privileged-exec seam (satisfied by *proxmox.ExecRunner). Declared consumer-side so
|
||||
// tests inject a scripted runner and no unit test goes near pct.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error)
|
||||
}
|
||||
|
||||
// Mode is how the guest is configured to get its address.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
ModeDHCP Mode = "dhcp"
|
||||
ModeStatic Mode = "static"
|
||||
ModeUnknown Mode = "unknown"
|
||||
)
|
||||
|
||||
// State is a guest's verdict for one cycle.
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateHealthy State = "healthy"
|
||||
// StateUnhealthy: DHCP-configured and something is wrong that dhclient can fix.
|
||||
StateUnhealthy State = "unhealthy"
|
||||
// StateStaticFault: a static guest missing its address/route. Reported loudly, NEVER healed —
|
||||
// re-running dhclient on a statically-configured guest would fight its own configuration, and
|
||||
// the durable answer is R-50 (island bridge), not a point fix here.
|
||||
StateStaticFault State = "static_fault"
|
||||
// StateUnknown: the guest could not be probed at all (pct exec failed, an interface file we
|
||||
// cannot read, a probe tool missing). Never healed — acting blind is how the incident happened.
|
||||
StateUnknown State = "unknown"
|
||||
)
|
||||
|
||||
// Probe is one guest's observed network facts.
|
||||
type Probe struct {
|
||||
VMID int
|
||||
Mode Mode
|
||||
IP string // empty when absent
|
||||
HasRoute bool
|
||||
DHCPAlive bool
|
||||
Reachable bool // pct exec worked at all
|
||||
Detail string // human-readable reason, operator-tier English
|
||||
}
|
||||
|
||||
// eth0 is the guest interface every felhom guest uses (the LXC veth peer inside the guest).
|
||||
const eth0 = "eth0"
|
||||
|
||||
// probe runs the four fixed-shape reads. Every argv is a constant plus the vmid — no guest-supplied
|
||||
// data is ever interpolated into a command, and there is no shell anywhere in this path.
|
||||
func (w *Watchdog) probe(ctx context.Context, vmid int) Probe {
|
||||
p := Probe{VMID: vmid, Mode: ModeUnknown}
|
||||
id := itoa(vmid)
|
||||
|
||||
// 1. Address. This also settles reachability: if pct exec cannot run here, nothing else is
|
||||
// worth attempting.
|
||||
out, errOut, err := w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "-4", "-o", "addr", "show", "dev", eth0)
|
||||
if err != nil {
|
||||
p.Detail = "address probe failed: " + firstLine(string(errOut))
|
||||
return p // Reachable stays false → StateUnknown
|
||||
}
|
||||
p.Reachable = true
|
||||
p.IP = parseInet(string(out))
|
||||
|
||||
// 2. Default route.
|
||||
out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "route", "show", "default")
|
||||
if err == nil {
|
||||
p.HasRoute = hasDefaultRoute(string(out))
|
||||
}
|
||||
|
||||
// 3. Configured mode. An unreadable interfaces file leaves ModeUnknown, which never heals.
|
||||
out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "cat", "/etc/network/interfaces")
|
||||
if err == nil {
|
||||
p.Mode = parseMode(string(out), eth0)
|
||||
}
|
||||
|
||||
// 4. DHCP client liveness — the probe the incident was invisible to.
|
||||
// pgrep exits 1 with EMPTY stderr when there is no match; anything on stderr means the probe
|
||||
// itself failed (pgrep absent, guest wedged), which must read as unknown rather than as a
|
||||
// dead client, or a missing tool would trigger heals forever.
|
||||
out, errOut, err = w.runner.Run(ctx, "pct", "exec", id, "--", "pgrep", "-x", "dhclient")
|
||||
switch {
|
||||
case err == nil && strings.TrimSpace(string(out)) != "":
|
||||
p.DHCPAlive = true
|
||||
case err != nil && strings.TrimSpace(string(errOut)) != "":
|
||||
p.Reachable = false
|
||||
p.Detail = "dhclient liveness probe failed: " + firstLine(string(errOut))
|
||||
default:
|
||||
p.DHCPAlive = false
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// classify turns observed facts into the verdict. Pure — table-tested.
|
||||
func classify(p Probe) (State, string) {
|
||||
if !p.Reachable {
|
||||
d := p.Detail
|
||||
if d == "" {
|
||||
d = "guest not reachable via pct exec"
|
||||
}
|
||||
return StateUnknown, d
|
||||
}
|
||||
switch p.Mode {
|
||||
case ModeStatic:
|
||||
if p.IP != "" && p.HasRoute {
|
||||
return StateHealthy, "static address and default route present"
|
||||
}
|
||||
return StateStaticFault, "statically configured guest is missing its address or default route — reported only; dhclient must never be run against a static configuration (R-50 owns the durable fix)"
|
||||
case ModeDHCP:
|
||||
switch {
|
||||
case p.IP == "":
|
||||
return StateUnhealthy, "no IPv4 address on " + eth0
|
||||
case !p.HasRoute:
|
||||
return StateUnhealthy, "no default route"
|
||||
case !p.DHCPAlive:
|
||||
// THE incident state: address and route still present on an unexpired lease, with
|
||||
// nothing left to renew them. Damage is ~1-2 h in the future and invisible today.
|
||||
return StateUnhealthy, "dhclient is not running — the lease will not be renewed (the 2026-07-20 failure mode; address still present, renewal already dead)"
|
||||
default:
|
||||
return StateHealthy, "address, default route and dhclient all present"
|
||||
}
|
||||
default:
|
||||
return StateUnknown, "interface configuration mode could not be determined — not healing"
|
||||
}
|
||||
}
|
||||
|
||||
// --- parsing (fixtures captured live from guest 9201 on 2026-07-21, probe P3) -------------------
|
||||
|
||||
// parseInet extracts the address from `ip -4 -o addr show dev eth0` output, e.g.
|
||||
//
|
||||
// 2: eth0 inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\ valid_lft ...
|
||||
//
|
||||
// Returns "" when there is no inet line at all (the post-lease-expiry state: the command succeeds
|
||||
// and prints NOTHING).
|
||||
func parseInet(out string) string {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
for i, f := range fields {
|
||||
if f == "inet" && i+1 < len(fields) {
|
||||
addr := fields[i+1]
|
||||
if idx := strings.IndexByte(addr, '/'); idx > 0 {
|
||||
addr = addr[:idx]
|
||||
}
|
||||
return addr
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hasDefaultRoute parses `ip route show default`, e.g. "default via 192.168.0.1 dev eth0 ".
|
||||
// Empty output = no default route (the incident state).
|
||||
func hasDefaultRoute(out string) bool {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "default ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseMode reads the iface stanza for dev out of /etc/network/interfaces:
|
||||
//
|
||||
// auto eth0
|
||||
// iface eth0 inet dhcp
|
||||
//
|
||||
// Anything else (no stanza, a manual/loopback mode, a file we could not read) is ModeUnknown, and
|
||||
// unknown never heals.
|
||||
func parseMode(out, dev string) Mode {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(strings.TrimSpace(line))
|
||||
// iface <dev> inet <mode>
|
||||
if len(f) >= 4 && f[0] == "iface" && f[1] == dev && f[2] == "inet" {
|
||||
switch f[3] {
|
||||
case "dhcp":
|
||||
return ModeDHCP
|
||||
case "static":
|
||||
return ModeStatic
|
||||
default:
|
||||
return ModeUnknown
|
||||
}
|
||||
}
|
||||
}
|
||||
return ModeUnknown
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if len(s) > 200 {
|
||||
s = s[:200]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// itoa avoids pulling strconv into every call site's readability.
|
||||
func itoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := i < 0
|
||||
if neg {
|
||||
i = -i
|
||||
}
|
||||
var b [20]byte
|
||||
pos := len(b)
|
||||
for i > 0 {
|
||||
pos--
|
||||
b[pos] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
if neg {
|
||||
pos--
|
||||
b[pos] = '-'
|
||||
}
|
||||
return string(b[pos:])
|
||||
}
|
||||
Reference in New Issue
Block a user