feat(v0.92.0): guest-network watchdog (R-54) — supervise the guest's DHCP client

Closes the OPEN RISK in INCIDENT-guest-dhclient-killed-2026-07-20 §5. The guest's dhclient is started once by ifupdown at boot and nothing supervises it; when it died on 2026-07-20 the guest ran another ~80 minutes on its unexpired lease, then lost its address and default route and took the tunnel, hub reports, catalog sync and the controller->agent channel with it (1h15m outage, healthy-looking for the first 80 minutes). So liveness of the DHCP client is itself a probe: a DHCP guest is unhealthy the moment `pgrep -x dhclient` comes back empty, while the lease is still live. Waiting for the address to vanish is waiting out the silent window. internal/guestnet: four fixed-shape pct exec probes (address, default route, interfaces mode, dhclient liveness — parsers pinned to output captured live from 9201), the incident's heal invocation verbatim, and dampers throughout: two consecutive bad probes, >=10 min between heals, <=3/hour, observe-only while guest or agent uptime < 3 min. Refuses to act on a static guest, an unknown mode, an unprobeable guest, or an unproven guest list (the source is the pool-verified ListLXC ∩ felhom pool, never a bare ListLXC). A failed probe reads as unknown, never as a dead client. Healthy cycles log a Debug line so "no alarms" and "never probed" stay distinguishable. Not in the errc fan-out — a guest watchdog must never be able to kill the agent. guest_net is the repo's first default-ON gate (opt-out is `{"disable": true}`): it looks only inward at guests we already own, and the failure exists on every box today. Report block ships as GuestNetStatus, not the spec's WireGuestNet: Wire* is the DOWN direction in this repo, report stanzas are *Status. Red-proofs: classify reverted to IP-presence-only -> the July-20 fixture reports "healthy" with zero heals; un-wiring the reporter and the goroutine fails the AST wiring test. Also: `var version` was stale at 0.89.0 (ldflags hid it; `go run` did not).
2026-07-21 12:29:37 +02:00
parent 08b55a1015
commit c0966d753d
12 changed files with 1575 additions and 1 deletions
@@ -65,6 +65,12 @@ type PBSDRReporter interface {
 	PBSDRStatus(ctx context.Context) *PBSDRStatus
 }

+// GuestNetReporter is the R-54 seam the guestnet watchdog plugs into (same consumer-side pattern —
+// hub does not import guestnet). nil (feature not wired) → no guest_net stanza.
+type GuestNetReporter interface {
+	GuestNetStatus(ctx context.Context) *GuestNetStatus
+}
+
 // Collector builds a HostReport from read-only sources. All deps are behind narrow
 // interfaces for unit testing.
 type Collector struct {
@@ -79,6 +85,7 @@ type Collector struct {
 	leafFP       string                                        // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled)
 	wg           WireguardReporter                             // S3: offsite-tunnel status (nil → stanza omitted)
 	pbsdr        PBSDRReporter                                 // slice 2: PBS DR tier bridge state (nil → stanza omitted)
+	guestNet     GuestNetReporter                              // R-54: per-guest network watchdog (nil → stanza omitted)
 	selfUpdate   SelfUpdateReporter                            // D1: agent self-update pending status (nil → false)
 	mgmtPlane    MgmtPlaneReporter                             // G1: management-plane health (nil → stanza omitted)
 	oob          OOBReporter                                   // H1: operator-access health (nil → stanza omitted)
@@ -145,6 +152,13 @@ func (c *Collector) SetPBSDRReporter(p PBSDRReporter) *Collector {
 	return c
 }

+// SetGuestNetReporter wires the R-54 guest-network watchdog as a report source (nil-safe → stanza
+// omitted). Returns the collector for chaining.
+func (c *Collector) SetGuestNetReporter(g GuestNetReporter) *Collector {
+	c.guestNet = g
+	return c
+}
+
 // SelfUpdateReporter is the D1 seam the selfupdate commit-manager plugs into (same consumer-side
 // pattern — hub does not import selfupdate). nil (feature not wired) → pending=false on the report.
 type SelfUpdateReporter interface {
@@ -227,6 +241,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
 	if c.pbsdr != nil {
 		report.PBSDR = c.pbsdr.PBSDRStatus(ctx)
 	}
+	// R-54: guest-network watchdog state (nil reporter = feature not wired → stanza omitted).
+	if c.guestNet != nil {
+		report.GuestNet = c.guestNet.GuestNetStatus(ctx)
+	}
 	// D1: agent self-update pending status (nil reporter → pending=false, the steady state).
 	if c.selfUpdate != nil {
 		report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending()
@@ -0,0 +1,80 @@
+package hub
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+)
+
+// R-54 §9 rule 6: the guest_net stanza must appear in a report built through the PRODUCTION collect
+// path, not only in a struct a test constructed. The v0.91.0 defect was exactly this gap — a seam
+// with green tests and no caller.
+
+type fakeGuestNet struct{ st *GuestNetStatus }
+
+func (f fakeGuestNet) GuestNetStatus(context.Context) *GuestNetStatus { return f.st }
+
+func TestCollect_GuestNetOmittedWhenReporterNil(t *testing.T) {
+	px := &fakePx{node: "n", ns: newTestNodeStatus()}
+	c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
+	r, err := c.Collect(context.Background())
+	if err != nil {
+		t.Fatalf("Collect: %v", err)
+	}
+	if r.GuestNet != nil {
+		t.Fatalf("no reporter wired → guest_net must be omitted, got %+v", r.GuestNet)
+	}
+	// And it must be absent from the WIRE, not merely nil in Go — an always-present empty stanza
+	// would make "watchdog not wired" indistinguishable from "watchdog found nothing".
+	b, _ := json.Marshal(r)
+	var m map[string]any
+	if err := json.Unmarshal(b, &m); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := m["guest_net"]; ok {
+		t.Fatalf("guest_net key present on the wire with no reporter wired: %s", b)
+	}
+}
+
+func TestCollect_GuestNetPopulatedWhenWired(t *testing.T) {
+	px := &fakePx{node: "n", ns: newTestNodeStatus()}
+	c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
+	c.SetGuestNetReporter(fakeGuestNet{st: &GuestNetStatus{
+		CheckedAt: "2026-07-21T10:00:00Z",
+		Guests: []GuestNetGuest{{
+			VMID: 9201, State: "healthy", Mode: "dhcp", IP: "192.168.0.104",
+			HasRoute: true, DHClientAlive: true, CheckedAt: "2026-07-21T10:00:00Z",
+		}},
+	}})
+	r, err := c.Collect(context.Background())
+	if err != nil {
+		t.Fatalf("Collect: %v", err)
+	}
+	if r.GuestNet == nil || len(r.GuestNet.Guests) != 1 {
+		t.Fatalf("guest_net stanza missing from a collected report: %+v", r.GuestNet)
+	}
+
+	// The wire keys are the contract the hub will read; pin the ones an operator diagnoses with.
+	b, _ := json.Marshal(r)
+	var m map[string]any
+	if err := json.Unmarshal(b, &m); err != nil {
+		t.Fatal(err)
+	}
+	gn, ok := m["guest_net"].(map[string]any)
+	if !ok {
+		t.Fatalf("guest_net missing or wrong shape on the wire: %s", b)
+	}
+	guests, ok := gn["guests"].([]any)
+	if !ok || len(guests) != 1 {
+		t.Fatalf("guest_net.guests wrong on the wire: %v", gn)
+	}
+	g := guests[0].(map[string]any)
+	for _, key := range []string{"vmid", "state", "mode", "ip", "has_route", "dhclient_alive"} {
+		if _, ok := g[key]; !ok {
+			t.Fatalf("guest_net.guests[0] is missing the %q key: %v", key, g)
+		}
+	}
+	if g["dhclient_alive"] != true {
+		t.Fatalf("dhclient_alive must survive the round trip: %v", g)
+	}
+}
@@ -85,6 +85,17 @@ type HostReport struct {
 	// Carries NO secret.
 	PBSDR *PBSDRStatus `json:"pbs_dr,omitempty"`

+	// GuestNet is the per-guest network-watchdog stanza (R-54). Present only when the guestnet
+	// watchdog is wired. Same additive/opaque contract as PBSDR and Wireguard above — no
+	// hub-schema change, absent when the reporter is not wired. Carries NO secret: addresses,
+	// route/liveness booleans, heal timestamps and counters only.
+	//
+	// NAMING NOTE (deliberate deviation from TASK-D, which called this block `WireGuestNet`):
+	// in this repo `Wire*` types are the DOWN direction (WireDesiredState/WirePBSDR — what the
+	// hub sends the agent), while UP-direction report stanzas are `*Status`. A `WireGuestNet`
+	// on HostReport would have been the only report block named against that convention.
+	GuestNet *GuestNetStatus `json:"guest_net,omitempty"`
+
 	// LogTail is the agent's on-demand debug-ring tail (v0.83.0 observability) — the agent
 	// mirror of the controller's report log_tails channel. Present ONLY on the heartbeat
 	// right after the control envelope requested it (log_tail_requested); consume-once on
@@ -109,6 +120,33 @@ type HostReport struct {
 // "verify_failed" (fingerprint/reachability pre-consume check failing — retrying, NOTHING
 // consumed), "consumed_failed" (LOUD: secret burned, apply failed, no auto-retry — operator
 // re-issue required), "disabled" (descriptor enabled:false). Carries no secret.
+// GuestNetStatus is the R-54 guest-network watchdog stanza. `guests` carries one entry per owned
+// RUNNING guest that has been probed at least once; an empty list with a fresh `checked_at` means
+// the watchdog ran and found nothing to report, which is deliberately distinguishable from the
+// stanza being absent (= the watchdog is not wired at all).
+type GuestNetStatus struct {
+	CheckedAt string          `json:"checked_at"` // RFC3339, the sweep this snapshot came from
+	Guests    []GuestNetGuest `json:"guests,omitempty"`
+}
+
+// GuestNetGuest mirrors guestnet.GuestReport on the wire. The two structs are deliberately separate:
+// internal/hub owns the wire contract and imports no feature package (the consumer-side seam rule).
+type GuestNetGuest struct {
+	VMID          int    `json:"vmid"`
+	State         string `json:"state"` // healthy | unhealthy | static_fault | unknown
+	Mode          string `json:"mode"`  // dhcp | static | unknown
+	IP            string `json:"ip,omitempty"`
+	HasRoute      bool   `json:"has_route"`
+	DHClientAlive bool   `json:"dhclient_alive"`
+	CheckedAt     string `json:"checked_at,omitempty"`
+	Healed        bool   `json:"healed,omitempty"`
+	HealSucceeded bool   `json:"heal_succeeded,omitempty"`
+	LastHealAt    string `json:"last_heal_at,omitempty"`
+	HealsLastHour int    `json:"heals_last_hour,omitempty"`
+	Damped        bool   `json:"damped,omitempty"`
+	Message       string `json:"message,omitempty"`
+}
+
 type PBSDRStatus struct {
 	State          string `json:"state"`
 	StorageID      string `json:"storage_id,omitempty"`