admin
c0966d753d
Closes the OPEN RISK in INCIDENT-guest-dhclient-killed-2026-07-20 §5. The guest's dhclient
is started once by ifupdown at boot and nothing supervises it; when it died on 2026-07-20
the guest ran another ~80 minutes on its unexpired lease, then lost its address and default
route and took the tunnel, hub reports, catalog sync and the controller->agent channel with
it (1h15m outage, healthy-looking for the first 80 minutes).
So liveness of the DHCP client is itself a probe: a DHCP guest is unhealthy the moment
`pgrep -x dhclient` comes back empty, while the lease is still live. Waiting for the address
to vanish is waiting out the silent window.
internal/guestnet: four fixed-shape pct exec probes (address, default route, interfaces
mode, dhclient liveness — parsers pinned to output captured live from 9201), the incident's
heal invocation verbatim, and dampers throughout: two consecutive bad probes, >=10 min
between heals, <=3/hour, observe-only while guest or agent uptime < 3 min. Refuses to act on
a static guest, an unknown mode, an unprobeable guest, or an unproven guest list (the source
is the pool-verified ListLXC ∩ felhom pool, never a bare ListLXC). A failed probe reads as
unknown, never as a dead client. Healthy cycles log a Debug line so "no alarms" and "never
probed" stay distinguishable. Not in the errc fan-out — a guest watchdog must never be able
to kill the agent.
guest_net is the repo's first default-ON gate (opt-out is `{"disable": true}`): it looks only
inward at guests we already own, and the failure exists on every box today.
Report block ships as GuestNetStatus, not the spec's WireGuestNet: Wire* is the DOWN
direction in this repo, report stanzas are *Status.
Red-proofs: classify reverted to IP-presence-only -> the July-20 fixture reports "healthy"
with zero heals; un-wiring the reporter and the goroutine fails the AST wiring test.
Also: `var version` was stale at 0.89.0 (ldflags hid it; `go run` did not).
81 lines
2.7 KiB
Go
81 lines
2.7 KiB
Go
package hub
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"testing"
|
|
)
|
|
|
|
// R-54 §9 rule 6: the guest_net stanza must appear in a report built through the PRODUCTION collect
|
|
// path, not only in a struct a test constructed. The v0.91.0 defect was exactly this gap — a seam
|
|
// with green tests and no caller.
|
|
|
|
type fakeGuestNet struct{ st *GuestNetStatus }
|
|
|
|
func (f fakeGuestNet) GuestNetStatus(context.Context) *GuestNetStatus { return f.st }
|
|
|
|
func TestCollect_GuestNetOmittedWhenReporterNil(t *testing.T) {
|
|
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
|
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
|
|
r, err := c.Collect(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Collect: %v", err)
|
|
}
|
|
if r.GuestNet != nil {
|
|
t.Fatalf("no reporter wired → guest_net must be omitted, got %+v", r.GuestNet)
|
|
}
|
|
// And it must be absent from the WIRE, not merely nil in Go — an always-present empty stanza
|
|
// would make "watchdog not wired" indistinguishable from "watchdog found nothing".
|
|
b, _ := json.Marshal(r)
|
|
var m map[string]any
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := m["guest_net"]; ok {
|
|
t.Fatalf("guest_net key present on the wire with no reporter wired: %s", b)
|
|
}
|
|
}
|
|
|
|
func TestCollect_GuestNetPopulatedWhenWired(t *testing.T) {
|
|
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
|
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
|
|
c.SetGuestNetReporter(fakeGuestNet{st: &GuestNetStatus{
|
|
CheckedAt: "2026-07-21T10:00:00Z",
|
|
Guests: []GuestNetGuest{{
|
|
VMID: 9201, State: "healthy", Mode: "dhcp", IP: "192.168.0.104",
|
|
HasRoute: true, DHClientAlive: true, CheckedAt: "2026-07-21T10:00:00Z",
|
|
}},
|
|
}})
|
|
r, err := c.Collect(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Collect: %v", err)
|
|
}
|
|
if r.GuestNet == nil || len(r.GuestNet.Guests) != 1 {
|
|
t.Fatalf("guest_net stanza missing from a collected report: %+v", r.GuestNet)
|
|
}
|
|
|
|
// The wire keys are the contract the hub will read; pin the ones an operator diagnoses with.
|
|
b, _ := json.Marshal(r)
|
|
var m map[string]any
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gn, ok := m["guest_net"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("guest_net missing or wrong shape on the wire: %s", b)
|
|
}
|
|
guests, ok := gn["guests"].([]any)
|
|
if !ok || len(guests) != 1 {
|
|
t.Fatalf("guest_net.guests wrong on the wire: %v", gn)
|
|
}
|
|
g := guests[0].(map[string]any)
|
|
for _, key := range []string{"vmid", "state", "mode", "ip", "has_route", "dhclient_alive"} {
|
|
if _, ok := g[key]; !ok {
|
|
t.Fatalf("guest_net.guests[0] is missing the %q key: %v", key, g)
|
|
}
|
|
}
|
|
if g["dhclient_alive"] != true {
|
|
t.Fatalf("dhclient_alive must survive the round trip: %v", g)
|
|
}
|
|
}
|