feat(v0.92.0): guest-network watchdog (R-54) — supervise the guest's DHCP client

Closes the OPEN RISK in INCIDENT-guest-dhclient-killed-2026-07-20 §5. The guest's dhclient
is started once by ifupdown at boot and nothing supervises it; when it died on 2026-07-20
the guest ran another ~80 minutes on its unexpired lease, then lost its address and default
route and took the tunnel, hub reports, catalog sync and the controller->agent channel with
it (1h15m outage, healthy-looking for the first 80 minutes).

So liveness of the DHCP client is itself a probe: a DHCP guest is unhealthy the moment
`pgrep -x dhclient` comes back empty, while the lease is still live. Waiting for the address
to vanish is waiting out the silent window.

internal/guestnet: four fixed-shape pct exec probes (address, default route, interfaces
mode, dhclient liveness — parsers pinned to output captured live from 9201), the incident's
heal invocation verbatim, and dampers throughout: two consecutive bad probes, >=10 min
between heals, <=3/hour, observe-only while guest or agent uptime < 3 min. Refuses to act on
a static guest, an unknown mode, an unprobeable guest, or an unproven guest list (the source
is the pool-verified ListLXC ∩ felhom pool, never a bare ListLXC). A failed probe reads as
unknown, never as a dead client. Healthy cycles log a Debug line so "no alarms" and "never
probed" stay distinguishable. Not in the errc fan-out — a guest watchdog must never be able
to kill the agent.

guest_net is the repo's first default-ON gate (opt-out is `{"disable": true}`): it looks only
inward at guests we already own, and the failure exists on every box today.

Report block ships as GuestNetStatus, not the spec's WireGuestNet: Wire* is the DOWN
direction in this repo, report stanzas are *Status.

Red-proofs: classify reverted to IP-presence-only -> the July-20 fixture reports "healthy"
with zero heals; un-wiring the reporter and the goroutine fails the AST wiring test.

Also: `var version` was stale at 0.89.0 (ldflags hid it; `go run` did not).
This commit is contained in:
2026-07-21 12:29:37 +02:00
parent 08b55a1015
commit c0966d753d
12 changed files with 1575 additions and 1 deletions
+100
View File
@@ -0,0 +1,100 @@
package main
import (
"go/ast"
"go/parser"
"go/token"
"testing"
)
// R-54 §9 rule 6 — the seam-discipline test, and the one this repo has the most reason to write:
// v0.91.0 shipped the PBS auth-probe seam with `main.go` never calling `SetAuthSink`, every unit
// test green because they all injected the seam directly. The guestnet watchdog has the identical
// shape (a component + a reporter seam + a goroutine), so its wiring is asserted here rather than
// trusted.
//
// This walks the AST rather than grepping the source: a commented-out call still satisfies a
// substring match (found while red-proofing the controller's twin of this test), and a comment is
// not a caller.
func TestMainWiresGuestNetWatchdog(t *testing.T) {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "main.go", nil, 0)
if err != nil {
t.Fatalf("parse main.go: %v", err)
}
var constructed, reporterWired, started bool
ast.Inspect(f, func(n ast.Node) bool {
switch node := n.(type) {
case *ast.CallExpr:
switch fn := node.Fun.(type) {
case *ast.SelectorExpr:
switch fn.Sel.Name {
case "New":
// guestnet.New(...)
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "guestnet" {
constructed = true
}
case "SetGuestNetReporter":
reporterWired = true
}
}
case *ast.GoStmt:
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
started = true
}
}
}
return true
})
if !constructed {
t.Error("main.go never calls guestnet.New — the watchdog does not exist at runtime")
}
if !reporterWired {
t.Error("main.go never calls collector.SetGuestNetReporter — the guest_net stanza would " +
"never reach the hub (the exact v0.91.0 inert-seam defect)")
}
if !started {
t.Error("main.go never starts the watchdog with `go gnWatchdog.Watch(ctx)` — it would be " +
"constructed, reported on, and never probe anything")
}
}
// The watchdog must NOT join the errc fan-out: a guest-network watchdog that can terminate the
// agent turns a customer's DHCP problem into an operator-plane outage. If it is ever changed to
// `errc <- ...`, the drain bound at the bottom of main() also has to change — this catches the
// first half of that mistake.
func TestGuestNetWatchdogIsNotInTheErrcFanout(t *testing.T) {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "main.go", nil, 0)
if err != nil {
t.Fatalf("parse main.go: %v", err)
}
bad := false
ast.Inspect(f, func(n ast.Node) bool {
send, ok := n.(*ast.SendStmt)
if !ok {
return true
}
if ch, ok := send.Chan.(*ast.Ident); !ok || ch.Name != "errc" {
return true
}
call, ok := send.Value.(*ast.CallExpr)
if !ok {
return true
}
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
bad = true
}
}
return true
})
if bad {
t.Fatal("the guestnet watchdog was added to the errc fan-out — a watchdog over customer " +
"guests must never be able to bring the agent down (and the drain bound in main() " +
"would now be off by one, hanging shutdown)")
}
}
+28 -1
View File
@@ -37,6 +37,7 @@ import (
"gitea.dooplex.hu/admin/felhom-agent/internal/fasttick"
"gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd"
"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
"gitea.dooplex.hu/admin/felhom-agent/internal/guestnet"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
@@ -57,7 +58,7 @@ import (
// version is the agent version. Overridable at build time with
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
var version = "0.89.0"
var version = "0.92.0"
// runGuestHook is the PVE hook body (`felhom-agent guest-hook <vmid> <phase>`). On pre-start it
// creates placeholder dirs for any absent bind-mount source so the guest always boots (the C1 net);
@@ -1020,6 +1021,32 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
go shMgr.Watch(ctx)
}
// R-54: guest-network watchdog. Closes the OPEN RISK left by
// INCIDENT-guest-dhclient-killed-2026-07-20 §5 — the guest's DHCP client is started once by
// ifupdown at boot and nothing supervises it, so its death takes the box off the internet
// ~1-2 h later, when the lease expires. Host-tier by necessity: a guest with no default
// route cannot repair its own default route. Deliberately NOT part of the errc fan-out — a
// watchdog over customer guests must never be able to bring the agent down.
if gn := cfg.GuestNet.WithDefaults(); gn.Enabled() {
gnMode := proxmox.RunnerMode(cfg.Privileged.Mode)
if gnMode == "" {
gnMode = proxmox.RunnerSudo
}
gnRunner := &proxmox.ExecRunner{Mode: gnMode, SudoPath: cfg.Privileged.SudoPath}
// The guest source is the POOL-VERIFIED one (ListLXC ∩ felhom pool, audit A1) — never a
// bare ListLXC, which under a broad token would run dhclient inside a co-tenant's guest.
gnGuests := localapi.NewStaleLockController(px, gnRunner, reconcile.DefaultPool, logger)
gnWatchdog := guestnet.New(gnRunner, gnGuests, time.Duration(gn.IntervalSeconds)*time.Second, logger)
gnWatchdog.SetDampers(
time.Duration(gn.MinHealIntervalSeconds)*time.Second,
gn.MaxHealsPerHour,
time.Duration(gn.SettleSeconds)*time.Second,
)
collector.SetGuestNetReporter(gnWatchdog)
go gnWatchdog.Watch(ctx)
} else {
logger.Info("guestnet: watchdog disabled by config (guest_net.disable)")
}
// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
localSrv.RecoverFormatJob(ctx)
// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete