feat(v0.92.0): guest-network watchdog (R-54) — supervise the guest's DHCP client
Closes the OPEN RISK in INCIDENT-guest-dhclient-killed-2026-07-20 §5. The guest's dhclient
is started once by ifupdown at boot and nothing supervises it; when it died on 2026-07-20
the guest ran another ~80 minutes on its unexpired lease, then lost its address and default
route and took the tunnel, hub reports, catalog sync and the controller->agent channel with
it (1h15m outage, healthy-looking for the first 80 minutes).
So liveness of the DHCP client is itself a probe: a DHCP guest is unhealthy the moment
`pgrep -x dhclient` comes back empty, while the lease is still live. Waiting for the address
to vanish is waiting out the silent window.
internal/guestnet: four fixed-shape pct exec probes (address, default route, interfaces
mode, dhclient liveness — parsers pinned to output captured live from 9201), the incident's
heal invocation verbatim, and dampers throughout: two consecutive bad probes, >=10 min
between heals, <=3/hour, observe-only while guest or agent uptime < 3 min. Refuses to act on
a static guest, an unknown mode, an unprobeable guest, or an unproven guest list (the source
is the pool-verified ListLXC ∩ felhom pool, never a bare ListLXC). A failed probe reads as
unknown, never as a dead client. Healthy cycles log a Debug line so "no alarms" and "never
probed" stay distinguishable. Not in the errc fan-out — a guest watchdog must never be able
to kill the agent.
guest_net is the repo's first default-ON gate (opt-out is `{"disable": true}`): it looks only
inward at guests we already own, and the failure exists on every box today.
Report block ships as GuestNetStatus, not the spec's WireGuestNet: Wire* is the DOWN
direction in this repo, report stanzas are *Status.
Red-proofs: classify reverted to IP-presence-only -> the July-20 fixture reports "healthy"
with zero heals; un-wiring the reporter and the goroutine fails the AST wiring test.
Also: `var version` was stale at 0.89.0 (ldflags hid it; `go run` did not).
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-54 §9 rule 6 — the seam-discipline test, and the one this repo has the most reason to write:
|
||||
// v0.91.0 shipped the PBS auth-probe seam with `main.go` never calling `SetAuthSink`, every unit
|
||||
// test green because they all injected the seam directly. The guestnet watchdog has the identical
|
||||
// shape (a component + a reporter seam + a goroutine), so its wiring is asserted here rather than
|
||||
// trusted.
|
||||
//
|
||||
// This walks the AST rather than grepping the source: a commented-out call still satisfies a
|
||||
// substring match (found while red-proofing the controller's twin of this test), and a comment is
|
||||
// not a caller.
|
||||
func TestMainWiresGuestNetWatchdog(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
|
||||
var constructed, reporterWired, started bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
switch fn := node.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
switch fn.Sel.Name {
|
||||
case "New":
|
||||
// guestnet.New(...)
|
||||
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "guestnet" {
|
||||
constructed = true
|
||||
}
|
||||
case "SetGuestNetReporter":
|
||||
reporterWired = true
|
||||
}
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !constructed {
|
||||
t.Error("main.go never calls guestnet.New — the watchdog does not exist at runtime")
|
||||
}
|
||||
if !reporterWired {
|
||||
t.Error("main.go never calls collector.SetGuestNetReporter — the guest_net stanza would " +
|
||||
"never reach the hub (the exact v0.91.0 inert-seam defect)")
|
||||
}
|
||||
if !started {
|
||||
t.Error("main.go never starts the watchdog with `go gnWatchdog.Watch(ctx)` — it would be " +
|
||||
"constructed, reported on, and never probe anything")
|
||||
}
|
||||
}
|
||||
|
||||
// The watchdog must NOT join the errc fan-out: a guest-network watchdog that can terminate the
|
||||
// agent turns a customer's DHCP problem into an operator-plane outage. If it is ever changed to
|
||||
// `errc <- ...`, the drain bound at the bottom of main() also has to change — this catches the
|
||||
// first half of that mistake.
|
||||
func TestGuestNetWatchdogIsNotInTheErrcFanout(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
bad := false
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
send, ok := n.(*ast.SendStmt)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if ch, ok := send.Chan.(*ast.Ident); !ok || ch.Name != "errc" {
|
||||
return true
|
||||
}
|
||||
call, ok := send.Value.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
bad = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if bad {
|
||||
t.Fatal("the guestnet watchdog was added to the errc fan-out — a watchdog over customer " +
|
||||
"guests must never be able to bring the agent down (and the drain bound in main() " +
|
||||
"would now be off by one, hanging shutdown)")
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/fasttick"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/guestnet"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
@@ -57,7 +58,7 @@ import (
|
||||
|
||||
// version is the agent version. Overridable at build time with
|
||||
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
||||
var version = "0.89.0"
|
||||
var version = "0.92.0"
|
||||
|
||||
// runGuestHook is the PVE hook body (`felhom-agent guest-hook <vmid> <phase>`). On pre-start it
|
||||
// creates placeholder dirs for any absent bind-mount source so the guest always boots (the C1 net);
|
||||
@@ -1020,6 +1021,32 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
|
||||
go shMgr.Watch(ctx)
|
||||
}
|
||||
// R-54: guest-network watchdog. Closes the OPEN RISK left by
|
||||
// INCIDENT-guest-dhclient-killed-2026-07-20 §5 — the guest's DHCP client is started once by
|
||||
// ifupdown at boot and nothing supervises it, so its death takes the box off the internet
|
||||
// ~1-2 h later, when the lease expires. Host-tier by necessity: a guest with no default
|
||||
// route cannot repair its own default route. Deliberately NOT part of the errc fan-out — a
|
||||
// watchdog over customer guests must never be able to bring the agent down.
|
||||
if gn := cfg.GuestNet.WithDefaults(); gn.Enabled() {
|
||||
gnMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||
if gnMode == "" {
|
||||
gnMode = proxmox.RunnerSudo
|
||||
}
|
||||
gnRunner := &proxmox.ExecRunner{Mode: gnMode, SudoPath: cfg.Privileged.SudoPath}
|
||||
// The guest source is the POOL-VERIFIED one (ListLXC ∩ felhom pool, audit A1) — never a
|
||||
// bare ListLXC, which under a broad token would run dhclient inside a co-tenant's guest.
|
||||
gnGuests := localapi.NewStaleLockController(px, gnRunner, reconcile.DefaultPool, logger)
|
||||
gnWatchdog := guestnet.New(gnRunner, gnGuests, time.Duration(gn.IntervalSeconds)*time.Second, logger)
|
||||
gnWatchdog.SetDampers(
|
||||
time.Duration(gn.MinHealIntervalSeconds)*time.Second,
|
||||
gn.MaxHealsPerHour,
|
||||
time.Duration(gn.SettleSeconds)*time.Second,
|
||||
)
|
||||
collector.SetGuestNetReporter(gnWatchdog)
|
||||
go gnWatchdog.Watch(ctx)
|
||||
} else {
|
||||
logger.Info("guestnet: watchdog disabled by config (guest_net.disable)")
|
||||
}
|
||||
// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
|
||||
localSrv.RecoverFormatJob(ctx)
|
||||
// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete
|
||||
|
||||
Reference in New Issue
Block a user