package guestnet import ( "bytes" "context" "errors" "io" "log/slog" "strings" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // --- fixtures captured LIVE from guest 9201 on 2026-07-21 (probe P3, via `ssh felhom-pve`) ------- // // These are the byte shapes the parser must survive; note the literal backslash `ip -o` emits and // the trailing space on the route line. const ( fxAddr = "2: eth0 inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\\ valid_lft 4916sec preferred_lft 4916sec\n" fxRoute = "default via 192.168.0.1 dev eth0 \n" fxPgrep = "235839\n" fxIfacesDHCP = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n" fxIfacesStat = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet static\n\taddress 192.168.0.162/24\n\tgateway 192.168.0.1\n" // pct exec against a guest that does not exist / is not running (rc=2, message on stderr). fxNoGuestErr = "Configuration file 'nodes/demo-felhom/lxc/9999.conf' does not exist\n" ) // scriptedRunner answers per probe kind and records EVERY argv. The counts are the assertions that // matter: a watchdog that heals when it must not is worse than one that never heals. type scriptedRunner struct { mu sync.Mutex out map[string]string // kind → stdout fail map[string]error // kind → error errs map[string]string // kind → stderr call [][]string // healFixes models what a successful dhclient actually does: the client is running again and // the lease is renewed. Set false to model a guest whose network is broken beyond dhclient. healFixes bool } func newRunner() *scriptedRunner { return &scriptedRunner{ out: map[string]string{ "addr": fxAddr, "route": fxRoute, "iface": fxIfacesDHCP, "pgrep": fxPgrep, "dhclient": "", }, fail: map[string]error{}, errs: map[string]string{}, healFixes: true, } } // kind classifies a `pct exec -- ...` argv. func kind(args []string) string { if len(args) < 4 || args[0] != "exec" { return "other" } rest := args[3:] switch rest[0] { case "ip": if len(rest) > 1 && rest[1] == "route" { return "route" } return "addr" case "cat": return "iface" case "pgrep": return "pgrep" case "dhclient": return "dhclient" } return "other" } func (r *scriptedRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { r.mu.Lock() defer r.mu.Unlock() r.call = append(r.call, append([]string{name}, args...)) k := kind(args) if k == "dhclient" && r.fail[k] == nil && r.healFixes { // a real dhclient re-acquires the lease and stays resident r.out["pgrep"], r.out["addr"], r.out["route"] = fxPgrep, fxAddr, fxRoute delete(r.fail, "pgrep") } return []byte(r.out[k]), []byte(r.errs[k]), r.fail[k] } func (r *scriptedRunner) countOf(k string) int { r.mu.Lock() defer r.mu.Unlock() n := 0 for _, c := range r.call { if len(c) > 1 && kind(c[1:]) == k { n++ } } return n } func (r *scriptedRunner) lastOf(k string) []string { r.mu.Lock() defer r.mu.Unlock() for i := len(r.call) - 1; i >= 0; i-- { if len(r.call[i]) > 1 && kind(r.call[i][1:]) == k { return r.call[i] } } return nil } // killDHClient models the exact 2026-07-20 state: lease still valid (address AND route present), // dhclient gone. This is the fixture the whole feature exists for. func (r *scriptedRunner) killDHClient() { r.mu.Lock() defer r.mu.Unlock() r.out["pgrep"] = "" r.fail["pgrep"] = errors.New("exit status 1") // pgrep: no match, EMPTY stderr } func (r *scriptedRunner) reviveDHClient() { r.mu.Lock() defer r.mu.Unlock() r.out["pgrep"] = fxPgrep delete(r.fail, "pgrep") } // expireLease models the state 80 minutes later: address and route gone too. func (r *scriptedRunner) expireLease() { r.mu.Lock() defer r.mu.Unlock() r.out["addr"] = "" r.out["route"] = "" } type fakeGuests struct { guests []proxmox.Guest err error } func (f *fakeGuests) Guests(context.Context) ([]proxmox.Guest, error) { return f.guests, f.err } func running9201() *fakeGuests { return &fakeGuests{guests: []proxmox.Guest{ {VMID: 9201, Name: "felhom-demo", Status: "running", Type: "lxc", Uptime: 7200}, }} } // newTestWatchdog wires a watchdog with a manual clock the test advances, and captures the log so // the "healthy cycles are observable" contract can be asserted rather than assumed. func newTestWatchdog(r Runner, g GuestSource) (*Watchdog, *time.Time, *bytes.Buffer) { clock := time.Unix(1_784_000_000, 0).UTC() buf := &bytes.Buffer{} logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) w := New(r, g, time.Minute, logger) w.now = func() time.Time { return clock } // The agent's own settle window is measured from startedAt, which New stamped with the REAL // clock; restamp it against the fake one, well in the past. w.startedAt = clock.Add(-time.Hour) return w, &clock, buf } // --- Scenario E: the TIMED failure, detected instantly ------------------------------------------ // // RED-PROOF (recorded in REPORT.md): reverting classify()'s dhcp arm to IP-presence-only — // // case ModeDHCP: // if p.IP == "" { return StateUnhealthy, ... } // return StateHealthy, ... // // makes TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy report "healthy" for the July-20 fixture, and // TestWatchdog_HealsTheIncidentState records ZERO heals. That is the 80-minute silent window, exactly // as it happened. func TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy(t *testing.T) { r := newRunner() r.killDHClient() w, _, _ := newTestWatchdog(r, running9201()) p := w.probe(context.Background(), 9201) if !p.Reachable { t.Fatalf("guest must read as reachable: %+v", p) } if p.IP != "192.168.0.104" || !p.HasRoute { t.Fatalf("the lease is still live in this fixture — IP/route must be present: %+v", p) } if p.DHCPAlive { t.Fatalf("dhclient must read as dead: %+v", p) } state, detail := classify(p) if state != StateUnhealthy { t.Fatalf("classify = %q, want %q — waiting for the IP to vanish is the 80-minute silent "+ "window the incident proved (detail: %s)", state, StateUnhealthy, detail) } if !strings.Contains(detail, "dhclient") { t.Fatalf("the reason must name the dead client, got %q", detail) } } func TestWatchdog_HealsTheIncidentState(t *testing.T) { r := newRunner() r.killDHClient() w, clock, logBuf := newTestWatchdog(r, running9201()) ctx := context.Background() // Cycle 1: unhealthy, but one bad probe is not a diagnosis. w.Tick(ctx) if n := r.countOf("dhclient"); n != 0 { t.Fatalf("healed after ONE bad probe (%d heals) — a single blip must never trigger a heal", n) } // Cycle 2: second consecutive bad probe → heal. The heal makes the client live again. *clock = clock.Add(time.Minute) w.Tick(ctx) if n := r.countOf("dhclient"); n != 1 { t.Fatalf("heal ran %d times, want exactly 1", n) } // The invocation must be the incident's, verbatim. want := []string{"pct", "exec", "9201", "--", "dhclient", "-pf", "/run/dhclient.eth0.pid", "-lf", "/var/lib/dhcp/dhclient.eth0.leases", "eth0"} got := r.lastOf("dhclient") if len(got) != len(want) { t.Fatalf("heal argv = %v, want %v", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("heal argv[%d] = %q, want %q (full: %v)", i, got[i], want[i], got) } } // The report must show the heal AND the verified-healthy re-probe. snap := w.Snapshot() if len(snap) != 1 { t.Fatalf("snapshot = %+v, want one guest", snap) } g := snap[0] if !g.Healed || !g.HealSucceeded { t.Fatalf("report must record a successful heal: %+v", g) } if g.State != string(StateHealthy) || g.LastHealAt == "" || g.HealsLastHour != 1 { t.Fatalf("post-heal report is wrong: %+v", g) } if !strings.Contains(logBuf.String(), "guestnet: guest network healed") { t.Fatalf("the heal was not logged: %s", logBuf.String()) } } // A healthy box must be silent about alarms but NOT silent about having looked. func TestWatchdog_HealthyCycleProbesAndNeverHeals(t *testing.T) { r := newRunner() w, _, logBuf := newTestWatchdog(r, running9201()) w.Tick(context.Background()) if n := r.countOf("dhclient"); n != 0 { t.Fatalf("a healthy guest was healed %d times, want 0", n) } if r.countOf("pgrep") != 1 || r.countOf("addr") != 1 { t.Fatalf("the healthy path must still probe: %v", r.call) } if !strings.Contains(logBuf.String(), "guest network healthy") { t.Fatalf("a healthy cycle must be observable — otherwise 'no alarms' and 'never probed' "+ "are the same evidence (v0.91.2's lesson). Log: %s", logBuf.String()) } snap := w.Snapshot() if len(snap) != 1 || snap[0].State != string(StateHealthy) || !snap[0].DHClientAlive { t.Fatalf("healthy snapshot wrong: %+v", snap) } } // --- Scenario F: configuration and damping guards ------------------------------------------------ func TestWatchdog_StaticGuestIsNeverHealedWithDHClient(t *testing.T) { r := newRunner() r.out["iface"] = fxIfacesStat r.killDHClient() // on a static guest this is NORMAL w, clock, _ := newTestWatchdog(r, running9201()) for i := 0; i < 5; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("a static guest was healed with dhclient %d times, want 0", n) } if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateHealthy) { t.Fatalf("a static guest with address+route is healthy, got %+v", s) } } func TestWatchdog_StaticGuestMissingAddressReportsButNeverHeals(t *testing.T) { r := newRunner() r.out["iface"] = fxIfacesStat r.expireLease() // no address, no route on a STATIC guest → R-50 territory, not ours w, clock, logBuf := newTestWatchdog(r, running9201()) for i := 0; i < 5; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("healed a static guest %d times, want 0 — dhclient must never fight a static config", n) } s := w.Snapshot() if len(s) != 1 || s[0].State != string(StateStaticFault) { t.Fatalf("state = %+v, want static_fault", s) } if !strings.Contains(logBuf.String(), "not actionable") { t.Fatalf("a static fault must be reported loudly: %s", logBuf.String()) } // Loud ONCE per transition, not once per cycle. if n := strings.Count(logBuf.String(), "not actionable"); n != 1 { t.Fatalf("static fault logged %d times over 5 cycles, want 1 (per transition)", n) } } func TestWatchdog_UnreachableGuestIsUnknownAndNeverHealed(t *testing.T) { r := newRunner() r.fail["addr"] = errors.New("exit status 2") r.errs["addr"] = fxNoGuestErr w, clock, _ := newTestWatchdog(r, running9201()) for i := 0; i < 4; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("healed a guest we could not probe %d times, want 0 — acting blind is how the "+ "incident happened", n) } if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) { t.Fatalf("state = %+v, want unknown", s) } } // A missing pgrep (or any probe tool) must read as unknown, never as a dead client — otherwise a // broken probe would heal forever. func TestWatchdog_ProbeToolFailureIsUnknownNotDead(t *testing.T) { r := newRunner() r.fail["pgrep"] = errors.New("exit status 127") r.errs["pgrep"] = "pgrep: command not found\n" w, clock, _ := newTestWatchdog(r, running9201()) for i := 0; i < 4; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("a failed liveness probe caused %d heals, want 0", n) } if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) { t.Fatalf("state = %+v, want unknown", s) } } func TestWatchdog_DampingCeilings(t *testing.T) { r := newRunner() r.killDHClient() r.healFixes = false // the heal "works" but the client dies again immediately w, clock, _ := newTestWatchdog(r, running9201()) ctx := context.Background() // 10 hours of one-minute cycles against a permanently broken guest. for i := 0; i < 600; i++ { w.Tick(ctx) *clock = clock.Add(time.Minute) } heals := r.countOf("dhclient") // Ceiling: 3 per hour AND ≥10 min apart ⇒ at most 3 in any rolling hour. Over 10 h the // min-interval rule dominates: 6 slots/hour capped to 3/hour ⇒ ≤ 30. if heals > 30 { t.Fatalf("heals = %d over 10 h, want ≤ 30 (≤3/hour) — the damper is not holding", heals) } if heals == 0 { t.Fatalf("heals = 0 — the damper has become a mute") } } func TestWatchdog_MinimumIntervalBetweenHeals(t *testing.T) { r := newRunner() r.killDHClient() r.healFixes = false w, clock, _ := newTestWatchdog(r, running9201()) ctx := context.Background() w.Tick(ctx) // bad probe 1 *clock = clock.Add(time.Minute) w.Tick(ctx) // bad probe 2 → heal #1 if r.countOf("dhclient") != 1 { t.Fatalf("expected exactly one heal by now, got %d", r.countOf("dhclient")) } // Nine more minutes of failure: still inside the 10-minute cool-off. for i := 0; i < 9; i++ { *clock = clock.Add(time.Minute) w.Tick(ctx) } if n := r.countOf("dhclient"); n != 1 { t.Fatalf("heals = %d within the 10-minute cool-off, want 1", n) } if s := w.Snapshot(); len(s) != 1 || !s[0].Damped { t.Fatalf("a damped cycle must say so in the report: %+v", s) } // Past the cool-off, one more heal is allowed. *clock = clock.Add(2 * time.Minute) w.Tick(ctx) if n := r.countOf("dhclient"); n != 2 { t.Fatalf("heals = %d after the cool-off expired, want 2", n) } } func TestWatchdog_BootRacesObserveOnly(t *testing.T) { t.Run("young guest", func(t *testing.T) { r := newRunner() r.killDHClient() g := running9201() g.guests[0].Uptime = 40 // seconds w, clock, _ := newTestWatchdog(r, g) for i := 0; i < 4; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("healed a guest that booted 40 s ago %d times, want 0 — it has no lease YET", n) } }) t.Run("young agent", func(t *testing.T) { r := newRunner() r.killDHClient() w, clock, _ := newTestWatchdog(r, running9201()) w.startedAt = *clock // the agent just started for i := 0; i < 2; i++ { w.Tick(context.Background()) *clock = clock.Add(time.Minute) } if n := r.countOf("dhclient"); n != 0 { t.Fatalf("healed %d times within the agent's own settle window, want 0", n) } }) } func TestWatchdog_StoppedGuestIsNotProbed(t *testing.T) { r := newRunner() g := running9201() g.guests[0].Status = "stopped" w, _, _ := newTestWatchdog(r, g) w.Tick(context.Background()) if len(r.call) != 0 { t.Fatalf("a stopped guest was probed: %v", r.call) } } func TestWatchdog_GuestListFailureSkipsTheSweep(t *testing.T) { r := newRunner() w, _, logBuf := newTestWatchdog(r, &fakeGuests{err: errors.New("pool membership read: 403")}) w.Tick(context.Background()) if len(r.call) != 0 { t.Fatalf("acted with unproven ownership: %v", r.call) } if !strings.Contains(logBuf.String(), "ownership unproven") { t.Fatalf("the skip must be logged: %s", logBuf.String()) } } // A transient bad probe followed by recovery must never heal, and must log the recovery. func TestWatchdog_SingleBlipNeverHeals(t *testing.T) { r := newRunner() w, clock, logBuf := newTestWatchdog(r, running9201()) ctx := context.Background() w.Tick(ctx) // healthy r.killDHClient() *clock = clock.Add(time.Minute) w.Tick(ctx) // bad probe 1 r.reviveDHClient() *clock = clock.Add(time.Minute) w.Tick(ctx) // healthy again if n := r.countOf("dhclient"); n != 0 { t.Fatalf("a single blip caused %d heals, want 0", n) } if !strings.Contains(logBuf.String(), "guest network recovered") { t.Fatalf("the recovery must be visible: %s", logBuf.String()) } } // --- parsers over the live P3 fixtures ------------------------------------------------------------ func TestParsers_OverLiveFixtures(t *testing.T) { if got := parseInet(fxAddr); got != "192.168.0.104" { t.Fatalf("parseInet = %q, want 192.168.0.104", got) } if got := parseInet(""); got != "" { t.Fatalf("parseInet(empty) = %q, want empty (the post-expiry state prints nothing)", got) } if !hasDefaultRoute(fxRoute) { t.Fatalf("hasDefaultRoute(%q) = false", fxRoute) } if hasDefaultRoute("") || hasDefaultRoute("172.17.0.0/16 dev docker0 proto kernel scope link\n") { t.Fatal("docker bridge routes must not read as a default route (the incident's exact leftovers)") } if got := parseMode(fxIfacesDHCP, "eth0"); got != ModeDHCP { t.Fatalf("parseMode(dhcp) = %q", got) } if got := parseMode(fxIfacesStat, "eth0"); got != ModeStatic { t.Fatalf("parseMode(static) = %q", got) } if got := parseMode("auto lo\niface lo inet loopback\n", "eth0"); got != ModeUnknown { t.Fatalf("parseMode(no eth0 stanza) = %q, want unknown", got) } if got := parseMode("iface eth0 inet manual\n", "eth0"); got != ModeUnknown { t.Fatalf("parseMode(manual) = %q, want unknown", got) } } func TestClassify_Table(t *testing.T) { cases := []struct { name string p Probe want State }{ {"dhcp all green", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true, DHCPAlive: true}, StateHealthy}, {"dhcp dead client, live lease", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true}, StateUnhealthy}, {"dhcp no address", Probe{Reachable: true, Mode: ModeDHCP, DHCPAlive: true}, StateUnhealthy}, {"dhcp no route", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", DHCPAlive: true}, StateUnhealthy}, {"static green", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy}, {"static no client is normal", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy}, {"static broken", Probe{Reachable: true, Mode: ModeStatic}, StateStaticFault}, {"unreachable", Probe{Mode: ModeDHCP}, StateUnknown}, {"unknown mode", Probe{Reachable: true, Mode: ModeUnknown, IP: "1.2.3.4", HasRoute: true}, StateUnknown}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got, _ := classify(tc.p); got != tc.want { t.Fatalf("classify = %q, want %q", got, tc.want) } }) } } var _ io.Writer = (*bytes.Buffer)(nil) // A3 (R-50): the guestnet healer is eth0-only and MUST stay blind to the island NIC. A guest on an // island host presents eth0 DHCP (the LAN leg the healer owns) PLUS eth1 static (the island). Because // parseMode is interface-scoped, adding eth1 static cannot flip eth0's detected mode — so the healer // keeps treating eth0 as DHCP and never runs dhclient against the static island NIC (which would // sabotage it). This is the verify-only guarantee that let R-50 ship the island NIC without a healer // change. Red-proof: make parseMode scan globally instead of per-dev and the eth0 assertion fails. func TestParseMode_IslandStaticNICDoesNotConfuseEth0(t *testing.T) { interfaces := "auto lo\niface lo inet loopback\n\n" + "auto eth0\niface eth0 inet dhcp\n\n" + "auto eth1\niface eth1 inet static\n address 169.254.253.2/30\n" if got := parseMode(interfaces, "eth0"); got != ModeDHCP { t.Errorf("eth0 must classify DHCP even with an island eth1 static present, got %q", got) } if got := parseMode(interfaces, "eth1"); got != ModeStatic { t.Errorf("eth1 (island) must classify static when asked directly (dev-scoped), got %q", got) } }