guest-power: add the liveness observable it shipped without (v0.109.0)

The v0.107.0 watchdog was silent on a healthy box, so its health could only be inferred
from absence — F-OBS's shape, shipped in the same session F-OBS was fixed. INFO summary
every 10th sweep with what it saw; aborted sweeps are not counted. Red-proofs 7 and 8.
This commit is contained in:
2026-07-28 11:14:56 +02:00
parent 8db92947cd
commit f27f7a2659
4 changed files with 184 additions and 1 deletions
+32
View File
@@ -2,6 +2,7 @@ package localapi
import (
"context"
"log/slog"
"sync"
"time"
@@ -53,6 +54,16 @@ const (
// guestPowerMaxAttempts bounds the retry. A guest that will not start must not be started in a
// loop forever (Scenario C) — after this many failures the watchdog stops trying and raises it.
guestPowerMaxAttempts = 3
// guestPowerHeartbeatEvery emits a summary line every Nth sweep. 10 x 60s = 10 minutes, matching
// the controller's deadapp heartbeat.
//
// WHY THIS EXISTS, and it is a correction to this file's OWN first version (v0.107.0): the
// watchdog logged at startup and when it ACTED, and was otherwise silent. A silent watchdog is
// indistinguishable from a dead one — which is F-OBS, the very finding fixed in the same session
// this file shipped in, and it is what standing rule 3 exists to prevent. An operator needs a
// POSITIVE observable that the sweep is running; "no start lines" must not be the only evidence.
guestPowerHeartbeatEvery = 10
)
// guestPowerBackoff is the delay before each retry: 1m, 2m, 4m.
@@ -106,12 +117,33 @@ func (s *Server) GuestPowerTick(ctx context.Context) {
s.logger.Warn("guest-power: guest list unavailable — skipping sweep (ownership unproven)", "err", err)
return
}
var stopped int
for _, g := range guests {
if ctx.Err() != nil {
return
}
if g.Status != "running" {
stopped++
}
s.recoverOneStoppedGuest(ctx, g)
}
s.guestPowerSweeps++
noteGuestPowerSweep(s.logger, s.guestPowerSweeps, len(guests), stopped)
}
// noteGuestPowerSweep emits the liveness observable every guestPowerHeartbeatEvery sweeps.
//
// It carries WHAT THE SWEEP SAW, not merely that it ran: a line saying "I am alive" cannot
// distinguish "alive, all guests up" from "alive, one guest down and being left alone on purpose",
// and the second is the state an operator needs to see. Pure and separately testable — the mistake
// being corrected here was untestable precisely because it lived inline.
func noteGuestPowerSweep(logger *slog.Logger, sweeps, evaluated, stopped int) {
if logger == nil || sweeps <= 0 || sweeps%guestPowerHeartbeatEvery != 0 {
return
}
logger.Info("guest-power: watchdog alive",
"sweeps_since_boot", sweeps, "guests_evaluated", evaluated, "currently_stopped", stopped)
}
// recoverOneStoppedGuest starts a single guest that should be running and is not.