admin
f27f7a2659
The v0.107.0 watchdog was silent on a healthy box, so its health could only be inferred from absence — F-OBS's shape, shipped in the same session F-OBS was fixed. INFO summary every 10th sweep with what it saw; aborted sweeps are not counted. Red-proofs 7 and 8.
121 lines
5.0 KiB
Go
121 lines
5.0 KiB
Go
package localapi
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// A CORRECTION TO THIS PACKAGE'S OWN v0.107.0. The guest-power watchdog logged at startup and when it
|
|
// ACTED, and was silent otherwise — so on a healthy box the only evidence it was running was the
|
|
// absence of start lines, which is equally consistent with the sweep having died. That is F-OBS's
|
|
// shape and what standing rule 3 forbids, shipped in the same session F-OBS was fixed.
|
|
//
|
|
// These tests assert the emitted LINE. Asserting that a function was called would reproduce the
|
|
// original mistake, which was invisible precisely because nothing pinned the output.
|
|
|
|
// RED-PROOF: delete the noteGuestPowerSweep call at the end of GuestPowerTick (or the Info line
|
|
// inside it) → this fails with "no liveness observable after 10 sweeps — silence is
|
|
// indistinguishable from a dead watchdog".
|
|
func TestGuestPowerSweep_EmitsLivenessObservable(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
ctl := &fakeGuestPowerCtl{
|
|
guests: []proxmox.Guest{{VMID: 9201, Status: "running"}, {VMID: 9100, Status: "stopped"}},
|
|
locks: map[int]string{9201: "", 9100: ""},
|
|
onboot: map[int]bool{9201: true, 9100: false}, // 9100 is a deliberate stop
|
|
}
|
|
s := gpServer(t, ctl, nil)
|
|
s.logger = slog.New(slog.NewTextHandler(&buf, nil))
|
|
|
|
for i := 0; i < guestPowerHeartbeatEvery; i++ {
|
|
s.GuestPowerTick(context.Background())
|
|
}
|
|
|
|
out := buf.String()
|
|
if !strings.Contains(out, "watchdog alive") {
|
|
t.Fatalf("no liveness observable after %d sweeps — silence is indistinguishable from a dead watchdog:\n%s",
|
|
guestPowerHeartbeatEvery, out)
|
|
}
|
|
if !strings.Contains(out, "level=INFO") {
|
|
t.Errorf("the observable is not at INFO — a box on the default level would never see it:\n%s", out)
|
|
}
|
|
// It must carry WHAT IT SAW. "currently_stopped=1" is the operator-relevant fact here: the sweep is
|
|
// alive AND is deliberately leaving one guest down, which "I ran" alone cannot express.
|
|
for _, want := range []string{"sweeps_since_boot=", "guests_evaluated=2", "currently_stopped=1"} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("the observable omits %q — it proves the sweep ran but not what it found:\n%s", want, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
// It must be a summary, not a line per sweep: at 60 s that would be 1440 lines/day, which is the
|
|
// pressure that made silence attractive in the first place.
|
|
//
|
|
// RED-PROOF: change the guard to `sweeps%1 != 0` → this fails with
|
|
// "emitted 30 observables across 30 sweeps — that is the flood that made silence attractive".
|
|
func TestGuestPowerSweep_IsASummaryNotAFlood(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
lg := slog.New(slog.NewTextHandler(&buf, nil))
|
|
|
|
const sweeps = 30
|
|
for i := 1; i <= sweeps; i++ {
|
|
noteGuestPowerSweep(lg, i, 1, 0)
|
|
}
|
|
|
|
got := strings.Count(buf.String(), "watchdog alive")
|
|
want := sweeps / guestPowerHeartbeatEvery
|
|
if got == sweeps {
|
|
t.Fatalf("emitted %d observables across %d sweeps — that is the flood that made silence attractive", got, sweeps)
|
|
}
|
|
if got != want {
|
|
t.Errorf("emitted %d observables across %d sweeps, want %d", got, sweeps, want)
|
|
}
|
|
}
|
|
|
|
// The heartbeat period must stay short enough that a STALLED sweep is obvious well inside the outage
|
|
// window this watchdog exists to close (the finding's incident was 9m47s of total appliance
|
|
// downtime). If someone widens the cadence to hours the observable stops being a liveness signal.
|
|
func TestGuestPowerHeartbeat_StaysUsefulAsALivenessSignal(t *testing.T) {
|
|
period := guestPowerHeartbeatEvery * int(guestPowerInterval.Seconds())
|
|
if period > 15*60 {
|
|
t.Errorf("heartbeat period is %ds (>15min) — too sparse to notice a stalled watchdog", period)
|
|
}
|
|
if guestPowerHeartbeatEvery < 2 {
|
|
t.Errorf("heartbeat every %d sweeps is a per-sweep flood", guestPowerHeartbeatEvery)
|
|
}
|
|
}
|
|
|
|
// Off-cadence sweeps stay quiet; a nil logger must not panic (the ticker goroutine has no recovery).
|
|
func TestGuestPowerSweep_QuietOffCadenceAndNilSafe(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
lg := slog.New(slog.NewTextHandler(&buf, nil))
|
|
noteGuestPowerSweep(lg, guestPowerHeartbeatEvery-1, 1, 0)
|
|
if buf.Len() != 0 {
|
|
t.Errorf("emitted off-cadence:\n%s", buf.String())
|
|
}
|
|
noteGuestPowerSweep(nil, guestPowerHeartbeatEvery, 1, 0) // must not panic
|
|
}
|
|
|
|
// A sweep that ABORTED on unproven ownership must NOT count as a healthy sweep — otherwise the
|
|
// heartbeat would report liveness for a watchdog that is examining nothing, which is a worse lie than
|
|
// silence.
|
|
//
|
|
// RED-PROOF: move the s.guestPowerSweeps++ above the Guests() error return → this fails with
|
|
// "an aborted sweep was counted as healthy".
|
|
func TestGuestPowerSweep_AbortedSweepIsNotCounted(t *testing.T) {
|
|
ctl := &fakeGuestPowerCtl{guestsErr: errors.New("pool read failed")}
|
|
s := gpServer(t, ctl, nil)
|
|
for i := 0; i < guestPowerHeartbeatEvery*2; i++ {
|
|
s.GuestPowerTick(context.Background())
|
|
}
|
|
if s.guestPowerSweeps != 0 {
|
|
t.Errorf("an aborted sweep was counted as healthy (sweeps=%d) — the heartbeat would claim liveness for a watchdog examining nothing",
|
|
s.guestPowerSweeps)
|
|
}
|
|
}
|