admin
fb91c8d766
deadapp-check had no observable at default info level: its per-cycle line goes through Scheduler.dbg(), gated on logging.level==debug, so on a default box it is never PRODUCED (not merely filtered) and cannot reach the always-DEBUG ring. A 30s interval also puts it on the scheduler's quiet path. 'No alarms' was therefore indistinguishable from 'the detector never ran' — which undermines confidence in the F-CRIT-1 fix in the field. A periodic summary, not a line per run: at 30s a per-run line is 2880 lines/day, which is why the original author chose silence. Every 20th scan (~10 min) emits one INFO with the scan count, apps evaluated and apps down. A test pins the cadence so it cannot be widened into uselessness. Also corrects the 'unquiesce guaranteed by defer' comment — fault 10 established the guarantee is the crash marker plus Recover().
98 lines
4.1 KiB
Go
98 lines
4.1 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// F-OBS (Campaign 8): on a default `info`-level box there was NO positive observable that
|
|
// `deadapp-check` had run. Its per-cycle scheduler line goes through Scheduler.dbg(), which is gated
|
|
// on logging.level==debug and therefore never PRODUCED on a default box — so it could not even reach
|
|
// the always-DEBUG ring — and a 30 s interval also puts the job on the scheduler's quiet path.
|
|
//
|
|
// "No alarms" was therefore indistinguishable from "the detector never ran", which is exactly the
|
|
// fallacy this project now has a standing rule against, and it undermines confidence in the
|
|
// F-CRIT-1 fix in the field.
|
|
//
|
|
// Scenario F — the observable must appear AT INFO LEVEL. These tests assert the emitted LINE, not
|
|
// merely that a function was called; asserting the call would reproduce the original mistake.
|
|
|
|
// RED-PROOF: delete the logger.Printf in noteDeadAppScan (or drop the whole call from the job
|
|
// closure) → every case below sees an empty buffer and this fails with
|
|
// "no observable emitted at scan 20 — silence is indistinguishable from not running".
|
|
func TestNoteDeadAppScan_EmitsAtInfoLevel(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
lg := log.New(&buf, "", 0)
|
|
|
|
noteDeadAppScan(lg, deadAppHeartbeatEvery, 7, 2)
|
|
|
|
out := buf.String()
|
|
if out == "" {
|
|
t.Fatalf("no observable emitted at scan %d — silence is indistinguishable from not running", deadAppHeartbeatEvery)
|
|
}
|
|
if !strings.Contains(out, "[INFO]") {
|
|
t.Errorf("the observable is not at INFO level, so a default `logging.level: info` box would never see it:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "[deadapp]") {
|
|
t.Errorf("the observable does not identify the check that produced it:\n%s", out)
|
|
}
|
|
// it must carry WHAT IT SAW, not just "I ran" — an operator needs to distinguish
|
|
// "running and everything is up" from "running and 2 apps are down".
|
|
for _, want := range []string{"scans since boot", "evaluated", "currently down"} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("the observable omits %q — it proves the check ran but not what it found:\n%s", want, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
// It must NOT be a line per run. At a 30 s cadence that is 2880 lines/day, which is precisely why
|
|
// the original author chose silence — so a fix that floods is not a fix.
|
|
//
|
|
// RED-PROOF: change the guard to `scans%1 != 0` (i.e. emit every run) → this fails with
|
|
// "emitted 60 lines across 60 scans — that is the flood that made silence attractive".
|
|
func TestNoteDeadAppScan_IsASummaryNotAFlood(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
lg := log.New(&buf, "", 0)
|
|
|
|
const scans = 60
|
|
for i := 1; i <= scans; i++ {
|
|
noteDeadAppScan(lg, i, 3, 0)
|
|
}
|
|
|
|
got := strings.Count(buf.String(), "[deadapp] check alive")
|
|
want := scans / deadAppHeartbeatEvery
|
|
if got == scans {
|
|
t.Fatalf("emitted %d lines across %d scans — that is the flood that made silence attractive", got, scans)
|
|
}
|
|
if got != want {
|
|
t.Errorf("emitted %d heartbeat lines across %d scans, want %d (one per %d)", got, scans, want, deadAppHeartbeatEvery)
|
|
}
|
|
}
|
|
|
|
// The cadence must be frequent enough that a STALLED detector is obvious well inside the 180 s alarm
|
|
// grace this check feeds. 20 scans x 30 s = 10 min; if someone widens it to hours the observable
|
|
// stops being useful as a liveness signal, and this is the tripwire.
|
|
func TestDeadAppHeartbeatEvery_StaysUsefulAsALivenessSignal(t *testing.T) {
|
|
const scanInterval = 30 // seconds, matching sched.Every("deadapp-check", 30*time.Second, ...)
|
|
periodSec := deadAppHeartbeatEvery * scanInterval
|
|
if periodSec > 15*60 {
|
|
t.Errorf("heartbeat period is %ds (>15min) — too sparse to notice a stalled detector", periodSec)
|
|
}
|
|
if deadAppHeartbeatEvery < 2 {
|
|
t.Errorf("heartbeat every %d scans is a per-run flood", deadAppHeartbeatEvery)
|
|
}
|
|
}
|
|
|
|
// Off-cadence scans stay quiet, and a nil logger is tolerated (the job closure must never panic).
|
|
func TestNoteDeadAppScan_QuietOffCadenceAndNilSafe(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
lg := log.New(&buf, "", 0)
|
|
noteDeadAppScan(lg, deadAppHeartbeatEvery-1, 1, 0)
|
|
if buf.Len() != 0 {
|
|
t.Errorf("emitted off-cadence:\n%s", buf.String())
|
|
}
|
|
noteDeadAppScan(nil, deadAppHeartbeatEvery, 1, 0) // must not panic
|
|
}
|