admin
b505ee9125
isStale counted from last_run, written unconditionally on failure, so a nightly-failing tier read as fresh forever. Now anchored on last_success with an explicit legacy degrade (logged once) and the never-ran branch untouched. emitStale states the real reason.
238 lines
11 KiB
Go
238 lines
11 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// R-100 — PRESENCE IS NOT SUCCESS.
|
|
//
|
|
// isStale counted from `LastRun`, which the controller writes unconditionally at the end of EVERY run,
|
|
// failures included. So it asked "how long since we last TRIED", and a tier failing on every single
|
|
// night refreshed the clock nightly and read as perfectly fresh forever.
|
|
//
|
|
// The fixtures below are the REAL wire shapes, taken from 4000 live hub reports on 2026-07-28:
|
|
// last_status "ok" x2269 (always with last_run) · absent x541 (always with last_run EMPTY —
|
|
// never-ran) · "error" x27 (with last_run) · "running" x7 (a report captured mid-run).
|
|
// `running` is a real value and is neither success nor failure; the verdict must ignore it.
|
|
|
|
// wire builds an offsiteReport by round-tripping JSON, so a field the struct cannot decode fails here
|
|
// rather than silently reading as its zero value.
|
|
func wire(t *testing.T, js string) *offsiteReport {
|
|
t.Helper()
|
|
var r struct {
|
|
Offsite *offsiteReport `json:"offsite"`
|
|
}
|
|
if err := json.Unmarshal([]byte(`{"offsite":`+js+`}`), &r); err != nil {
|
|
t.Fatalf("fixture does not decode: %v", err)
|
|
}
|
|
if r.Offsite == nil {
|
|
t.Fatal("fixture decoded to nil")
|
|
}
|
|
return r.Offsite
|
|
}
|
|
|
|
// checkerAt builds a checker with a frozen clock over an empty real store (the constructor seeds from
|
|
// it, so it cannot be nil). Scenario B seeds that store; the other scenarios never reach the store,
|
|
// because only the never-ran branch consults it.
|
|
func checkerAt(t *testing.T, now time.Time) *OffsiteChecker {
|
|
t.Helper()
|
|
oc := NewOffsiteChecker(newNeverRanStore(t), 48*time.Hour, nil, log.New(io.Discard, "", 0))
|
|
oc.now = func() time.Time { return now }
|
|
return oc
|
|
}
|
|
|
|
// SCENARIO A — a persistently failing tier goes stale.
|
|
//
|
|
// This is the defect. Runs happen nightly and fail nightly; `last_run` is always fresh.
|
|
//
|
|
// RED-PROOF: restore the LastRun-only anchor (parse off.LastRun instead of off.LastSuccess in the
|
|
// final branch) → this fails with "a tier that has not succeeded in 6 days reads as FRESH".
|
|
func TestIsStale_PersistentlyFailingTierGoesStale(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
off := wire(t, `{
|
|
"enabled": true, "escrow_state": "escrowed",
|
|
"last_run": "2026-07-28T02:15:00Z",
|
|
"last_success": "2026-07-22T02:15:00Z",
|
|
"last_status": "error"
|
|
}`)
|
|
if !checkerAt(t, now).isStale("demo-hp", off) {
|
|
t.Fatalf("a tier that has not succeeded in 6 days reads as FRESH — that is R-100 (last_run=%s last_success=%s)",
|
|
off.LastRun, off.LastSuccess)
|
|
}
|
|
}
|
|
|
|
// SCENARIO B — a newborn box still does not alarm. The v0.73.0 cry-wolf must stay fixed; this is the
|
|
// branch most likely to be broken by accident here.
|
|
//
|
|
// RED-PROOF: delete the `off.LastRun == ""` never-ran branch → this fails with
|
|
// "a newborn box alarmed".
|
|
func TestIsStale_NewbornBoxDoesNotAlarm(t *testing.T) {
|
|
// The real never-ran shape: no last_run AND no last_status (541 live reports look like this).
|
|
off := wire(t, `{"enabled": true, "escrow_state": "escrowed"}`)
|
|
|
|
// Delivery completed an hour ago — well inside the 48h grace.
|
|
st := newNeverRanStore(t)
|
|
seedConsumedSecret(t, st, "newborn", time.Hour)
|
|
var events []string
|
|
oc := neverRanChecker(st, &events)
|
|
if oc.isStale("newborn", off) {
|
|
t.Error("a newborn box alarmed — this is the 2026-07-23 cry-wolf that v0.73.0 fixed")
|
|
}
|
|
|
|
// ...and once the grace elapses it DOES alarm, or the never-ran path would be a silence hole.
|
|
st2 := newNeverRanStore(t)
|
|
seedConsumedSecret(t, st2, "older", 72*time.Hour)
|
|
oc2 := neverRanChecker(st2, &events)
|
|
if !oc2.isStale("older", off) {
|
|
t.Error("a box that has never run 72h after delivery did NOT alarm — the never-ran path went silent")
|
|
}
|
|
}
|
|
|
|
// SCENARIO C — a healthy tier reads fresh.
|
|
func TestIsStale_HealthyTierIsNotStale(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
off := wire(t, `{
|
|
"enabled": true, "escrow_state": "escrowed",
|
|
"last_run": "2026-07-28T02:15:48Z", "last_success": "2026-07-28T02:15:48Z", "last_status": "ok"
|
|
}`)
|
|
if checkerAt(t, now).isStale("demo-hp", off) {
|
|
t.Error("a tier that succeeded 10h ago reads as stale — spurious alarm on a working tier")
|
|
}
|
|
}
|
|
|
|
// SCENARIO D — ONE transient failure inside the threshold does NOT alarm.
|
|
//
|
|
// This is the over-correction guard. "LastStatus == error ⇒ stale" is the tempting one-line fix and it
|
|
// pages on every flaky night — the F-A1 noise path that trains an operator to ignore the alarm.
|
|
//
|
|
// RED-PROOF: collapse the verdict to `return off.LastStatus == "error"` → this fails with
|
|
// "a single transient failure alarmed".
|
|
func TestIsStale_SingleTransientFailureDoesNotAlarm(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
off := wire(t, `{
|
|
"enabled": true, "escrow_state": "escrowed",
|
|
"last_run": "2026-07-28T02:15:00Z",
|
|
"last_success": "2026-07-27T16:00:00Z",
|
|
"last_status": "error"
|
|
}`) // last success 20h ago, threshold 48h
|
|
if checkerAt(t, now).isStale("demo-hp", off) {
|
|
t.Error("a single transient failure alarmed — 20h since the last success is well inside the 48h threshold (the F-A1 noise path)")
|
|
}
|
|
}
|
|
|
|
// `running` is a real wire value (7 live reports). A report captured mid-run is neither a success nor a
|
|
// failure and must not move the verdict either way.
|
|
func TestIsStale_RunningStatusIsNotAVerdict(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
fresh := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:16:58Z","last_success":"2026-07-28T02:15:00Z","last_status":"running"}`)
|
|
if checkerAt(t, now).isStale("demo-felhom", fresh) {
|
|
t.Error("a mid-run report on a healthy tier alarmed")
|
|
}
|
|
old := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:16:58Z","last_success":"2026-07-20T02:15:00Z","last_status":"running"}`)
|
|
if !checkerAt(t, now).isStale("demo-felhom", old) {
|
|
t.Error("a mid-run report suppressed a genuinely stale tier — status must not be a verdict in either direction")
|
|
}
|
|
}
|
|
|
|
// SCENARIO E — a LEGACY controller behaves exactly as today.
|
|
//
|
|
// It has run (last_run set) but sends no last_success. Treating that as failure would alarm every
|
|
// un-upgraded box in the fleet at once; treating it as success keeps the bug. It must degrade to the
|
|
// old LastRun behaviour, and say so once.
|
|
//
|
|
// RED-PROOF: delete the `off.LastSuccess == ""` degrade branch → the fresh legacy box falls through to
|
|
// the anchored verdict, parses "" as unparseable, and this fails with
|
|
// "a legacy controller alarmed — that is a fleet-wide alarm storm on an un-upgraded fleet".
|
|
func TestIsStale_LegacyControllerDegradesToTodaysBehaviour(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
|
|
// Pre-v0.181.0 shape: no last_success key at all.
|
|
freshLegacy := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"ok"}`)
|
|
if freshLegacy.LastSuccess != "" {
|
|
t.Fatalf("fixture is not the legacy shape (last_success=%q)", freshLegacy.LastSuccess)
|
|
}
|
|
if checkerAt(t, now).isStale("legacy", freshLegacy) {
|
|
t.Error("a legacy controller alarmed — that is a fleet-wide alarm storm on an un-upgraded fleet")
|
|
}
|
|
|
|
// And the old behaviour is genuinely preserved: an OLD last_run still goes stale.
|
|
staleLegacy := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-20T02:15:00Z","last_status":"ok"}`)
|
|
if !checkerAt(t, now).isStale("legacy", staleLegacy) {
|
|
t.Error("a legacy controller with an 8-day-old run did NOT alarm — the degrade lost today's behaviour")
|
|
}
|
|
}
|
|
|
|
// The legacy degrade must be VISIBLE — a fleet silently running on the old anchor is exactly the kind
|
|
// of thing that gets assumed rather than known — but only ONCE per customer, since it is a steady state.
|
|
func TestIsStale_LegacyDegradeIsLoggedOncePerCustomer(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
var buf strings.Builder
|
|
oc := NewOffsiteChecker(newNeverRanStore(t), 48*time.Hour, nil, log.New(&buf, "", 0))
|
|
oc.now = func() time.Time { return now }
|
|
|
|
off := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"ok"}`)
|
|
for i := 0; i < 5; i++ {
|
|
oc.isStale("legacy-a", off)
|
|
}
|
|
oc.isStale("legacy-b", off)
|
|
|
|
out := buf.String()
|
|
if !strings.Contains(out, "no last_success") {
|
|
t.Fatalf("the legacy degrade was SILENT — a fleet on the old anchor would be invisible:\n%s", out)
|
|
}
|
|
if n := strings.Count(out, "legacy-a"); n != 1 {
|
|
t.Errorf("logged the degrade %d times for one customer, want 1 — it is a steady state, not an event", n)
|
|
}
|
|
if !strings.Contains(out, "legacy-b") {
|
|
t.Error("the second customer's degrade was suppressed — the once-guard must be per customer")
|
|
}
|
|
}
|
|
|
|
// Disabled / not-yet-escrowed tiers are normal onboarding and never stale — unchanged, and pinned
|
|
// because the new branches sit above it.
|
|
func TestIsStale_DisabledOrPendingNeverStale(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
for _, js := range []string{
|
|
`{"enabled":false,"escrow_state":"escrowed","last_run":"2026-07-01T02:15:00Z","last_success":"2026-07-01T02:15:00Z"}`,
|
|
`{"enabled":true,"escrow_state":"pending","last_run":"2026-07-01T02:15:00Z","last_success":"2026-07-01T02:15:00Z"}`,
|
|
} {
|
|
if checkerAt(t, now).isStale("c", wire(t, js)) {
|
|
t.Errorf("a disabled/pending tier alarmed: %s", js)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SCENARIO F (hub half) — the ALARM MUST NOT LIE. When the verdict moved to the success anchor, the
|
|
// message still read "last run 8h ago", so a tier running and failing nightly would have alarmed with a
|
|
// fresh-looking timestamp and been dismissed as a false positive.
|
|
func TestStaleAge_NamesTheRealReason(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
oc := checkerAt(t, now)
|
|
|
|
// runs happening, failing
|
|
age, hint := oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_success":"2026-07-22T02:15:00Z","last_status":"error"}`))
|
|
if strings.Contains(age, "last run 9h ago") || !strings.Contains(age, "SUCCESSFUL") {
|
|
t.Errorf("the alarm does not say the last SUCCESS is what is stale: %q", age)
|
|
}
|
|
if !strings.Contains(age, "attempted") || !strings.Contains(hint, "failing") {
|
|
t.Errorf("the alarm hides that runs ARE happening and failing — the operator would check the schedule instead of the error: %q / %q", age, hint)
|
|
}
|
|
|
|
// never succeeded at all
|
|
age, hint = oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"error"}`))
|
|
if !strings.Contains(age, "EVER succeeded") || !strings.Contains(hint, "failing") {
|
|
t.Errorf("a never-succeeded tier is not described as such: %q / %q", age, hint)
|
|
}
|
|
|
|
// genuinely not running
|
|
age, hint = oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed"}`))
|
|
if age != "never ran" || !strings.Contains(hint, "schedule") {
|
|
t.Errorf("a never-ran tier lost its schedule hint: %q / %q", age, hint)
|
|
}
|
|
}
|