package monitor import ( "encoding/json" "io" "log" "strings" "testing" "time" ) // R-100 — PRESENCE IS NOT SUCCESS. // // isStale counted from `LastRun`, which the controller writes unconditionally at the end of EVERY run, // failures included. So it asked "how long since we last TRIED", and a tier failing on every single // night refreshed the clock nightly and read as perfectly fresh forever. // // The fixtures below are the REAL wire shapes, taken from 4000 live hub reports on 2026-07-28: // last_status "ok" x2269 (always with last_run) · absent x541 (always with last_run EMPTY — // never-ran) · "error" x27 (with last_run) · "running" x7 (a report captured mid-run). // `running` is a real value and is neither success nor failure; the verdict must ignore it. // wire builds an offsiteReport by round-tripping JSON, so a field the struct cannot decode fails here // rather than silently reading as its zero value. func wire(t *testing.T, js string) *offsiteReport { t.Helper() var r struct { Offsite *offsiteReport `json:"offsite"` } if err := json.Unmarshal([]byte(`{"offsite":`+js+`}`), &r); err != nil { t.Fatalf("fixture does not decode: %v", err) } if r.Offsite == nil { t.Fatal("fixture decoded to nil") } return r.Offsite } // checkerAt builds a checker with a frozen clock over an empty real store (the constructor seeds from // it, so it cannot be nil). Scenario B seeds that store; the other scenarios never reach the store, // because only the never-ran branch consults it. func checkerAt(t *testing.T, now time.Time) *OffsiteChecker { t.Helper() oc := NewOffsiteChecker(newNeverRanStore(t), 48*time.Hour, nil, log.New(io.Discard, "", 0)) oc.now = func() time.Time { return now } return oc } // SCENARIO A — a persistently failing tier goes stale. // // This is the defect. Runs happen nightly and fail nightly; `last_run` is always fresh. // // RED-PROOF: restore the LastRun-only anchor (parse off.LastRun instead of off.LastSuccess in the // final branch) → this fails with "a tier that has not succeeded in 6 days reads as FRESH". func TestIsStale_PersistentlyFailingTierGoesStale(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) off := wire(t, `{ "enabled": true, "escrow_state": "escrowed", "last_run": "2026-07-28T02:15:00Z", "last_success": "2026-07-22T02:15:00Z", "last_status": "error" }`) if !checkerAt(t, now).isStale("demo-hp", off) { t.Fatalf("a tier that has not succeeded in 6 days reads as FRESH — that is R-100 (last_run=%s last_success=%s)", off.LastRun, off.LastSuccess) } } // SCENARIO B — a newborn box still does not alarm. The v0.73.0 cry-wolf must stay fixed; this is the // branch most likely to be broken by accident here. // // RED-PROOF: delete the `off.LastRun == ""` never-ran branch → this fails with // "a newborn box alarmed". func TestIsStale_NewbornBoxDoesNotAlarm(t *testing.T) { // The real never-ran shape: no last_run AND no last_status (541 live reports look like this). off := wire(t, `{"enabled": true, "escrow_state": "escrowed"}`) // Delivery completed an hour ago — well inside the 48h grace. st := newNeverRanStore(t) seedConsumedSecret(t, st, "newborn", time.Hour) var events []string oc := neverRanChecker(st, &events) if oc.isStale("newborn", off) { t.Error("a newborn box alarmed — this is the 2026-07-23 cry-wolf that v0.73.0 fixed") } // ...and once the grace elapses it DOES alarm, or the never-ran path would be a silence hole. st2 := newNeverRanStore(t) seedConsumedSecret(t, st2, "older", 72*time.Hour) oc2 := neverRanChecker(st2, &events) if !oc2.isStale("older", off) { t.Error("a box that has never run 72h after delivery did NOT alarm — the never-ran path went silent") } } // SCENARIO C — a healthy tier reads fresh. func TestIsStale_HealthyTierIsNotStale(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) off := wire(t, `{ "enabled": true, "escrow_state": "escrowed", "last_run": "2026-07-28T02:15:48Z", "last_success": "2026-07-28T02:15:48Z", "last_status": "ok" }`) if checkerAt(t, now).isStale("demo-hp", off) { t.Error("a tier that succeeded 10h ago reads as stale — spurious alarm on a working tier") } } // SCENARIO D — ONE transient failure inside the threshold does NOT alarm. // // This is the over-correction guard. "LastStatus == error ⇒ stale" is the tempting one-line fix and it // pages on every flaky night — the F-A1 noise path that trains an operator to ignore the alarm. // // RED-PROOF: collapse the verdict to `return off.LastStatus == "error"` → this fails with // "a single transient failure alarmed". func TestIsStale_SingleTransientFailureDoesNotAlarm(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) off := wire(t, `{ "enabled": true, "escrow_state": "escrowed", "last_run": "2026-07-28T02:15:00Z", "last_success": "2026-07-27T16:00:00Z", "last_status": "error" }`) // last success 20h ago, threshold 48h if checkerAt(t, now).isStale("demo-hp", off) { t.Error("a single transient failure alarmed — 20h since the last success is well inside the 48h threshold (the F-A1 noise path)") } } // `running` is a real wire value (7 live reports). A report captured mid-run is neither a success nor a // failure and must not move the verdict either way. func TestIsStale_RunningStatusIsNotAVerdict(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) fresh := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:16:58Z","last_success":"2026-07-28T02:15:00Z","last_status":"running"}`) if checkerAt(t, now).isStale("demo-felhom", fresh) { t.Error("a mid-run report on a healthy tier alarmed") } old := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:16:58Z","last_success":"2026-07-20T02:15:00Z","last_status":"running"}`) if !checkerAt(t, now).isStale("demo-felhom", old) { t.Error("a mid-run report suppressed a genuinely stale tier — status must not be a verdict in either direction") } } // SCENARIO E — a LEGACY controller behaves exactly as today. // // It has run (last_run set) but sends no last_success. Treating that as failure would alarm every // un-upgraded box in the fleet at once; treating it as success keeps the bug. It must degrade to the // old LastRun behaviour, and say so once. // // RED-PROOF: delete the `off.LastSuccess == ""` degrade branch → the fresh legacy box falls through to // the anchored verdict, parses "" as unparseable, and this fails with // "a legacy controller alarmed — that is a fleet-wide alarm storm on an un-upgraded fleet". func TestIsStale_LegacyControllerDegradesToTodaysBehaviour(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) // Pre-v0.181.0 shape: no last_success key at all. freshLegacy := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"ok"}`) if freshLegacy.LastSuccess != "" { t.Fatalf("fixture is not the legacy shape (last_success=%q)", freshLegacy.LastSuccess) } if checkerAt(t, now).isStale("legacy", freshLegacy) { t.Error("a legacy controller alarmed — that is a fleet-wide alarm storm on an un-upgraded fleet") } // And the old behaviour is genuinely preserved: an OLD last_run still goes stale. staleLegacy := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-20T02:15:00Z","last_status":"ok"}`) if !checkerAt(t, now).isStale("legacy", staleLegacy) { t.Error("a legacy controller with an 8-day-old run did NOT alarm — the degrade lost today's behaviour") } } // The legacy degrade must be VISIBLE — a fleet silently running on the old anchor is exactly the kind // of thing that gets assumed rather than known — but only ONCE per customer, since it is a steady state. func TestIsStale_LegacyDegradeIsLoggedOncePerCustomer(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) var buf strings.Builder oc := NewOffsiteChecker(newNeverRanStore(t), 48*time.Hour, nil, log.New(&buf, "", 0)) oc.now = func() time.Time { return now } off := wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"ok"}`) for i := 0; i < 5; i++ { oc.isStale("legacy-a", off) } oc.isStale("legacy-b", off) out := buf.String() if !strings.Contains(out, "no last_success") { t.Fatalf("the legacy degrade was SILENT — a fleet on the old anchor would be invisible:\n%s", out) } if n := strings.Count(out, "legacy-a"); n != 1 { t.Errorf("logged the degrade %d times for one customer, want 1 — it is a steady state, not an event", n) } if !strings.Contains(out, "legacy-b") { t.Error("the second customer's degrade was suppressed — the once-guard must be per customer") } } // Disabled / not-yet-escrowed tiers are normal onboarding and never stale — unchanged, and pinned // because the new branches sit above it. func TestIsStale_DisabledOrPendingNeverStale(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) for _, js := range []string{ `{"enabled":false,"escrow_state":"escrowed","last_run":"2026-07-01T02:15:00Z","last_success":"2026-07-01T02:15:00Z"}`, `{"enabled":true,"escrow_state":"pending","last_run":"2026-07-01T02:15:00Z","last_success":"2026-07-01T02:15:00Z"}`, } { if checkerAt(t, now).isStale("c", wire(t, js)) { t.Errorf("a disabled/pending tier alarmed: %s", js) } } } // SCENARIO F (hub half) — the ALARM MUST NOT LIE. When the verdict moved to the success anchor, the // message still read "last run 8h ago", so a tier running and failing nightly would have alarmed with a // fresh-looking timestamp and been dismissed as a false positive. func TestStaleAge_NamesTheRealReason(t *testing.T) { now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) oc := checkerAt(t, now) // runs happening, failing age, hint := oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_success":"2026-07-22T02:15:00Z","last_status":"error"}`)) if strings.Contains(age, "last run 9h ago") || !strings.Contains(age, "SUCCESSFUL") { t.Errorf("the alarm does not say the last SUCCESS is what is stale: %q", age) } if !strings.Contains(age, "attempted") || !strings.Contains(hint, "failing") { t.Errorf("the alarm hides that runs ARE happening and failing — the operator would check the schedule instead of the error: %q / %q", age, hint) } // never succeeded at all age, hint = oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed","last_run":"2026-07-28T02:15:00Z","last_status":"error"}`)) if !strings.Contains(age, "EVER succeeded") || !strings.Contains(hint, "failing") { t.Errorf("a never-succeeded tier is not described as such: %q / %q", age, hint) } // genuinely not running age, hint = oc.staleAge(wire(t, `{"enabled":true,"escrow_state":"escrowed"}`)) if age != "never ran" || !strings.Contains(hint, "schedule") { t.Errorf("a never-ran tier lost its schedule hint: %q / %q", age, hint) } }