package monitor // R-321 — A MACHINE WE TOLD TO BE QUIET IS NOT A MACHINE THAT DIED. // // Switching a box's hub reporting off is a supported product state. The controller sends one final // report carrying health.status = "disabled" and then goes quiet BY DESIGN. The hub parses and stores // that status, and the operator roll-up already renders such a customer as `disabled` — but this // checker measured only the AGE of the last report, so the machine went stale at 30 minutes, down at // 60, and e-mailed the operator twice about an outage we caused on purpose. // // The discriminator is the box's OWN last word, not an inference, and it is in the data this checker // already reads (`CustomerSummary.HealthStatus`). Suppressing on a guess would be worse than the // false alarm it removes; suppressing on the machine's own declaration is not a guess. import ( "database/sql" "io" "log" "path/filepath" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" _ "modernc.org/sqlite" ) // seedStalenessCustomer creates a store with one customer and one report of the given health status, // backdated by `age`. Reports are inserted through the REAL SaveReport path so the health_status // denormalization is exercised rather than hand-set — a test that writes the column directly cannot // see a decode that never happens, which is the R-260 class. func seedStalenessCustomer(t *testing.T, health string, age time.Duration) (*store.Store, string) { t.Helper() path := filepath.Join(t.TempDir(), "t.db") st, err := store.New(path, log.New(io.Discard, "", 0)) if err != nil { t.Fatalf("store.New: %v", err) } t.Cleanup(func() { st.Close() }) if err := st.SaveCustomerConfig(&store.CustomerConfig{ CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p", Status: "active", }); err != nil { t.Fatalf("SaveCustomerConfig: %v", err) } saveReportAged(t, st, path, health, age) return st, path } // saveReportAged saves a controller report with the given health status and then backdates its // received_at. The backdate is a raw UPDATE because there is no production path that writes an old // timestamp — which is exactly why it is confined to this one helper. func saveReportAged(t *testing.T, st *store.Store, path, health string, age time.Duration) { t.Helper() body := `{"customer_id":"c1","health":{"status":"` + health + `"}}` if err := st.SaveReport("c1", []byte(body)); err != nil { t.Fatalf("SaveReport: %v", err) } if age <= 0 { return } // Backdating goes through a SECOND connection to the same file rather than a test-only method on // Store. No production path writes an old received_at, so exposing one would widen the store's // API for a fixture — and this keeps the ageing confined to the one helper that needs it. db, err := sql.Open("sqlite", path) if err != nil { t.Fatalf("open for backdate: %v", err) } defer db.Close() // GetCustomers selects the report with MAX(received_at), not MAX(id) — so backdating only the // row just written would leave an EARLIER, fresher-looking row as the one the checker reads, and // the fixture would quietly test the wrong report. (Found by two scenarios failing on a change // that was correct: the instrument was wrong, not the code.) Every older row is therefore pushed // further back, so the row just written is unambiguously the latest. when := time.Now().UTC().Add(-age) newest := when.Format("2006-01-02 15:04:05") older := when.Add(-time.Hour).Format("2006-01-02 15:04:05") if _, err := db.Exec( `UPDATE reports SET received_at = ? WHERE customer_id = 'c1' AND id < (SELECT MAX(id) FROM reports WHERE customer_id = 'c1')`, older); err != nil { t.Fatalf("age the older rows: %v", err) } if _, err := db.Exec( `UPDATE reports SET received_at = ? WHERE id = (SELECT MAX(id) FROM reports WHERE customer_id = 'c1')`, newest); err != nil { t.Fatalf("backdate: %v", err) } } // newChecker builds a checker with a 30-minute threshold and records every event it emits. func newChecker(t *testing.T, st *store.Store) (*StalenessChecker, *[]string) { t.Helper() var events []string sc := NewStalenessChecker(st, 30*time.Minute, func(cid, et, sev, msg, det, src string) { events = append(events, et) }, log.New(io.Discard, "", 0)) return sc, &events } // ── A — a machine deliberately silent for days ────────────────────────────────────────────────── // // WRONG OUTCOME GUARDED: stale, then down, then two e-mails — which is what happens today. func TestStaleness_A_DeliberatelySilentDoesNotAlarm(t *testing.T) { // The box is HEALTHY and OBSERVED first, then switched off. Seeding it already-disabled would // make this test pass vacuously: the checker's new-customer branch sets the first state without // an event, so a customer that was never seen healthy can never emit a transition — and the // assertion below would hold even with the suppression deleted. (Confirmed: it did. The // red-proof for this scenario passed on the state assertion alone until this line was added.) st, path := seedStalenessCustomer(t, "ok", 0) sc, events := newChecker(t, st) sc.Check() if sc.GetState("c1") != "ok" { t.Fatalf("setup: the machine should start observed-healthy, got %q", sc.GetState("c1")) } // Reporting is switched off. The box says so on the way out, then goes quiet for three days. saveReportAged(t, st, path, "disabled", 72*time.Hour) for i := 0; i < 3; i++ { // several passes: a suppression that only holds once is not a suppression sc.Check() } if len(*events) != 0 { t.Fatalf("a deliberately-disabled machine emitted %v — three days quiet BY REQUEST", *events) } // The silence must be VISIBLE, not merely un-alarmed: quiet-on-purpose and quiet-by-accident // must not look identical, and an unknown is never drawn as healthy. if got := sc.GetState("c1"); got != StateDisabled { t.Errorf("state = %q, want %q — the deliberate silence is invisible", got, StateDisabled) } } // ── B — a machine that simply stopped reporting ───────────────────────────────────────────────── // // WRONG OUTCOME GUARDED: silence mistaken for a deliberate switch-off — THE ALARM THAT MATTERS, // suppressed. This is the one that must never regress. func TestStaleness_B_GenuineSilenceStillAlarms(t *testing.T) { st, path := seedStalenessCustomer(t, "ok", 0) sc, events := newChecker(t, st) sc.Check() // first observation: healthy, no event // The same customer now goes quiet for real. saveReportAged(t, st, path, "ok", 3*time.Hour) sc.Check() if got := sc.GetState("c1"); got != "down" { t.Fatalf("a genuinely silent machine is %q, want \"down\" — a real alarm was swallowed", got) } if len(*events) == 0 { t.Fatal("a genuinely silent machine emitted NO event — the suppression is over-broad") } } // ── C — a disabled machine that is re-enabled and reports promptly ────────────────────────────── // // WRONG OUTCOME GUARDED: a recovery e-mail for an outage that never happened. func TestStaleness_C_ReEnabledReportsCleanlyWithNoRecoveryEvent(t *testing.T) { st, path := seedStalenessCustomer(t, "disabled", 72*time.Hour) sc, events := newChecker(t, st) sc.Check() if sc.GetState("c1") != StateDisabled { t.Fatalf("setup: expected the disabled state, got %q", sc.GetState("c1")) } // Reporting is switched back on and the box reports immediately. saveReportAged(t, st, path, "ok", 0) sc.Check() if len(*events) != 0 { t.Fatalf("re-enabling emitted %v — there was no outage to recover from", *events) } if got := sc.GetState("c1"); got != "ok" { t.Errorf("state after re-enable = %q, want \"ok\"", got) } } // ── D — a disabled machine that is re-enabled and then goes genuinely quiet ───────────────────── // // WRONG OUTCOME GUARDED: permanently silenced because it was once disabled. Also pins THE CLOCK // JUDGEMENT: timing runs from the report the box sent on re-enabling, not from the last report // before the switch-off — otherwise coming back would fire an instant false alarm about a quiet // period we asked for. func TestStaleness_D_ReEnabledThenQuietAlarmsFromReEnablement(t *testing.T) { st, path := seedStalenessCustomer(t, "disabled", 72*time.Hour) sc, events := newChecker(t, st) sc.Check() // Re-enabled, reports promptly — the clean first observation. saveReportAged(t, st, path, "ok", 0) sc.Check() if len(*events) != 0 { t.Fatalf("the re-enable itself emitted %v", *events) } // …and then goes quiet for real. Timed from THIS report, it is down. saveReportAged(t, st, path, "ok", 3*time.Hour) sc.Check() if got := sc.GetState("c1"); got != "down" { t.Fatalf("state = %q, want \"down\" — a once-disabled machine was silenced for ever", got) } if len(*events) == 0 { t.Fatal("no event for a machine that genuinely died after being re-enabled") } } // The deadline check is a SECOND DOOR onto the same false alarm: a disabled box is not "down", so // without its own skip it would keep e-mailing expected_backup_missed every morning. R-195's shape, // which is why both doors are closed together. func TestStaleness_DisabledIsAlsoSkippedByTheDeadlineCheck(t *testing.T) { st, _ := seedStalenessCustomer(t, "disabled", 72*time.Hour) if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { t.Fatalf("UpsertHost: %v", err) } sc, _ := newChecker(t, st) sc.Check() var events []string CheckBackupDeadlines(st, sc, func(cid, et, sev, msg, det, src string) { events = append(events, et) }, log.New(io.Discard, "", 0)) for _, e := range events { if e == "expected_backup_missed" || e == "expected_dbdump_missed" { t.Fatalf("the deadline check alarmed on a deliberately-disabled machine: %v", events) } } }