188 lines
6.4 KiB
Go
188 lines
6.4 KiB
Go
package monitor
|
|
|
|
import (
|
|
"database/sql"
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// R-1001 (measured 2026-10-13 18:33 UTC, hub v0.149.0 rollout): Tester 2, silent 216 h, got a fresh host_down and
|
|
// restore_test_stale mail on a hub restart. A restart is a NEW checker over the SAME database — that is what these
|
|
// tests build: alarm with checker 1, then a second checker on the same DB (the restarted hub) must not mail again,
|
|
// and a host that goes silent after the start must still mail once.
|
|
|
|
func r1001Store(t *testing.T) (*store.Store, *sql.DB) {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "r1001.db")
|
|
st, err := store.New(path, log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
return st, db
|
|
}
|
|
|
|
func r1001Host(t *testing.T, st *store.Store, db *sql.DB, cid, hid string, minutesAgo int) {
|
|
t.Helper()
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: cid, APIKey: "k-" + cid, RetrievalPassword: "p"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.UpsertHost(&store.Host{HostID: hid, CustomerID: cid, APIKey: "hk-" + hid}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.SaveHostReport(hid, cid, []byte(`{}`), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backdate(t, db, hid, minutesAgo)
|
|
}
|
|
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with the R-1001 seeding removed from NewHostStalenessChecker (the pre-fix
|
|
// F2 shape), the restart case fails with
|
|
//
|
|
// r1001_restart_test.go: the restarted hub re-sent [host_down] for a host whose silence was already alarmed
|
|
//
|
|
// Restored.
|
|
func TestHostStaleness_R1001_RestartDoesNotReMail(t *testing.T) {
|
|
st, db := r1001Store(t)
|
|
r1001Host(t, st, db, "tester-2", "Tester-2-be8404", 216*60) // down for 9 days
|
|
var got []string
|
|
on := func(_, et, _, _, _, _ string) { got = append(got, et) }
|
|
quiet := log.New(io.Discard, "", 0)
|
|
|
|
first := NewHostStalenessChecker(st, 30*time.Minute, on, quiet)
|
|
first.Check()
|
|
if len(got) != 1 || got[0] != "host_down" {
|
|
t.Fatalf("the first hub ever to see the silence must alarm once: %v", got)
|
|
}
|
|
|
|
got = nil
|
|
restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet)
|
|
restarted.Check()
|
|
restarted.Check()
|
|
if len(got) != 0 {
|
|
t.Fatalf("the restarted hub re-sent %v for a host whose silence was already alarmed", got)
|
|
}
|
|
if s := restarted.GetState("Tester-2-be8404"); s != "down" {
|
|
t.Fatalf("the already-alarmed host must be seeded with its alarm's state, got %q", s)
|
|
}
|
|
|
|
// The box reports again → host_recovered once, and its next silence alarms again (a new run).
|
|
backdate(t, db, "Tester-2-be8404", 0)
|
|
restarted.Check()
|
|
if len(got) != 1 || got[0] != "host_recovered" {
|
|
t.Fatalf("a seeded host that reports again must recover once: %v", got)
|
|
}
|
|
got = nil
|
|
backdate(t, db, "Tester-2-be8404", 70)
|
|
again := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) // a restart in the middle of a NEW silence
|
|
again.Check()
|
|
if len(got) != 1 || got[0] != "host_down" {
|
|
t.Fatalf("a new silence after a report must alarm (the old alarm is older than the report): %v", got)
|
|
}
|
|
}
|
|
|
|
// The other half of the rule: a host that goes down AFTER the start mails once; an alarm seeded „stale" still
|
|
// escalates to host_down when it crosses the down threshold (a new transition, not a repeat).
|
|
func TestHostStaleness_R1001_NewSilenceStillMails(t *testing.T) {
|
|
st, db := r1001Host2(t)
|
|
var got []string
|
|
on := func(_, et, _, _, _, _ string) { got = append(got, et) }
|
|
quiet := log.New(io.Discard, "", 0)
|
|
|
|
sc := NewHostStalenessChecker(st, 30*time.Minute, on, quiet)
|
|
sc.Check()
|
|
if len(got) != 0 {
|
|
t.Fatalf("a healthy host at start: no mail, got %v", got)
|
|
}
|
|
backdate(t, db, "h-new", 70) // down after the start
|
|
sc.Check()
|
|
if len(got) != 1 || got[0] != "host_down" {
|
|
t.Fatalf("a host down after the start must mail once: %v", got)
|
|
}
|
|
|
|
// stale alarmed, then a restart, then down → host_down (new transition), once.
|
|
got = nil
|
|
backdate(t, db, "h-new", 0)
|
|
sc.Check() // recovered
|
|
backdate(t, db, "h-new", 40)
|
|
sc.Check() // host_stale
|
|
got = nil
|
|
restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet)
|
|
restarted.Check()
|
|
if len(got) != 0 {
|
|
t.Fatalf("restart during an alarmed stale: no mail, got %v", got)
|
|
}
|
|
backdate(t, db, "h-new", 70)
|
|
restarted.Check()
|
|
if len(got) != 1 || got[0] != "host_down" {
|
|
t.Fatalf("stale → down after the restart must mail host_down once: %v", got)
|
|
}
|
|
}
|
|
|
|
func r1001Host2(t *testing.T) (*store.Store, *sql.DB) {
|
|
st, db := r1001Store(t)
|
|
r1001Host(t, st, db, "c-new", "h-new", 0)
|
|
return st, db
|
|
}
|
|
|
|
// restore_test_stale across a restart: the same DB, a second checker. Both tiers unproven for 20 days.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with the staleAlreadyReported case removed from checkStaleness, it fails
|
|
// with
|
|
//
|
|
// r1001_restart_test.go: the restarted hub re-sent 2 restore_test_stale signal(s)
|
|
//
|
|
// Restored.
|
|
func TestRestoreTestStale_R1001_RestartDoesNotReMail(t *testing.T) {
|
|
st, db := r1001Store(t)
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Now().UTC()
|
|
if err := st.SaveHostReport("h1", "c1", []byte(rtReport(t, "", false, now, "")), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.Exec(`UPDATE host_reports SET received_at = datetime('now', '-20 days')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got []capturedEvent
|
|
first := rtChecker(st, &got, nil)
|
|
first.Check()
|
|
if len(got) != 2 {
|
|
t.Fatalf("two tiers unproven for 20 days: want 2 stale signals, got %v", got)
|
|
}
|
|
|
|
got = nil
|
|
restarted := rtChecker(st, &got, nil)
|
|
restarted.Check()
|
|
restarted.Check()
|
|
if len(got) != 0 {
|
|
t.Fatalf("the restarted hub re-sent %d restore_test_stale signal(s)", len(got))
|
|
}
|
|
|
|
// An earlier signal outside the read window does not hold the next one back.
|
|
if _, err := db.Exec(`UPDATE events SET created_at = datetime('now', '-15 days') WHERE event_type = ?`, EventRestoreTestStale); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got = nil
|
|
later := rtChecker(st, &got, nil)
|
|
later.Check()
|
|
if len(got) != 2 {
|
|
t.Fatalf("a signal older than the read window must not silence the tier: got %v", got)
|
|
}
|
|
}
|