package monitor import ( "database/sql" "io" "log" "path/filepath" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" _ "modernc.org/sqlite" ) // R-1001 (measured 2026-10-13 18:33 UTC, hub v0.149.0 rollout): Tester 2, silent 216 h, got a fresh host_down and // restore_test_stale mail on a hub restart. A restart is a NEW checker over the SAME database — that is what these // tests build: alarm with checker 1, then a second checker on the same DB (the restarted hub) must not mail again, // and a host that goes silent after the start must still mail once. func r1001Store(t *testing.T) (*store.Store, *sql.DB) { t.Helper() path := filepath.Join(t.TempDir(), "r1001.db") st, err := store.New(path, log.New(io.Discard, "", 0)) if err != nil { t.Fatal(err) } t.Cleanup(func() { st.Close() }) db, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } t.Cleanup(func() { db.Close() }) return st, db } func r1001Host(t *testing.T, st *store.Store, db *sql.DB, cid, hid string, minutesAgo int) { t.Helper() if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: cid, APIKey: "k-" + cid, RetrievalPassword: "p"}); err != nil { t.Fatal(err) } if err := st.UpsertHost(&store.Host{HostID: hid, CustomerID: cid, APIKey: "hk-" + hid}); err != nil { t.Fatal(err) } if err := st.SaveHostReport(hid, cid, []byte(`{}`), store.HostReportDenorm{}); err != nil { t.Fatal(err) } backdate(t, db, hid, minutesAgo) } // COMPANION RED-PROOF (observed 2026-10-19): with the R-1001 seeding removed from NewHostStalenessChecker (the pre-fix // F2 shape), the restart case fails with // // r1001_restart_test.go: the restarted hub re-sent [host_down] for a host whose silence was already alarmed // // Restored. func TestHostStaleness_R1001_RestartDoesNotReMail(t *testing.T) { st, db := r1001Store(t) r1001Host(t, st, db, "tester-2", "Tester-2-be8404", 216*60) // down for 9 days var got []string on := func(_, et, _, _, _, _ string) { got = append(got, et) } quiet := log.New(io.Discard, "", 0) first := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) first.Check() if len(got) != 1 || got[0] != "host_down" { t.Fatalf("the first hub ever to see the silence must alarm once: %v", got) } got = nil restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) restarted.Check() restarted.Check() if len(got) != 0 { t.Fatalf("the restarted hub re-sent %v for a host whose silence was already alarmed", got) } if s := restarted.GetState("Tester-2-be8404"); s != "down" { t.Fatalf("the already-alarmed host must be seeded with its alarm's state, got %q", s) } // The box reports again → host_recovered once, and its next silence alarms again (a new run). backdate(t, db, "Tester-2-be8404", 0) restarted.Check() if len(got) != 1 || got[0] != "host_recovered" { t.Fatalf("a seeded host that reports again must recover once: %v", got) } got = nil backdate(t, db, "Tester-2-be8404", 70) again := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) // a restart in the middle of a NEW silence again.Check() if len(got) != 1 || got[0] != "host_down" { t.Fatalf("a new silence after a report must alarm (the old alarm is older than the report): %v", got) } } // The other half of the rule: a host that goes down AFTER the start mails once; an alarm seeded „stale" still // escalates to host_down when it crosses the down threshold (a new transition, not a repeat). func TestHostStaleness_R1001_NewSilenceStillMails(t *testing.T) { st, db := r1001Host2(t) var got []string on := func(_, et, _, _, _, _ string) { got = append(got, et) } quiet := log.New(io.Discard, "", 0) sc := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) sc.Check() if len(got) != 0 { t.Fatalf("a healthy host at start: no mail, got %v", got) } backdate(t, db, "h-new", 70) // down after the start sc.Check() if len(got) != 1 || got[0] != "host_down" { t.Fatalf("a host down after the start must mail once: %v", got) } // stale alarmed, then a restart, then down → host_down (new transition), once. got = nil backdate(t, db, "h-new", 0) sc.Check() // recovered backdate(t, db, "h-new", 40) sc.Check() // host_stale got = nil restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) restarted.Check() if len(got) != 0 { t.Fatalf("restart during an alarmed stale: no mail, got %v", got) } backdate(t, db, "h-new", 70) restarted.Check() if len(got) != 1 || got[0] != "host_down" { t.Fatalf("stale → down after the restart must mail host_down once: %v", got) } } func r1001Host2(t *testing.T) (*store.Store, *sql.DB) { st, db := r1001Store(t) r1001Host(t, st, db, "c-new", "h-new", 0) return st, db } // restore_test_stale across a restart: the same DB, a second checker. Both tiers unproven for 20 days. // // COMPANION RED-PROOF (observed 2026-10-19): with the staleAlreadyReported case removed from checkStaleness, it fails // with // // r1001_restart_test.go: the restarted hub re-sent 2 restore_test_stale signal(s) // // Restored. func TestRestoreTestStale_R1001_RestartDoesNotReMail(t *testing.T) { st, db := r1001Store(t) if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"}); err != nil { t.Fatal(err) } if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { t.Fatal(err) } now := time.Now().UTC() if err := st.SaveHostReport("h1", "c1", []byte(rtReport(t, "", false, now, "")), store.HostReportDenorm{}); err != nil { t.Fatal(err) } if _, err := db.Exec(`UPDATE host_reports SET received_at = datetime('now', '-20 days')`); err != nil { t.Fatal(err) } var got []capturedEvent first := rtChecker(st, &got, nil) first.Check() if len(got) != 2 { t.Fatalf("two tiers unproven for 20 days: want 2 stale signals, got %v", got) } got = nil restarted := rtChecker(st, &got, nil) restarted.Check() restarted.Check() if len(got) != 0 { t.Fatalf("the restarted hub re-sent %d restore_test_stale signal(s)", len(got)) } // An earlier signal outside the read window does not hold the next one back. if _, err := db.Exec(`UPDATE events SET created_at = datetime('now', '-15 days') WHERE event_type = ?`, EventRestoreTestStale); err != nil { t.Fatal(err) } got = nil later := rtChecker(st, &got, nil) later.Check() if len(got) != 2 { t.Fatalf("a signal older than the read window must not silence the tier: got %v", got) } }