b119301c6f
gates / gates (push) Successful in 2m48s
Hub code unreleased; ships with tomorrow's hub release. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
165 lines
6.2 KiB
Go
165 lines
6.2 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// R-243 — a box whose off-site backup is ON but whose escrow was never done never backs up off-site, and
|
|
// until now nothing fired. Every test asserts the CONSEQUENCE: does the operator get the mail (an event
|
|
// through onEvent), how many, and when does it stop?
|
|
|
|
type r243Event struct{ typ, sev, msg string }
|
|
|
|
func r243Setup(t *testing.T, firstReportAgo time.Duration) (*store.Store, *OffsiteChecker, *[]r243Event) {
|
|
t.Helper()
|
|
st := newDiskStore(t) // customer c1, host h1
|
|
if err := st.SaveHostReport("h1", "c1", []byte(`{}`), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.SetHostReportsReceivedAtForTest("c1", sqliteAgo(firstReportAgo)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var evs []r243Event
|
|
oc := NewOffsiteChecker(st, 0, func(_, typ, sev, msg, _, _ string) {
|
|
evs = append(evs, r243Event{typ, sev, msg})
|
|
}, log.New(io.Discard, "", 0))
|
|
return st, oc, &evs
|
|
}
|
|
|
|
func r243Count(evs []r243Event, typ string) int {
|
|
n := 0
|
|
for _, e := range evs {
|
|
if e.typ == typ {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// THE ONE THAT MATTERS: on for 8 days, escrow pending, never ran → ONE warning to the operator; a second sweep
|
|
// sends nothing more. RED-PROOF: remove the oc.checkEscrowPending call from Check → zero events → FAILS.
|
|
func TestR243_PendingEightDays_OneMail(t *testing.T) {
|
|
st, oc, evs := r243Setup(t, 8*24*time.Hour)
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "pending", "", "", 0, 50))
|
|
oc.Check()
|
|
oc.Check()
|
|
if n := r243Count(*evs, eventEscrowPending); n != 1 {
|
|
t.Fatalf("offsite_escrow_pending mails = %d, want exactly 1 (events %v)", n, *evs)
|
|
}
|
|
e := (*evs)[0]
|
|
if e.sev != "warning" || !strings.Contains(e.msg, "escrow step is pending") {
|
|
t.Fatalf("event = %+v, want a warning naming the pending escrow", e)
|
|
}
|
|
if r243Count(*evs, "offsite_stale") != 0 {
|
|
t.Fatalf("a pending box must not raise offsite_stale (onboarding is not staleness): %v", *evs)
|
|
}
|
|
}
|
|
|
|
// Inside the 7-day line: the household may still be doing the step — nothing.
|
|
func TestR243_PendingSixDays_Silent(t *testing.T) {
|
|
st, oc, evs := r243Setup(t, 6*24*time.Hour)
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "pending", "", "", 0, 50))
|
|
oc.Check()
|
|
if len(*evs) != 0 {
|
|
t.Fatalf("6 days pending must be silent; got %v", *evs)
|
|
}
|
|
}
|
|
|
|
// Quiet for a week, then re-sent while still true; a hub restart in between neither re-mails nor forgets.
|
|
func TestR243_WeeklyResendAndRestartSafe(t *testing.T) {
|
|
st, oc, evs := r243Setup(t, 8*24*time.Hour)
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "pending", "", "", 0, 50))
|
|
oc.Check()
|
|
|
|
// A hub restart: a new checker over the same store.
|
|
oc2 := NewOffsiteChecker(st, 0, func(_, typ, sev, msg, _, _ string) {
|
|
*evs = append(*evs, r243Event{typ, sev, msg})
|
|
}, log.New(io.Discard, "", 0))
|
|
oc2.Check()
|
|
if n := r243Count(*evs, eventEscrowPending); n != 1 {
|
|
t.Fatalf("after a restart: %d mails, want still 1", n)
|
|
}
|
|
oc2.now = func() time.Time { return time.Now().Add(6 * 24 * time.Hour) }
|
|
oc2.Check()
|
|
if n := r243Count(*evs, eventEscrowPending); n != 1 {
|
|
t.Fatalf("6 days after the mail: %d mails, want still 1 (quiet for a week)", n)
|
|
}
|
|
oc2.now = func() time.Time { return time.Now().Add(8 * 24 * time.Hour) }
|
|
oc2.Check()
|
|
if n := r243Count(*evs, eventEscrowPending); n != 2 {
|
|
t.Fatalf("8 days after the mail and still pending: %d mails, want 2 (weekly re-send)", n)
|
|
}
|
|
}
|
|
|
|
// It clears when the escrow is done, with one info line (recorded, never mailed), and the next episode alarms again.
|
|
func TestR243_ClearsWhenEscrowed(t *testing.T) {
|
|
st, oc, evs := r243Setup(t, 8*24*time.Hour)
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "pending", "", "", 0, 50))
|
|
oc.Check()
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
|
|
oc.Check()
|
|
if n := r243Count(*evs, eventEscrowPendingCleared); n != 1 {
|
|
t.Fatalf("cleared events = %d, want 1 (events %v)", n, *evs)
|
|
}
|
|
if !st.OSAlarmRaised(escrowPendingKey("c1")).IsZero() {
|
|
t.Fatal("the raise record must be cleared")
|
|
}
|
|
oc.Check()
|
|
if n := r243Count(*evs, eventEscrowPendingCleared); n != 1 {
|
|
t.Fatalf("the clear is one line, not one per sweep: %d", n)
|
|
}
|
|
}
|
|
|
|
// A box that ran once and then fell back to pending counts from its last SUCCESSFUL run.
|
|
func TestR243_FellBackToPending_CountsFromLastSuccess(t *testing.T) {
|
|
st, oc, evs := r243Setup(t, 60*24*time.Hour)
|
|
last := time.Now().UTC().Add(-3 * 24 * time.Hour).Format(time.RFC3339)
|
|
saveOffsiteReport(t, st, "c1", `{"enabled":true,"escrow_state":"pending","last_run":"`+last+`","last_status":"ok","last_success":"`+last+`"}`)
|
|
oc.Check()
|
|
if len(*evs) != 0 {
|
|
t.Fatalf("last success 3 days ago → silent; got %v", *evs)
|
|
}
|
|
old := time.Now().UTC().Add(-9 * 24 * time.Hour).Format(time.RFC3339)
|
|
saveOffsiteReport(t, st, "c1", `{"enabled":true,"escrow_state":"pending","last_run":"`+old+`","last_status":"ok","last_success":"`+old+`","state":"awaiting_recovery_key"}`)
|
|
oc.Check()
|
|
if n := r243Count(*evs, eventEscrowPending); n != 1 {
|
|
t.Fatalf("last success 9 days ago and pending → 1 mail; got %v", *evs)
|
|
}
|
|
if !strings.Contains((*evs)[0].msg, "awaiting_recovery_key") {
|
|
t.Errorf("the mail should name the box's declared state; got %q", (*evs)[0].msg)
|
|
}
|
|
}
|
|
|
|
// Off-site OFF, or already escrowed, never raises this.
|
|
func TestR243_OffOrEscrowedNeverRaises(t *testing.T) {
|
|
for _, js := range []string{
|
|
offsiteJSON(false, "pending", "", "", 0, 50),
|
|
offsiteJSON(true, "escrowed", time.Now().UTC().Format(time.RFC3339), "ok", 0, 50),
|
|
} {
|
|
st, oc, evs := r243Setup(t, 30*24*time.Hour)
|
|
saveOffsiteReport(t, st, "c1", js)
|
|
oc.Check()
|
|
if r243Count(*evs, eventEscrowPending) != 0 {
|
|
t.Fatalf("report %s raised offsite_escrow_pending: %v", js, *evs)
|
|
}
|
|
}
|
|
}
|
|
|
|
// No anchor (no host report, no success) → not judged (never a guess-fire).
|
|
func TestR243_NoAnchor_Silent(t *testing.T) {
|
|
st := newDiskStore(t)
|
|
var evs []r243Event
|
|
oc := NewOffsiteChecker(st, 0, func(_, typ, sev, msg, _, _ string) { evs = append(evs, r243Event{typ, sev, msg}) }, log.New(io.Discard, "", 0))
|
|
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "pending", "", "", 0, 50))
|
|
oc.Check()
|
|
if len(evs) != 0 {
|
|
t.Fatalf("no anchor must not fire: %v", evs)
|
|
}
|
|
}
|