package monitor import ( "fmt" "strings" "testing" "time" ) // saveOffsiteReport records a customer report whose report_json carries the given offsite object // (or none, when raw==""). The 1.1s sleep before a RE-save of the same customer makes received_at // strictly increasing (second resolution) so GetCustomers' MAX(received_at) picks the new row // deterministically. var offsiteSaved = map[string]bool{} func saveOffsiteReport(t *testing.T, st interface { SaveReport(string, []byte) error }, customerID, raw string) { t.Helper() if offsiteSaved[customerID] { time.Sleep(1100 * time.Millisecond) } offsiteSaved[customerID] = true body := `{"customer_id":"` + customerID + `"` if raw != "" { body += `,"offsite":` + raw } body += `}` if err := st.SaveReport(customerID, []byte(body)); err != nil { t.Fatal(err) } } func offsiteJSON(enabled bool, escrow, lastRun, lastStatus string, sizeBytes int64, quotaGB int) string { return fmt.Sprintf(`{"enabled":%v,"escrow_state":%q,"last_run":%q,"last_status":%q,"snapshot_count":3,"repo_size_bytes":%d,"quota_gb":%d}`, enabled, escrow, lastRun, lastStatus, sizeBytes, quotaGB) } // Scenario D — staleness: enabled+escrowed with no run >48h (or never) alerts once (deduped across // sweeps); a pending customer NEVER alerts (normal onboarding — the red-proofed filter); recovery re-arms. func TestOffsite_StalenessAlert(t *testing.T) { st := newDiskStore(t) now := time.Now().UTC() old := now.Add(-72 * time.Hour).Format(time.RFC3339) fresh := now.Add(-1 * time.Hour).Format(time.RFC3339) // stale-c: escrowed, last run 72h ago → stale. pend-c: PENDING with the same old run → silent. saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50)) saveOffsiteReport(t, st, "pend-c", offsiteJSON(true, "pending", old, "", 1<<30, 50)) var types []string oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog()) oc.Check() if got := count(types, "offsite_stale"); got != 1 { t.Fatalf("want exactly 1 offsite_stale (stale-c only; pending must NEVER alert), got %d (%v)", got, types) } if oc.GetStaleState("pend-c") != "ok" { t.Fatalf("a pending customer must not be stale, got %s", oc.GetStaleState("pend-c")) } // repeated sweep with the same data → NO re-page (dedupe) oc.Check() if got := count(types, "offsite_stale"); got != 1 { t.Fatalf("staleness must not re-page every sweep, got %d", got) } // recovery: a fresh run clears + re-arms; going stale again re-alerts saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", fresh, "ok", 1<<30, 50)) oc.Check() if oc.GetStaleState("stale-c") != "ok" { t.Fatal("recovery must clear the stale state") } saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50)) oc.Check() if got := count(types, "offsite_stale"); got != 2 { t.Fatalf("re-staleness after recovery must alert again, got %d", got) } // never-ran escrowed customer is stale too saveOffsiteReport(t, st, "never-c", offsiteJSON(true, "escrowed", "", "", 0, 50)) oc.Check() if got := count(types, "offsite_stale"); got != 3 { t.Fatalf("a never-ran escrowed target must be stale, got %d", got) } } // Scenario E (fill) — 90/95 of quota_gb; quota 0 never alerts; escalation-only; recovery re-arms. func TestOffsite_FillAlert(t *testing.T) { st := newDiskStore(t) fresh := time.Now().UTC().Format(time.RFC3339) gb := int64(1) << 30 var types, sevs []string onEvent := func(_, et, sev, _, _, _ string) { types = append(types, et); sevs = append(sevs, sev) } saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 40*gb, 50)) // 80% — under warn saveOffsiteReport(t, st, "noq-c", offsiteJSON(true, "escrowed", fresh, "ok", 900*gb, 0)) // dedicated: quota 0 oc := NewOffsiteChecker(st, 48*time.Hour, onEvent, quietLog()) oc.Check() if len(types) != 0 { t.Fatalf("80%% and quota-0 must not alert, got %v", types) } saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 46*gb, 50)) // 92% warn oc.Check() if count(types, "offsite_fill_warning") != 1 { t.Fatalf("92%% must warn once, got %v", types) } oc.Check() // same data — no re-page if len(types) != 1 { t.Fatalf("no re-page on an unchanged band, got %v", types) } saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 48*gb, 50)) // 96% crit oc.Check() if count(types, "offsite_fill_critical") != 1 || sevs[len(sevs)-1] != "critical" { t.Fatalf("96%% must escalate to critical, got types=%v sevs=%v", types, sevs) } // recovery re-arms saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 10*gb, 50)) oc.Check() if oc.GetFillState("fill-c") != bandOK { t.Fatal("recovery must re-arm the fill state") } } // Nil-safety — reports without an offsite object (pre-v0.109 controllers / offbox disabled) are skipped: // no alert, no state. func TestOffsite_NilSafeOnOldReports(t *testing.T) { st := newDiskStore(t) saveOffsiteReport(t, st, "old-c", "") // no offsite key at all var types []string oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog()) oc.Check() if len(types) != 0 { t.Fatalf("a report without an offsite object must never alert, got %v", types) } if oc.GetStaleState("old-c") != "unknown" || oc.GetFillState("old-c") != "unknown" { t.Fatal("no state may exist for a customer without an offsite object") } } func count(list []string, want string) int { n := 0 for _, s := range list { if s == want { n++ } } return n } var _ = strings.Contains // keep strings import if unused by future edits