package monitor import ( "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "time" ) // R-431 — the snapshot-drop signal, proven in BOTH directions. // // The acceptance test is TestR431_RealHistoryProducesZeroAlarms: a detector that fires on healthy // boxes is the mistake this project caught twice in one week, and it is the one that would get this // signal switched off within a fortnight. // dropJSON builds a trustworthy offsite object (stats_known, no declared state, last run ok). func dropJSON(count int, statsKnown bool, state, lastStatus string) string { ts := time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) success := ts if lastStatus != "ok" { success = "" } return fmt.Sprintf( `{"enabled":true,"escrow_state":"escrowed","last_run":%q,"last_status":%q,"last_success":%q,`+ `"snapshot_count":%d,"repo_size_bytes":1073741824,"quota_gb":0,"stats_known":%v,"state":%q}`, ts, lastStatus, success, count, statsKnown, state) } // TestR431_FiresOnAMassDeletion — direction 1. A drop past the threshold alarms EXACTLY once. // // RED-PROOF (run 2026-09-01, recorded in REPORT.md): setting snapshotDropFraction to 0.99 makes this // fail — 69 → 4 is a 94% fall and would no longer qualify, which is what an over-loose threshold // looks like in production. func TestR431_FiresOnAMassDeletion(t *testing.T) { st := newDiskStore(t) var got []struct{ et, sev, msg string } saveOffsiteReport(t, st, "victim", dropJSON(69, true, "", "ok")) oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, sev, msg, _, _ string) { got = append(got, struct{ et, sev, msg string }{et, sev, msg}) }, quietLog()) // the constructor seeded the baseline at 69; now the store is emptied saveOffsiteReport(t, st, "victim", dropJSON(4, true, "", "ok")) oc.Check() var drops []struct{ et, sev, msg string } for _, g := range got { if g.et == "offsite_snapshots_dropped" { drops = append(drops, g) } } if len(drops) != 1 { t.Fatalf("want exactly 1 offsite_snapshots_dropped, got %d (%v)", len(drops), got) } // The severity MUST be in the hub's exact vocabulary — anything else is coerced to info and // mailed to nobody (08 §6.1, shipped twice). switch drops[0].sev { case "info", "warning", "error", "critical": default: t.Fatalf("severity %q is outside the hub vocabulary — it would be coerced to info and reach nobody", drops[0].sev) } // These fragments belong to the SIGNAL — the two counts, and where the older copy still is. // // THE WORDING FRAGMENT THAT USED TO SIT HERE IS GONE ON PURPOSE. This list asserted // "NOT confirmed data loss" until 2026-09-01, and it caught the R-434 fix, correctly — the // sentence changed because the alarm was promising a recovery that R-433 showed cannot be // performed. The wording now has ONE home, `offsite_r434_test.go`, which pins both what the // message must say and what it must never say again. Duplicating it here would create the // second source that makes the next correction land in one file and not the other. for _, frag := range []string{"69", "4", "read-only"} { if !strings.Contains(drops[0].msg, frag) { t.Fatalf("message must contain %q; got: %s", frag, drops[0].msg) } } if strings.Contains(drops[0].msg, "data is lost") || strings.Contains(drops[0].msg, "data lost") { t.Fatalf("the message must NOT claim data loss — the snapshots usually still hold it: %s", drops[0].msg) } } // TestR431_EscalationOnlyLatch — a CONTINUING deletion must not page on every report cycle. // // WRITTEN AFTER A HOLLOW FIRST ATTEMPT, and the failure is recorded because it is instructive: the // original assertion re-swept the SAME report and called that "escalation-only". It proved nothing — // the baseline had already moved to the new count, so the second sweep saw a drop of zero and the // latch was never consulted. Its red-proof (removing the latch) PASSED, which is how it was caught. // // This drives a count that keeps FALLING, which is the only shape where the latch is load-bearing. // // RED-PROOF (run 2026-09-01): replacing `dropped && oc.dropStates[...] != "dropped"` with `dropped` // makes this fail with 2 alarms — one per report cycle, which is the noise that trains an operator // to ignore the alarm. func TestR431_EscalationOnlyLatch(t *testing.T) { st := newDiskStore(t) var n int saveOffsiteReport(t, st, "sliding", dropJSON(69, true, "", "ok")) oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { if et == "offsite_snapshots_dropped" { n++ } }, quietLog()) saveOffsiteReport(t, st, "sliding", dropJSON(30, true, "", "ok")) // 69 -> 30: alarm oc.Check() if n != 1 { t.Fatalf("the first large drop must alarm exactly once; got %d", n) } saveOffsiteReport(t, st, "sliding", dropJSON(2, true, "", "ok")) // 30 -> 2: still falling oc.Check() if n != 1 { t.Fatalf("a CONTINUING deletion must not re-page while the latch is set; got %d alarms", n) } if oc.GetDropState("sliding") != "dropped" { t.Fatalf("the latch must be held, got %q", oc.GetDropState("sliding")) } // RECOVERY RE-ARMS: a clean sweep clears the latch, so a LATER deletion is caught again. saveOffsiteReport(t, st, "sliding", dropJSON(40, true, "", "ok")) // rebuilt, no drop oc.Check() if oc.GetDropState("sliding") != "ok" { t.Fatalf("a clean sweep must re-arm the latch, got %q", oc.GetDropState("sliding")) } saveOffsiteReport(t, st, "sliding", dropJSON(1, true, "", "ok")) // deleted again oc.Check() if n != 2 { t.Fatalf("after re-arming, a NEW deletion must alarm again; got %d", n) } } // TestR431_SilentWhenNotTrustworthy — direction 2, the three pre-conditions, each with its scar. func TestR431_SilentWhenNotTrustworthy(t *testing.T) { cases := []struct { name, first, second string }{ {"stats_known absent (R-331: a zero that means UNMEASURED)", dropJSON(69, true, "", "ok"), dropJSON(0, false, "", "ok")}, {"a DECLARED state (R-204: the box says what happened)", dropJSON(69, true, "", "ok"), dropJSON(0, true, "needs_credential", "ok")}, {"the run FAILED (R-100: presence is not success)", dropJSON(69, true, "", "ok"), dropJSON(0, true, "", "error")}, {"the run was INCOMPLETE (R-203: a partial run counts less)", dropJSON(69, true, "", "ok"), dropJSON(0, true, "", "incomplete")}, } for i, c := range cases { t.Run(c.name, func(t *testing.T) { st := newDiskStore(t) cid := fmt.Sprintf("c%d", i) var n int saveOffsiteReport(t, st, cid, c.first) oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { if et == "offsite_snapshots_dropped" { n++ } }, quietLog()) saveOffsiteReport(t, st, cid, c.second) oc.Check() if n != 0 { t.Fatalf("%s: must NOT alarm; got %d", c.name, n) } // AND the baseline must be untouched, so the RECOVERY does not read as a rise-then-drop. oc.mu.Lock() base := oc.lastCounts[cid] oc.mu.Unlock() if base != 69 { t.Fatalf("%s: an untrustworthy report must not overwrite the baseline; got %d", c.name, base) } }) } } // TestR431_OrdinaryRetentionIsSilent — a fall that retention CAN explain must not alarm. func TestR431_OrdinaryRetentionIsSilent(t *testing.T) { for _, c := range []struct { name string from, to int wantAlarms int }{ {"69 -> 60 (9 gone, under half)", 69, 60, 0}, {"10 -> 7 (3 gone, under the floor of 5)", 10, 7, 0}, {"10 -> 5 (5 gone, exactly half — NOT more than half)", 10, 5, 0}, {"69 -> 34 (35 gone, more than half)", 69, 34, 1}, } { t.Run(c.name, func(t *testing.T) { st := newDiskStore(t) cid := fmt.Sprintf("r%d%d", c.from, c.to) var n int saveOffsiteReport(t, st, cid, dropJSON(c.from, true, "", "ok")) oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { if et == "offsite_snapshots_dropped" { n++ } }, quietLog()) saveOffsiteReport(t, st, cid, dropJSON(c.to, true, "", "ok")) oc.Check() if n != c.wantAlarms { t.Fatalf("%s: want %d alarm(s), got %d", c.name, c.wantAlarms, n) } }) } } // TestR431_RealHistoryProducesZeroAlarms — THE ACCEPTANCE STEP. // // Replays the ACTUAL snapshot-count history of both live boxes, exported from the hub's own reports // table on 2026-09-01, through the real detector. It must produce ZERO alarms. A warning that fires // on healthy boxes is worse than no warning at all. // // The fixture is committed beside this test so the assertion does not depend on a live database. // If it is absent the test FAILS rather than skipping — a silent skip is how a green tick comes to // mean nothing. func TestR431_RealHistoryProducesZeroAlarms(t *testing.T) { path := filepath.Join("testdata", "r431_real_history.json") raw, err := os.ReadFile(path) if err != nil { t.Fatalf("the real-history fixture is missing (%v) — this is a FAILURE, never a skip: "+ "without it the acceptance step asserts nothing", err) } var hist map[string][]struct { At string `json:"at"` Count int `json:"count"` StatsKnown bool `json:"stats_known"` State string `json:"state"` LastStatus string `json:"last_status"` } if err := json.Unmarshal(raw, &hist); err != nil { t.Fatal(err) } if len(hist) == 0 { t.Fatal("the fixture is empty — it would pass vacuously") } total := 0 for cust, points := range hist { if len(points) < 2 { t.Fatalf("%s: fewer than 2 points — nothing to compare", cust) } total += len(points) st := newDiskStore(t) var alarms int var first = points[0] saveOffsiteReport(t, st, cust, dropJSON(first.Count, first.StatsKnown, first.State, first.LastStatus)) oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, msg, _, _ string) { if et == "offsite_snapshots_dropped" { alarms++ t.Errorf("%s: FIRED ON REAL HISTORY: %s", cust, msg) } }, quietLog()) // Feed the remaining points straight through the detector. Driving Check() per point would // need a 1.1s sleep each time (received_at is second-resolution) — hours for 5 000 points — // so the sweep's own decision path is exercised directly instead, with the same guards. for _, p := range points[1:] { off := &offsiteReport{ SnapshotCount: p.Count, StatsKnown: p.StatsKnown, State: p.State, LastStatus: p.LastStatus, LastSuccess: "2026-09-01T00:00:00Z", } if p.LastStatus != "ok" { off.LastSuccess = "" } oc.mu.Lock() if oc.countIsTrustworthy(off) { dropped, prev, cur := oc.snapshotDropped(cust, off) if dropped && oc.dropStates[cust] != "dropped" { alarms++ t.Errorf("%s at %s: FIRED ON REAL HISTORY %d -> %d", cust, p.At, prev, cur) oc.dropStates[cust] = "dropped" } else if !dropped { oc.dropStates[cust] = "ok" } oc.lastCounts[cust] = cur } oc.mu.Unlock() } if alarms != 0 { t.Fatalf("%s: %d alarm(s) on real history — the threshold is wrong", cust, alarms) } } // POSITIVE CONTROL: the replay must actually have looked at something. Without this the test // passes when the fixture is a list of empty lists. if total < 100 { t.Fatalf("only %d points replayed — too few for this to mean anything", total) } t.Logf("replayed %d real report points across %d customers: ZERO alarms", total, len(hist)) }