diff --git a/hub/internal/monitor/offsite.go b/hub/internal/monitor/offsite.go index 982d5ec2..1bd78645 100644 --- a/hub/internal/monitor/offsite.go +++ b/hub/internal/monitor/offsite.go @@ -43,6 +43,9 @@ type OffsiteChecker struct { // next sweep compares against. Only a TRUSTWORTHY report updates lastCounts — see snapshotDrop. dropStates map[string]string lastCounts map[string]int + // R-435: when the hub RECEIVED the report that set lastCounts — the start of the interval whose + // clean-up windows may explain the next fall (pinned tiers only, see snapshotDropped). + lastCountAt map[string]time.Time } const defaultOffsiteStaleAfter = 48 * time.Hour @@ -80,7 +83,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo oc := &OffsiteChecker{ store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now, fillStates: make(map[string]string), staleStates: make(map[string]string), - dropStates: make(map[string]string), lastCounts: make(map[string]int), + dropStates: make(map[string]string), lastCounts: make(map[string]int), lastCountAt: make(map[string]time.Time), } customers, err := s.GetCustomers() if err != nil { @@ -105,6 +108,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo // no baseline, and no baseline means no verdict. if oc.countIsTrustworthy(off) { oc.lastCounts[c.CustomerID] = off.SnapshotCount + oc.lastCountAt[c.CustomerID] = c.ReceivedAt oc.dropStates[c.CustomerID] = "ok" } } @@ -280,18 +284,55 @@ func (oc *OffsiteChecker) countIsTrustworthy(off *offsiteReport) bool { return true } +// windowSlack widens the start of the R-435 interval. The box closes a window inside its off-site run +// and measures the count a few minutes LATER in the same run, so a report sent in between still +// carries the pre-window count; without the slack the next interval would miss the window that +// explains its fall. Two hours covers a run comfortably; a window counted in two intervals only makes +// the detector more lenient, never noisier. +const windowSlack = 2 * time.Hour + +// pinnedTier reports whether the hub holds a CONFIRMED append-only key for the customer (decision 69): +// the box's own key cannot delete, so the only legitimate fall is a hub-opened clean-up window +// (decision 68). A read error is „not pinned" — the half-rule then applies, as before R-435. +func (oc *OffsiteChecker) pinnedTier(customerID string) bool { + k, err := oc.store.GetOffsiteKey(customerID) + return err == nil && k != nil && !k.ConfirmedAt.IsZero() +} + // snapshotDropped reports whether `off` is a drop worth alarming about, against the remembered -// baseline. Caller holds oc.mu. -func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport) (bool, int, int) { +// baseline. Caller holds oc.mu. The fourth result is true when the verdict came from the R-435 +// window rule (a pinned tier), so the message can say „outside any clean-up window". +// +// R-435 (D7, `09` §3 decision 191): on a PINNED tier any fall the hub's own windows do not explain +// alarms — even one snapshot. A window that cannot say what it removed (timeout, still open) explains +// anything: no alarm, one INFO line. Non-pinned tiers (the household's NAS, which still prunes by +// itself) keep the half-rule below. Pinned by r435_pinned_drop_test.go. +// +// LIMIT: the count is the box's own (R-895) and it is a NET count — a night's new snapshots between +// two reports hide the same number of deletions. +func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport, reportAt time.Time) (bool, int, int, bool) { prev, had := oc.lastCounts[customerID] if !had || prev <= 0 { - return false, prev, off.SnapshotCount + return false, prev, off.SnapshotCount, false } drop := prev - off.SnapshotCount - if drop < snapshotDropFloor { - return false, prev, off.SnapshotCount + if drop > 0 && oc.pinnedTier(customerID) { + from := oc.lastCountAt[customerID].Add(-windowSlack) + removed, unknown := oc.store.RemovedByWindowsBetween(customerID, from, reportAt) + if unknown { + oc.logger.Printf("[INFO] Offsite snapshot count for %s fell %d -> %d with a clean-up window that cannot say what it removed (timeout or still open) — treated as explained", customerID, prev, off.SnapshotCount) + return false, prev, off.SnapshotCount, true + } + if drop > removed { + return true, prev, off.SnapshotCount, true + } + oc.logger.Printf("[DEBUG] Offsite snapshot count for %s fell %d -> %d, explained by clean-up windows (%d removed)", customerID, prev, off.SnapshotCount, removed) + return false, prev, off.SnapshotCount, true } - return float64(drop) > float64(prev)*snapshotDropFraction, prev, off.SnapshotCount + if drop < snapshotDropFloor { + return false, prev, off.SnapshotCount, false + } + return float64(drop) > float64(prev)*snapshotDropFraction, prev, off.SnapshotCount, false } // emitSnapshotDrop — one event, at a severity inside the hub's exact vocabulary (08 §6.1). @@ -311,10 +352,15 @@ func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport) // sentence that asserts neither loss nor recovery is true under EVERY possible answer to the provider // question, so it never needs a second rewrite. An alarm rewritten twice in a week is worse than one // rewritten once, because the operator learns its words do not mean anything. -func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport, prev, cur int) { +func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport, prev, cur int, outsideWindow bool) { + why := "more than retention can explain" + if outsideWindow { + // R-435: on a pinned tier the box's key cannot delete; only a hub-opened window may. + why = "outside any clean-up window the hub opened" + } message := fmt.Sprintf( - "Customer %s: off-site backup count fell from %d to %d snapshot(s) in one report — more than "+ - "retention can explain. The daily Storage Box snapshots are read-only and still hold the "+ + "Customer %s: off-site backup count fell from %d to %d snapshot(s) in one report — "+why+ + ". The daily Storage Box snapshots are read-only and still hold the "+ "older copy. The route back out of them is not yet established, so treat this as neither "+ "confirmed data loss nor confirmed recovery. Get in touch before restoring anything, and "+ "check whether a deletion ran on the box.", @@ -322,6 +368,7 @@ func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "previous_count": prev, "current_count": cur, "drop": prev - cur, "last_success": off.LastSuccess, "last_status": off.LastStatus, + "outside_window": outsideWindow, }) oc.logger.Printf("[INFO] Offsite snapshot drop: %s %d -> %d (offsite_snapshots_dropped)", customerID, prev, cur) if _, err := oc.store.SaveEvent(customerID, "offsite_snapshots_dropped", "error", message, string(details), "hub"); err != nil { @@ -400,6 +447,7 @@ func (oc *OffsiteChecker) Check() { delete(oc.staleStates, c.CustomerID) delete(oc.dropStates, c.CustomerID) delete(oc.lastCounts, c.CustomerID) // no baseline survives a vanished object + delete(oc.lastCountAt, c.CustomerID) continue } if oc.store.IsCustomerBlocked(c.CustomerID) { @@ -407,6 +455,7 @@ func (oc *OffsiteChecker) Check() { delete(oc.staleStates, c.CustomerID) delete(oc.dropStates, c.CustomerID) delete(oc.lastCounts, c.CustomerID) + delete(oc.lastCountAt, c.CustomerID) continue } @@ -448,14 +497,15 @@ func (oc *OffsiteChecker) Check() { // alone rather than being overwritten with a number nobody could measure. That is what stops // a `needs_credential` zero from becoming the baseline and making the RECOVERY look like a rise. if oc.countIsTrustworthy(off) { - dropped, prev, cur := oc.snapshotDropped(c.CustomerID, off) + dropped, prev, cur, byWindow := oc.snapshotDropped(c.CustomerID, off, c.ReceivedAt) if dropped && oc.dropStates[c.CustomerID] != "dropped" { - oc.emitSnapshotDrop(c.CustomerID, off, prev, cur) + oc.emitSnapshotDrop(c.CustomerID, off, prev, cur, byWindow) oc.dropStates[c.CustomerID] = "dropped" } else if !dropped { oc.dropStates[c.CustomerID] = "ok" } oc.lastCounts[c.CustomerID] = cur + oc.lastCountAt[c.CustomerID] = c.ReceivedAt } } for k := range oc.fillStates { @@ -472,6 +522,7 @@ func (oc *OffsiteChecker) Check() { if !seen[k] { delete(oc.dropStates, k) delete(oc.lastCounts, k) + delete(oc.lastCountAt, k) } } } diff --git a/hub/internal/monitor/offsite_r431_test.go b/hub/internal/monitor/offsite_r431_test.go index 73d18095..ac1aa3c5 100644 --- a/hub/internal/monitor/offsite_r431_test.go +++ b/hub/internal/monitor/offsite_r431_test.go @@ -261,7 +261,7 @@ func TestR431_RealHistoryProducesZeroAlarms(t *testing.T) { } oc.mu.Lock() if oc.countIsTrustworthy(off) { - dropped, prev, cur := oc.snapshotDropped(cust, off) + dropped, prev, cur, _ := oc.snapshotDropped(cust, off, time.Time{}) if dropped && oc.dropStates[cust] != "dropped" { alarms++ t.Errorf("%s at %s: FIRED ON REAL HISTORY %d -> %d", cust, p.At, prev, cur) diff --git a/hub/internal/monitor/r435_pinned_drop_test.go b/hub/internal/monitor/r435_pinned_drop_test.go new file mode 100644 index 00000000..ee6955f9 --- /dev/null +++ b/hub/internal/monitor/r435_pinned_drop_test.go @@ -0,0 +1,148 @@ +package monitor + +import ( + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-435 (D7, `09` §3 decision 191). On a PINNED tier (the hub holds a confirmed append-only key) the +// only legitimate fall of the off-site snapshot count is a clean-up window the hub opened, so any fall +// the windows do not explain alarms — even one snapshot. Non-pinned tiers keep the half-rule. +// +// RED-PROOF (2026-10-08): on the pre-R-435 checker TestR435_PinnedUnexplainedFallAlarms fails — +// 69 → 60 is 9 < 34.5, so no event. + +type r435Setup struct { + pinned, confirmed bool + window func(st *store.Store) // runs between the baseline and the next report + next string +} + +func r435Run(t *testing.T, cust string, s r435Setup) []string { + t.Helper() + st := newDiskStore(t) + if s.pinned { + if err := st.RecordOffsiteKeyInstalled(cust, "fp-1"); err != nil { + t.Fatal(err) + } + if s.confirmed { + if ok, err := st.RecordOffsiteKeyConfirmed(cust, "fp-1"); err != nil || !ok { + t.Fatalf("confirm: %v %v", ok, err) + } + } + } + var msgs []string + saveOffsiteReport(t, st, cust, dropJSON(69, true, "", "ok")) + oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, sev, msg, _, _ string) { + if et == "offsite_snapshots_dropped" { + if sev != "error" { + t.Errorf("severity %q, want error", sev) + } + msgs = append(msgs, msg) + } + }, quietLog()) + if s.window != nil { + s.window(st) + } + saveOffsiteReport(t, st, cust, s.next) + oc.Check() + return msgs +} + +func closedWindow(t *testing.T, cust string, before, after int) func(*store.Store) { + return func(st *store.Store) { + id, err := st.OpenOffsiteWindowRow(cust, time.Now().Add(30*time.Minute), before, 10) + if err != nil { + t.Fatal(err) + } + if _, err := st.CloseOffsiteWindowRow(id, after, "ok", "box"); err != nil { + t.Fatal(err) + } + } +} + +func TestR435_PinnedUnexplainedFallAlarms(t *testing.T) { + msgs := r435Run(t, "p1", r435Setup{pinned: true, confirmed: true, next: dropJSON(60, true, "", "ok")}) + if len(msgs) != 1 { + t.Fatalf("pinned 69 -> 60 with no window: want exactly 1 offsite_snapshots_dropped, got %d", len(msgs)) + } + if !strings.Contains(msgs[0], "outside any clean-up window") || !strings.Contains(msgs[0], "69") || !strings.Contains(msgs[0], "60") { + t.Fatalf("message must name both counts and say „outside any clean-up window\": %s", msgs[0]) + } +} + +func TestR435_PinnedOneSnapshotAlarms(t *testing.T) { + if msgs := r435Run(t, "p2", r435Setup{pinned: true, confirmed: true, next: dropJSON(68, true, "", "ok")}); len(msgs) != 1 { + t.Fatalf("pinned 69 -> 68: even one snapshot must alarm, got %d", len(msgs)) + } +} + +func TestR435_WindowExplainsFall(t *testing.T) { + if msgs := r435Run(t, "p3", r435Setup{pinned: true, confirmed: true, + window: closedWindow(t, "p3", 69, 60), next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 { + t.Fatalf("a closed window that removed 9 explains 69 -> 60; got %d alarm(s): %v", len(msgs), msgs) + } +} + +func TestR435_WindowExplainsOnlyPart(t *testing.T) { + if msgs := r435Run(t, "p4", r435Setup{pinned: true, confirmed: true, + window: closedWindow(t, "p4", 69, 64), next: dropJSON(60, true, "", "ok")}); len(msgs) != 1 { + t.Fatalf("a window that removed 5 does not explain 69 -> 60; got %d alarm(s)", len(msgs)) + } +} + +func TestR435_TimeoutWindowIsUnknownNoAlarm(t *testing.T) { + timeout := func(st *store.Store) { + id, err := st.OpenOffsiteWindowRow("p5", time.Now().Add(30*time.Minute), 69, 10) + if err != nil { + t.Fatal(err) + } + _, _ = st.CloseOffsiteWindowRow(id, -1, "", "timeout") + } + if msgs := r435Run(t, "p5", r435Setup{pinned: true, confirmed: true, window: timeout, next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 { + t.Fatalf("a timeout-closed window cannot say what it removed → no alarm; got %d", len(msgs)) + } +} + +// Controls: the half-rule still governs a non-pinned tier, and an installed-but-unconfirmed key. +func TestR435_NotPinnedKeepsHalfRule(t *testing.T) { + if msgs := r435Run(t, "n1", r435Setup{next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 { + t.Fatalf("NAS tier 69 -> 60 must stay silent (half-rule); got %d", len(msgs)) + } + if msgs := r435Run(t, "n2", r435Setup{pinned: true, next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 { + t.Fatalf("an unconfirmed key is not pinned; 69 -> 60 must stay silent; got %d", len(msgs)) + } + if msgs := r435Run(t, "n3", r435Setup{next: dropJSON(4, true, "", "ok")}); len(msgs) != 1 { + t.Fatalf("NAS tier 69 -> 4 still alarms by the half-rule; got %d", len(msgs)) + } else if strings.Contains(msgs[0], "outside any clean-up window") { + t.Fatalf("the half-rule message must not claim a window rule: %s", msgs[0]) + } +} + +// An untrustworthy report on a pinned tier: no alarm, and the baseline does not move. +func TestR435_UntrustworthyReportNoAlarmBaselineKept(t *testing.T) { + st := newDiskStore(t) + _ = st.RecordOffsiteKeyInstalled("u1", "fp") + _, _ = st.RecordOffsiteKeyConfirmed("u1", "fp") + n := 0 + saveOffsiteReport(t, st, "u1", dropJSON(69, true, "", "ok")) + oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { + if et == "offsite_snapshots_dropped" { + n++ + } + }, quietLog()) + saveOffsiteReport(t, st, "u1", dropJSON(0, false, "", "ok")) // stats unknown + oc.Check() + if n != 0 { + t.Fatalf("untrustworthy report must not alarm; got %d", n) + } + oc.mu.Lock() + base := oc.lastCounts["u1"] + oc.mu.Unlock() + if base != 69 { + t.Fatalf("baseline must stay 69 after an untrustworthy report, got %d", base) + } +} diff --git a/hub/internal/store/offsite_keys.go b/hub/internal/store/offsite_keys.go index e2f3f75d..3f384129 100644 --- a/hub/internal/store/offsite_keys.go +++ b/hub/internal/store/offsite_keys.go @@ -324,3 +324,43 @@ func (s *Store) ForceOffsiteAbandonDueForTest(id int64) error { _, err := s.db.Exec(`UPDATE offsite_abandon_requests SET due_at = datetime('now', '-1 minute') WHERE id = ?`, id) return err } + +// RemovedByWindowsBetween — R-435 (D7, `09` §3 decision 191). How many snapshots the clean-up windows +// the hub opened for this customer EXPLAIN between two reports: the sum of count_before − count_after +// over windows closed in (from, to]. On a pinned tier these windows are the only legitimate way the +// count can fall, so anything beyond the sum is unexplained. +// +// unknown = true when a window in the interval cannot say what it removed: closed by timeout (no box +// result, count_after −1 or NULL), or still open (opened at or before `to`, not closed). The caller +// then treats the fall as explained — the safe side for noise, recorded as a limit in `08`. A query +// error is unknown too. +func (s *Store) RemovedByWindowsBetween(customerID string, from, to time.Time) (removed int, unknown bool) { + const f = "2006-01-02 15:04:05" + rows, err := s.db.Query(` + SELECT count_before, count_after, closed_at IS NULL FROM offsite_windows + WHERE customer_id = ? + AND ((closed_at IS NULL AND opened_at <= ?) OR (closed_at > ? AND closed_at <= ?))`, + customerID, to.UTC().Format(f), from.UTC().Format(f), to.UTC().Format(f)) + if err != nil { + return 0, true + } + defer rows.Close() + for rows.Next() { + var before, after sql.NullInt64 + var open bool + if err := rows.Scan(&before, &after, &open); err != nil { + return 0, true + } + if open || !before.Valid || !after.Valid || after.Int64 < 0 { + unknown = true + continue + } + if d := int(before.Int64 - after.Int64); d > 0 { + removed += d + } + } + if rows.Err() != nil { + return 0, true + } + return removed, unknown +} diff --git a/hub/internal/store/r435_windows_between_test.go b/hub/internal/store/r435_windows_between_test.go new file mode 100644 index 00000000..547f1da6 --- /dev/null +++ b/hub/internal/store/r435_windows_between_test.go @@ -0,0 +1,46 @@ +package store + +import ( + "testing" + "time" +) + +// R-435: RemovedByWindowsBetween sums only windows closed inside the interval, and says „unknown" +// for a window that cannot say what it removed. +func TestR435_RemovedByWindowsBetween(t *testing.T) { + s := newTestStore(t) + at := func(ts string) time.Time { v, _ := time.Parse("2006-01-02 15:04:05", ts); return v } + ins := func(cust, opened, closed string, before, after any) { + t.Helper() + var c any + if closed != "" { + c = closed + } + if _, err := s.db.Exec(`INSERT INTO offsite_windows (customer_id, opened_at, closes_by, closed_at, count_before, count_after) VALUES (?, ?, ?, ?, ?, ?)`, + cust, opened, opened, c, before, after); err != nil { + t.Fatal(err) + } + } + ins("a", "2026-10-01 03:00:00", "2026-10-01 03:10:00", 69, 60) // inside → 9 + ins("a", "2026-09-20 03:00:00", "2026-09-20 03:10:00", 80, 69) // before the interval + ins("a", "2026-10-03 03:00:00", "2026-10-03 03:10:00", 60, 50) // after the interval + ins("b", "2026-10-01 03:00:00", "2026-10-01 03:10:00", 50, 40) // another customer + + from, to := at("2026-09-30 00:00:00"), at("2026-10-02 00:00:00") + if n, unk := s.RemovedByWindowsBetween("a", from, to); n != 9 || unk { + t.Fatalf("a: want 9 known, got %d unknown=%v", n, unk) + } + if n, unk := s.RemovedByWindowsBetween("c", from, to); n != 0 || unk { + t.Fatalf("no window: want 0 known, got %d unknown=%v", n, unk) + } + // a timeout close (count_after −1) inside the interval → unknown + ins("d", "2026-10-01 03:00:00", "2026-10-01 03:40:00", 69, -1) + if _, unk := s.RemovedByWindowsBetween("d", from, to); !unk { + t.Fatal("a timeout-closed window must make the interval unknown") + } + // a window still open → unknown + ins("e", "2026-10-01 03:00:00", "", 69, nil) + if _, unk := s.RemovedByWindowsBetween("e", from, to); !unk { + t.Fatal("an open window must make the interval unknown") + } +}