hub: one lost off-site snapshot outside a clean-up window is an error (R-435, D7)

On a pinned tier (the hub holds a CONFIRMED append-only key) the only
legitimate fall of the box's snapshot count is a clean-up window the hub
opened. The checker now compares prev - cur with what the windows closed
since the previous trustworthy report removed (store.RemovedByWindowsBetween,
2 h slack for the in-run count lag); any unexplained fall, even one snapshot,
raises offsite_snapshots_dropped (error) saying 'outside any clean-up window
the hub opened'. A window that cannot say what it removed (timeout, still
open) explains anything: no alarm, one INFO line. Non-pinned tiers keep the
half-rule. Untrustworthy reports: unchanged (no alarm, baseline kept).

Red tests: r435_pinned_drop_test.go (3 fail with the pinned rule disabled;
WindowExplainsFall fails when windows are ignored), r435_windows_between_test.go.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/monitor/offsite.go

@@ -43,6 +43,9 @@ type OffsiteChecker struct {
 	// next sweep compares against. Only a TRUSTWORTHY report updates lastCounts — see snapshotDrop.
 	dropStates map[string]string
 	lastCounts map[string]int
+	// R-435: when the hub RECEIVED the report that set lastCounts — the start of the interval whose
+	// clean-up windows may explain the next fall (pinned tiers only, see snapshotDropped).
+	lastCountAt map[string]time.Time
 }
 
 const defaultOffsiteStaleAfter = 48 * time.Hour
@@ -80,7 +83,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
 	oc := &OffsiteChecker{
 		store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now,
 		fillStates: make(map[string]string), staleStates: make(map[string]string),
-		dropStates: make(map[string]string), lastCounts: make(map[string]int),
+		dropStates: make(map[string]string), lastCounts: make(map[string]int), lastCountAt: make(map[string]time.Time),
 	}
 	customers, err := s.GetCustomers()
 	if err != nil {
@@ -105,6 +108,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
 		// no baseline, and no baseline means no verdict.
 		if oc.countIsTrustworthy(off) {
 			oc.lastCounts[c.CustomerID] = off.SnapshotCount
+			oc.lastCountAt[c.CustomerID] = c.ReceivedAt
 			oc.dropStates[c.CustomerID] = "ok"
 		}
 	}
@@ -280,18 +284,55 @@ func (oc *OffsiteChecker) countIsTrustworthy(off *offsiteReport) bool {
 	return true
 }
 
+// windowSlack widens the start of the R-435 interval. The box closes a window inside its off-site run
+// and measures the count a few minutes LATER in the same run, so a report sent in between still
+// carries the pre-window count; without the slack the next interval would miss the window that
+// explains its fall. Two hours covers a run comfortably; a window counted in two intervals only makes
+// the detector more lenient, never noisier.
+const windowSlack = 2 * time.Hour
+
+// pinnedTier reports whether the hub holds a CONFIRMED append-only key for the customer (decision 69):
+// the box's own key cannot delete, so the only legitimate fall is a hub-opened clean-up window
+// (decision 68). A read error is „not pinned" — the half-rule then applies, as before R-435.
+func (oc *OffsiteChecker) pinnedTier(customerID string) bool {
+	k, err := oc.store.GetOffsiteKey(customerID)
+	return err == nil && k != nil && !k.ConfirmedAt.IsZero()
+}
+
 // snapshotDropped reports whether `off` is a drop worth alarming about, against the remembered
-// baseline. Caller holds oc.mu.
-func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport) (bool, int, int) {
+// baseline. Caller holds oc.mu. The fourth result is true when the verdict came from the R-435
+// window rule (a pinned tier), so the message can say „outside any clean-up window".
+//
+// R-435 (D7, `09` §3 decision 191): on a PINNED tier any fall the hub's own windows do not explain
+// alarms — even one snapshot. A window that cannot say what it removed (timeout, still open) explains
+// anything: no alarm, one INFO line. Non-pinned tiers (the household's NAS, which still prunes by
+// itself) keep the half-rule below. Pinned by r435_pinned_drop_test.go.
+//
+// LIMIT: the count is the box's own (R-895) and it is a NET count — a night's new snapshots between
+// two reports hide the same number of deletions.
+func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport, reportAt time.Time) (bool, int, int, bool) {
 	prev, had := oc.lastCounts[customerID]
 	if !had || prev <= 0 {
-		return false, prev, off.SnapshotCount
+		return false, prev, off.SnapshotCount, false
 	}
 	drop := prev - off.SnapshotCount
-	if drop < snapshotDropFloor {
-		return false, prev, off.SnapshotCount
+	if drop > 0 && oc.pinnedTier(customerID) {
+		from := oc.lastCountAt[customerID].Add(-windowSlack)
+		removed, unknown := oc.store.RemovedByWindowsBetween(customerID, from, reportAt)
+		if unknown {
+			oc.logger.Printf("[INFO] Offsite snapshot count for %s fell %d -> %d with a clean-up window that cannot say what it removed (timeout or still open) — treated as explained", customerID, prev, off.SnapshotCount)
+			return false, prev, off.SnapshotCount, true
+		}
+		if drop > removed {
+			return true, prev, off.SnapshotCount, true
+		}
+		oc.logger.Printf("[DEBUG] Offsite snapshot count for %s fell %d -> %d, explained by clean-up windows (%d removed)", customerID, prev, off.SnapshotCount, removed)
+		return false, prev, off.SnapshotCount, true
 	}
-	return float64(drop) > float64(prev)*snapshotDropFraction, prev, off.SnapshotCount
+	if drop < snapshotDropFloor {
+		return false, prev, off.SnapshotCount, false
+	}
+	return float64(drop) > float64(prev)*snapshotDropFraction, prev, off.SnapshotCount, false
 }
 
 // emitSnapshotDrop — one event, at a severity inside the hub's exact vocabulary (08 §6.1).
@@ -311,10 +352,15 @@ func (oc *OffsiteChecker) snapshotDropped(customerID string, off *offsiteReport)
 // sentence that asserts neither loss nor recovery is true under EVERY possible answer to the provider
 // question, so it never needs a second rewrite. An alarm rewritten twice in a week is worse than one
 // rewritten once, because the operator learns its words do not mean anything.
-func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport, prev, cur int) {
+func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport, prev, cur int, outsideWindow bool) {
+	why := "more than retention can explain"
+	if outsideWindow {
+		// R-435: on a pinned tier the box's key cannot delete; only a hub-opened window may.
+		why = "outside any clean-up window the hub opened"
+	}
 	message := fmt.Sprintf(
-		"Customer %s: off-site backup count fell from %d to %d snapshot(s) in one report — more than "+
-			"retention can explain. The daily Storage Box snapshots are read-only and still hold the "+
+		"Customer %s: off-site backup count fell from %d to %d snapshot(s) in one report — "+why+
+			". The daily Storage Box snapshots are read-only and still hold the "+
 			"older copy. The route back out of them is not yet established, so treat this as neither "+
 			"confirmed data loss nor confirmed recovery. Get in touch before restoring anything, and "+
 			"check whether a deletion ran on the box.",
@@ -322,6 +368,7 @@ func (oc *OffsiteChecker) emitSnapshotDrop(customerID string, off *offsiteReport
 	details, _ := json.Marshal(map[string]any{
 		"customer_id": customerID, "previous_count": prev, "current_count": cur,
 		"drop": prev - cur, "last_success": off.LastSuccess, "last_status": off.LastStatus,
+		"outside_window": outsideWindow,
 	})
 	oc.logger.Printf("[INFO] Offsite snapshot drop: %s %d -> %d (offsite_snapshots_dropped)", customerID, prev, cur)
 	if _, err := oc.store.SaveEvent(customerID, "offsite_snapshots_dropped", "error", message, string(details), "hub"); err != nil {
@@ -400,6 +447,7 @@ func (oc *OffsiteChecker) Check() {
 			delete(oc.staleStates, c.CustomerID)
 			delete(oc.dropStates, c.CustomerID)
 			delete(oc.lastCounts, c.CustomerID) // no baseline survives a vanished object
+			delete(oc.lastCountAt, c.CustomerID)
 			continue
 		}
 		if oc.store.IsCustomerBlocked(c.CustomerID) {
@@ -407,6 +455,7 @@ func (oc *OffsiteChecker) Check() {
 			delete(oc.staleStates, c.CustomerID)
 			delete(oc.dropStates, c.CustomerID)
 			delete(oc.lastCounts, c.CustomerID)
+			delete(oc.lastCountAt, c.CustomerID)
 			continue
 		}
 
@@ -448,14 +497,15 @@ func (oc *OffsiteChecker) Check() {
 		// alone rather than being overwritten with a number nobody could measure. That is what stops
 		// a `needs_credential` zero from becoming the baseline and making the RECOVERY look like a rise.
 		if oc.countIsTrustworthy(off) {
-			dropped, prev, cur := oc.snapshotDropped(c.CustomerID, off)
+			dropped, prev, cur, byWindow := oc.snapshotDropped(c.CustomerID, off, c.ReceivedAt)
 			if dropped && oc.dropStates[c.CustomerID] != "dropped" {
-				oc.emitSnapshotDrop(c.CustomerID, off, prev, cur)
+				oc.emitSnapshotDrop(c.CustomerID, off, prev, cur, byWindow)
 				oc.dropStates[c.CustomerID] = "dropped"
 			} else if !dropped {
 				oc.dropStates[c.CustomerID] = "ok"
 			}
 			oc.lastCounts[c.CustomerID] = cur
+			oc.lastCountAt[c.CustomerID] = c.ReceivedAt
 		}
 	}
 	for k := range oc.fillStates {
@@ -472,6 +522,7 @@ func (oc *OffsiteChecker) Check() {
 		if !seen[k] {
 			delete(oc.dropStates, k)
 			delete(oc.lastCounts, k)
+			delete(oc.lastCountAt, k)
 		}
 	}
 }

hub/internal/monitor/offsite_r431_test.go

@@ -261,7 +261,7 @@ func TestR431_RealHistoryProducesZeroAlarms(t *testing.T) {
 			}
 			oc.mu.Lock()
 			if oc.countIsTrustworthy(off) {
-				dropped, prev, cur := oc.snapshotDropped(cust, off)
+				dropped, prev, cur, _ := oc.snapshotDropped(cust, off, time.Time{})
 				if dropped && oc.dropStates[cust] != "dropped" {
 					alarms++
 					t.Errorf("%s at %s: FIRED ON REAL HISTORY %d -> %d", cust, p.At, prev, cur)

hub/internal/monitor/r435_pinned_drop_test.go

@@ -0,0 +1,148 @@
+package monitor
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// R-435 (D7, `09` §3 decision 191). On a PINNED tier (the hub holds a confirmed append-only key) the
+// only legitimate fall of the off-site snapshot count is a clean-up window the hub opened, so any fall
+// the windows do not explain alarms — even one snapshot. Non-pinned tiers keep the half-rule.
+//
+// RED-PROOF (2026-10-08): on the pre-R-435 checker TestR435_PinnedUnexplainedFallAlarms fails —
+// 69 → 60 is 9 < 34.5, so no event.
+
+type r435Setup struct {
+	pinned, confirmed bool
+	window            func(st *store.Store) // runs between the baseline and the next report
+	next              string
+}
+
+func r435Run(t *testing.T, cust string, s r435Setup) []string {
+	t.Helper()
+	st := newDiskStore(t)
+	if s.pinned {
+		if err := st.RecordOffsiteKeyInstalled(cust, "fp-1"); err != nil {
+			t.Fatal(err)
+		}
+		if s.confirmed {
+			if ok, err := st.RecordOffsiteKeyConfirmed(cust, "fp-1"); err != nil || !ok {
+				t.Fatalf("confirm: %v %v", ok, err)
+			}
+		}
+	}
+	var msgs []string
+	saveOffsiteReport(t, st, cust, dropJSON(69, true, "", "ok"))
+	oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, sev, msg, _, _ string) {
+		if et == "offsite_snapshots_dropped" {
+			if sev != "error" {
+				t.Errorf("severity %q, want error", sev)
+			}
+			msgs = append(msgs, msg)
+		}
+	}, quietLog())
+	if s.window != nil {
+		s.window(st)
+	}
+	saveOffsiteReport(t, st, cust, s.next)
+	oc.Check()
+	return msgs
+}
+
+func closedWindow(t *testing.T, cust string, before, after int) func(*store.Store) {
+	return func(st *store.Store) {
+		id, err := st.OpenOffsiteWindowRow(cust, time.Now().Add(30*time.Minute), before, 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := st.CloseOffsiteWindowRow(id, after, "ok", "box"); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestR435_PinnedUnexplainedFallAlarms(t *testing.T) {
+	msgs := r435Run(t, "p1", r435Setup{pinned: true, confirmed: true, next: dropJSON(60, true, "", "ok")})
+	if len(msgs) != 1 {
+		t.Fatalf("pinned 69 -> 60 with no window: want exactly 1 offsite_snapshots_dropped, got %d", len(msgs))
+	}
+	if !strings.Contains(msgs[0], "outside any clean-up window") || !strings.Contains(msgs[0], "69") || !strings.Contains(msgs[0], "60") {
+		t.Fatalf("message must name both counts and say „outside any clean-up window\": %s", msgs[0])
+	}
+}
+
+func TestR435_PinnedOneSnapshotAlarms(t *testing.T) {
+	if msgs := r435Run(t, "p2", r435Setup{pinned: true, confirmed: true, next: dropJSON(68, true, "", "ok")}); len(msgs) != 1 {
+		t.Fatalf("pinned 69 -> 68: even one snapshot must alarm, got %d", len(msgs))
+	}
+}
+
+func TestR435_WindowExplainsFall(t *testing.T) {
+	if msgs := r435Run(t, "p3", r435Setup{pinned: true, confirmed: true,
+		window: closedWindow(t, "p3", 69, 60), next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 {
+		t.Fatalf("a closed window that removed 9 explains 69 -> 60; got %d alarm(s): %v", len(msgs), msgs)
+	}
+}
+
+func TestR435_WindowExplainsOnlyPart(t *testing.T) {
+	if msgs := r435Run(t, "p4", r435Setup{pinned: true, confirmed: true,
+		window: closedWindow(t, "p4", 69, 64), next: dropJSON(60, true, "", "ok")}); len(msgs) != 1 {
+		t.Fatalf("a window that removed 5 does not explain 69 -> 60; got %d alarm(s)", len(msgs))
+	}
+}
+
+func TestR435_TimeoutWindowIsUnknownNoAlarm(t *testing.T) {
+	timeout := func(st *store.Store) {
+		id, err := st.OpenOffsiteWindowRow("p5", time.Now().Add(30*time.Minute), 69, 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, _ = st.CloseOffsiteWindowRow(id, -1, "", "timeout")
+	}
+	if msgs := r435Run(t, "p5", r435Setup{pinned: true, confirmed: true, window: timeout, next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 {
+		t.Fatalf("a timeout-closed window cannot say what it removed → no alarm; got %d", len(msgs))
+	}
+}
+
+// Controls: the half-rule still governs a non-pinned tier, and an installed-but-unconfirmed key.
+func TestR435_NotPinnedKeepsHalfRule(t *testing.T) {
+	if msgs := r435Run(t, "n1", r435Setup{next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 {
+		t.Fatalf("NAS tier 69 -> 60 must stay silent (half-rule); got %d", len(msgs))
+	}
+	if msgs := r435Run(t, "n2", r435Setup{pinned: true, next: dropJSON(60, true, "", "ok")}); len(msgs) != 0 {
+		t.Fatalf("an unconfirmed key is not pinned; 69 -> 60 must stay silent; got %d", len(msgs))
+	}
+	if msgs := r435Run(t, "n3", r435Setup{next: dropJSON(4, true, "", "ok")}); len(msgs) != 1 {
+		t.Fatalf("NAS tier 69 -> 4 still alarms by the half-rule; got %d", len(msgs))
+	} else if strings.Contains(msgs[0], "outside any clean-up window") {
+		t.Fatalf("the half-rule message must not claim a window rule: %s", msgs[0])
+	}
+}
+
+// An untrustworthy report on a pinned tier: no alarm, and the baseline does not move.
+func TestR435_UntrustworthyReportNoAlarmBaselineKept(t *testing.T) {
+	st := newDiskStore(t)
+	_ = st.RecordOffsiteKeyInstalled("u1", "fp")
+	_, _ = st.RecordOffsiteKeyConfirmed("u1", "fp")
+	n := 0
+	saveOffsiteReport(t, st, "u1", dropJSON(69, true, "", "ok"))
+	oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) {
+		if et == "offsite_snapshots_dropped" {
+			n++
+		}
+	}, quietLog())
+	saveOffsiteReport(t, st, "u1", dropJSON(0, false, "", "ok")) // stats unknown
+	oc.Check()
+	if n != 0 {
+		t.Fatalf("untrustworthy report must not alarm; got %d", n)
+	}
+	oc.mu.Lock()
+	base := oc.lastCounts["u1"]
+	oc.mu.Unlock()
+	if base != 69 {
+		t.Fatalf("baseline must stay 69 after an untrustworthy report, got %d", base)
+	}
+}

hub/internal/store/offsite_keys.go

@@ -324,3 +324,43 @@ func (s *Store) ForceOffsiteAbandonDueForTest(id int64) error {
 	_, err := s.db.Exec(`UPDATE offsite_abandon_requests SET due_at = datetime('now', '-1 minute') WHERE id = ?`, id)
 	return err
 }
+
+// RemovedByWindowsBetween — R-435 (D7, `09` §3 decision 191). How many snapshots the clean-up windows
+// the hub opened for this customer EXPLAIN between two reports: the sum of count_before − count_after
+// over windows closed in (from, to]. On a pinned tier these windows are the only legitimate way the
+// count can fall, so anything beyond the sum is unexplained.
+//
+// unknown = true when a window in the interval cannot say what it removed: closed by timeout (no box
+// result, count_after −1 or NULL), or still open (opened at or before `to`, not closed). The caller
+// then treats the fall as explained — the safe side for noise, recorded as a limit in `08`. A query
+// error is unknown too.
+func (s *Store) RemovedByWindowsBetween(customerID string, from, to time.Time) (removed int, unknown bool) {
+	const f = "2006-01-02 15:04:05"
+	rows, err := s.db.Query(`
+		SELECT count_before, count_after, closed_at IS NULL FROM offsite_windows
+		WHERE customer_id = ?
+		  AND ((closed_at IS NULL AND opened_at <= ?) OR (closed_at > ? AND closed_at <= ?))`,
+		customerID, to.UTC().Format(f), from.UTC().Format(f), to.UTC().Format(f))
+	if err != nil {
+		return 0, true
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var before, after sql.NullInt64
+		var open bool
+		if err := rows.Scan(&before, &after, &open); err != nil {
+			return 0, true
+		}
+		if open || !before.Valid || !after.Valid || after.Int64 < 0 {
+			unknown = true
+			continue
+		}
+		if d := int(before.Int64 - after.Int64); d > 0 {
+			removed += d
+		}
+	}
+	if rows.Err() != nil {
+		return 0, true
+	}
+	return removed, unknown
+}

hub/internal/store/r435_windows_between_test.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"testing"
+	"time"
+)
+
+// R-435: RemovedByWindowsBetween sums only windows closed inside the interval, and says „unknown"
+// for a window that cannot say what it removed.
+func TestR435_RemovedByWindowsBetween(t *testing.T) {
+	s := newTestStore(t)
+	at := func(ts string) time.Time { v, _ := time.Parse("2006-01-02 15:04:05", ts); return v }
+	ins := func(cust, opened, closed string, before, after any) {
+		t.Helper()
+		var c any
+		if closed != "" {
+			c = closed
+		}
+		if _, err := s.db.Exec(`INSERT INTO offsite_windows (customer_id, opened_at, closes_by, closed_at, count_before, count_after) VALUES (?, ?, ?, ?, ?, ?)`,
+			cust, opened, opened, c, before, after); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ins("a", "2026-10-01 03:00:00", "2026-10-01 03:10:00", 69, 60) // inside → 9
+	ins("a", "2026-09-20 03:00:00", "2026-09-20 03:10:00", 80, 69) // before the interval
+	ins("a", "2026-10-03 03:00:00", "2026-10-03 03:10:00", 60, 50) // after the interval
+	ins("b", "2026-10-01 03:00:00", "2026-10-01 03:10:00", 50, 40) // another customer
+
+	from, to := at("2026-09-30 00:00:00"), at("2026-10-02 00:00:00")
+	if n, unk := s.RemovedByWindowsBetween("a", from, to); n != 9 || unk {
+		t.Fatalf("a: want 9 known, got %d unknown=%v", n, unk)
+	}
+	if n, unk := s.RemovedByWindowsBetween("c", from, to); n != 0 || unk {
+		t.Fatalf("no window: want 0 known, got %d unknown=%v", n, unk)
+	}
+	// a timeout close (count_after −1) inside the interval → unknown
+	ins("d", "2026-10-01 03:00:00", "2026-10-01 03:40:00", 69, -1)
+	if _, unk := s.RemovedByWindowsBetween("d", from, to); !unk {
+		t.Fatal("a timeout-closed window must make the interval unknown")
+	}
+	// a window still open → unknown
+	ins("e", "2026-10-01 03:00:00", "", 69, nil)
+	if _, unk := s.RemovedByWindowsBetween("e", from, to); !unk {
+		t.Fatal("an open window must make the interval unknown")
+	}
+}