From 1fac9c16f166d576b90daba7cedb5628f15bf1be Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 15 Nov 2026 14:50:55 +0100 Subject: [PATCH] hub: the first sight of a large request is the hub's own record; no box text in the ledger (security review) The wait of a large removed-app request was timed from a search of the windows' refusal text, which also held names a box had claimed. A box could plant the marker in a claimed name a week ahead and skip the wait and the mail. - offsite_large_requests (household, key of the app set, first_seen): written only by the hub; at most 50 rows per household; it goes with the household. No text is searched any more. - A claimed name that has not the shape of an app name is written nowhere: the ledger, the answer and the operator's message carry a fixed placeholder. Tests seen red; 6 mutations seen red (redproof-review-s1.txt). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../architecture/05-hub-architecture.md | 1 + .../A/redproof-review-s1.txt | 52 +++++++++++ hub/CHANGELOG.md | 9 ++ hub/internal/offsitekeys/ended.go | 36 ++++++-- hub/internal/offsitekeys/ended_bound_test.go | 12 +-- .../offsitekeys/ended_large_marker_test.go | 83 +++++++++++++++++ hub/internal/offsitekeys/ended_test.go | 2 +- hub/internal/store/customer_delete.go | 5 +- hub/internal/store/offsite_ended.go | 92 +++++++++++++++---- hub/internal/store/orphan_rows.go | 1 + 10 files changed, 261 insertions(+), 32 deletions(-) create mode 100644 documentation/audits/build-release-4-2026-11-15/A/redproof-review-s1.txt create mode 100644 hub/internal/offsitekeys/ended_large_marker_test.go diff --git a/documentation/architecture/05-hub-architecture.md b/documentation/architecture/05-hub-architecture.md index 3b397da5..d8ef7535 100644 --- a/documentation/architecture/05-hub-architecture.md +++ b/documentation/architecture/05-hub-architecture.md @@ -451,6 +451,7 @@ or a household. The ruling: a delete removes them, with one exception — the de | `os_host_settings`, `os_reports`, `os_kernel_notices` | the box | the box is deleted | `store.DeleteHost`, in its one transaction | | the hub's own alarm marks that name the box (`hub_settings`, `os_alarm::…`) | the box | the box is deleted | the same transaction; matched on the whole id between colons, so a box whose id only starts with this id keeps its marks | | `offsite_keys`, `offsite_windows`, `offsite_abandon_requests` | the household | the household is deleted (the purge leg, after the outside legs) | `store.PurgeCustomerResidue`; counted in the delete dialog's residue | +| `offsite_large_requests` (the hub's own first sight of a large removed-app request; `07` §8.2f — built, not released) | the household | the household is deleted; the daily orphan sweep | `store.PurgeCustomerResidue`, `store.PurgeOrphanRows`. At most 50 rows per household | | `offsite_app_presence` (the hub's own record of which apps a box runs; D72, `07` §8.2f — built, not released) | the household | the household is deleted; the daily orphan sweep takes what an earlier delete left | `store.PurgeCustomerResidue`, `store.PurgeOrphanRows`. At most 400 names per household; no age limit on a row (it would move a real app's 30 days) | | `operator_actions` (the operator's own action log) | the household | **one year after the deletion**, with `events` and `notification_log` | `store.PruneDeletedCustomerAudit`. *Decided by CC unattended on 2026-11-14; accepted by the operator on 2026-11-15 (`09` §3 decision 311):* it is an audit record of the same kind as the other two, and the one-year rule (decision 181) is the design's rule for those; the other option, at once, loses „who did what” on the day of the deletion | | **`host_deletions`, `customer_resets`** — the delete journal | both | **never** | the ruling's one exception: an id and timestamps. **An id can be a name, so this is not „no personal data”** — the design to change that is §14.3, and the privacy text waits for it (`09` §3 decision 312) | diff --git a/documentation/audits/build-release-4-2026-11-15/A/redproof-review-s1.txt b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-s1.txt new file mode 100644 index 00000000..161f340f --- /dev/null +++ b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-s1.txt @@ -0,0 +1,52 @@ +SECURITY FINDING 1 on the review-fix-1 commit (875c339c) — the hub found its „first sight of a large request” by searching window text that also held box-supplied names. + +### RED: the new tests before the fix (felhom.eu be272b7c) +=== RUN TestD84_ABoxCannotPlantTheLargeRequestMarker + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "more than half of the store (apps: ghost)" + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "more than half of the store" + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "the request is for 20 of 20 snapshots — more than half of the store (apps: ghost): it waits one week, and the operator is told" + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "large:ghost" + ended_large_marker_test.go:45: the box planted the marker a week ahead and the hub agreed to 10 ids at once — no wait, no warning +--- FAIL: TestD84_ABoxCannotPlantTheLargeRequestMarker (0.05s) +=== RUN TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold + ended_large_marker_test.go:70: 500 different requests left 0 rows (limit 50): +--- FAIL: TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold (0.04s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys 0.103s +FAIL + +### After the fix: mutations +== MUTATION: a claimed name that is not an app name is written into the ledger + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ABoxCannotPlantTheLargeRequestMarker (0.05s) + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "more than half of the store (apps: ghost)" + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "more than half of the store" + ended_large_marker_test.go:37: a claimed name that is not an app name was written into the window's row: "the request is for 20 of 20 snapshots — more than half of the store (apps: ghost): it waits one week, and the operator is to +== MUTATION: the hub does not record its own first sight + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.09s) + ended_bound_test.go:108: a week later: 0 ended ids agreed with a cap of 10 — want exactly the cap's size + --- FAIL: TestD84_TheRestGoesInTheNextWindow (0.04s) + ended_bound_test.go:161: a week later: 0 agreed, want the cap's size 10 + +2 mutations, 0 not a red-proof +== MUTATION: the first sight is read across households + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ABoxCannotPlantTheLargeRequestMarker (0.07s) + ended_large_marker_test.go:59: another household reads this household's first sight +== MUTATION: a first sight older than the memory still ends the wait + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ABoxCannotPlantTheLargeRequestMarker (0.05s) + ended_large_marker_test.go:56: a first sight older than `since` was returned +== MUTATION: no limit on one household's rows + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold (0.60s) + ended_large_marker_test.go:74: 500 different requests left 500 rows (limit 50): + +3 mutations, 0 not a red-proof +== MUTATION: the record does not go with the household + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold (0.26s) + ended_large_marker_test.go:74: 500 different requests left 0 rows (limit 50): + +1 mutations, 0 not a red-proof diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 6133677e..f2224061 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,14 @@ ## Unreleased (after v0.182.0) +- **Security fix on review fix 1 (2026-11-15): the hub's first sight of a large request is its OWN record.** It was + found by searching the windows' refusal text (`instr(ended_refused, …)`), and that text also held names a box had + claimed — a box could plant the marker in a claimed name a week ahead and skip the wait and the operator's mail. + Now the first sight is a row in `offsite_large_requests` (household, key of the app set, first_seen), written only + by the hub (`store.NoteLargeEndedRequest` / `LargeEndedRequestFirstSeen`; at most 50 rows per household; it goes + with the household), and no text is searched. A claimed name that has not the shape of an app name is written + nowhere: the ledger, the answer and the operator's refusal message carry `(not-an-app-name)` instead. Tests + `TestD84_ABoxCannotPlantTheLargeRequestMarker`, `TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold` — + seen red; 6 mutations seen red (`…/A/redproof-review-s1.txt`). - **Review fix 4 (2026-11-15): a box can no longer word or flood the household's receipt.** `offsite_end_scheduled` bypasses the household's switches, and it went through the ordinary customer leg: the subject and body were the box's own sentence, and 50 invented subjects gave 50 mails. Now `Dispatcher.processReceipt`: the details must name diff --git a/hub/internal/offsitekeys/ended.go b/hub/internal/offsitekeys/ended.go index fa8620a1..88280eb7 100644 --- a/hub/internal/offsitekeys/ended.go +++ b/hub/internal/offsitekeys/ended.go @@ -68,7 +68,7 @@ func reservedEndedName(app string) bool { func agreeEnded(now time.Time, presence map[string]store.AppPresence, listing store.OffsiteSnapshotListing, haveListing bool, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) { if len(claims) > maxEndedClaims { for _, c := range claims[maxEndedClaims:] { - refused = append(refused, store.EndedRefusal{App: c.App, Reason: fmt.Sprintf("more than %d apps in one request", maxEndedClaims)}) + refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: fmt.Sprintf("more than %d apps in one request", maxEndedClaims)}) } claims = claims[:maxEndedClaims] } @@ -97,7 +97,9 @@ func agreeEnded(now time.Time, presence map[string]store.AppPresence, listing st } if reason != "" { refusedApp[c.App] = true - refused = append(refused, store.EndedRefusal{App: c.App, Reason: reason}) + // A claimed name that has not the shape of an app name is the box's free text: it is never written + // into the hub's ledger, an operator message or the answer (security review of 2026-11-15). + refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: reason}) continue } agreedApp[c.App] = p.AbsentSince @@ -208,7 +210,7 @@ func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int if err != nil { s.logf("[WARN] offsitekeys: ended apps %s: the hub's own app record could not be read (%v) — nothing agreed", customerID, err) for _, c := range claims { - refused = append(refused, store.EndedRefusal{App: c.App, Reason: "the hub's own app record could not be read"}) + refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: "the hub's own app record could not be read"}) } return nil, nil, refused } @@ -224,10 +226,16 @@ func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int // (a) ONE window agrees to at most the cap's own size (boundEndedToCap); the rest waits for the next window. if total := len(listing.Entries); have && len(ids)*2 > total { // The week belongs to THESE apps: a large request for other apps waits its own week and gets its own mail. - mark := store.LargeEndedRefusalMark + " (apps: " + appNames(byApp) + ")" - first := s.Store.FirstLargeEndedRefusal(customerID, mark, now.Add(-largeEndedMemory)) + // The first sight is the hub's own record (store.offsite_large_requests) — never a search of text a box + // could have written. + key := largeSetKey(byApp) + first := s.Store.LargeEndedRequestFirstSeen(customerID, key, now.Add(-largeEndedMemory)) if first.IsZero() || now.Sub(first) < largeEndedWait { if first.IsZero() { + if nerr := s.Store.NoteLargeEndedRequest(customerID, key, now); nerr != nil { + // Not recorded → the next window sees it as new, tells again and waits again: the safe direction. + s.logf("[WARN] offsitekeys: ended apps %s: the first sight of a large request could not be recorded (%v) — it will wait again", customerID, nerr) + } s.event(customerID, EventEndedLargeAgreement, "warning", fmt.Sprintf("Off-site clean-up window %d: the box of %s asked to remove %d of %d off-site snapshot(s) as copies of removed apps (%s) — more than half of the store. "+ "NOTHING of them is removed in this window. The hub's own records agree to each app, so from %s a window removes them, at most %d per window. "+ @@ -235,8 +243,8 @@ func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int windowID, customerID, len(ids), total, appNames(byApp), now.Add(largeEndedWait).UTC().Format("2006-01-02"), endedCap(maxRemove, total)), map[string]any{"subject": "large:" + appNames(byApp), "window_id": windowID, "asked": len(ids), "listed": total}) } - refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("the request is for %d of %d snapshots — %s: it waits one week, and the operator is told", - len(ids), total, mark)}) + refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("the request is for %d of %d snapshots — more than half of the store: it waits one week, and the operator is told", + len(ids), total)}) ids, byApp = nil, nil } } @@ -338,3 +346,17 @@ func appNames(byApp map[string][]string) string { } return strings.Join(names, ", ") } + +// largeSetKey is the key of an app set in the hub's own record of large requests. +func largeSetKey(byApp map[string][]string) string { return appNames(byApp) } + +// ledgerName is a claimed name as the hub may write it down: itself when it has the shape of an app name, else a +// fixed placeholder. +func ledgerName(app string) string { + if store.ValidPresenceApp(app) { + return app + } + return notAnAppName +} + +const notAnAppName = "(not-an-app-name)" diff --git a/hub/internal/offsitekeys/ended_bound_test.go b/hub/internal/offsitekeys/ended_bound_test.go index 0cef4ebd..1cdf26ff 100644 --- a/hub/internal/offsitekeys/ended_bound_test.go +++ b/hub/internal/offsitekeys/ended_bound_test.go @@ -42,13 +42,11 @@ func ghostFixture(t *testing.T) (*Service, *[]evt, []string) { return s, events, all } -// aWeekLater moves every window of the household eight days into the past (the ledger is stamped by SQLite's clock). -func aWeekLater(t *testing.T, s *Service, ids ...int64) { +// aWeekLater moves the hub's own first sight of the household's large requests eight days into the past. +func aWeekLater(t *testing.T, s *Service, _ ...int64) { t.Helper() - for _, id := range ids { - if err := s.Store.BackdateOffsiteWindowForTest(id, 8*24*time.Hour); err != nil { - t.Fatal(err) - } + if err := s.Store.BackdateLargeEndedRequestsForTest("c1", 8*24*time.Hour); err != nil { + t.Fatal(err) } } @@ -96,7 +94,7 @@ func TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize(t *testi s2, events2, all2 := ghostFixture(t) g1 := openGranted(t, s2, 20, EndedClaim{App: "ghost", SnapshotIDs: all2}) _ = s2.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: g1.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) - if err := s2.Store.BackdateOffsiteWindowForTest(g1.WindowID, 2*24*time.Hour); err != nil { + if err := s2.Store.BackdateLargeEndedRequestsForTest("c1", 2*24*time.Hour); err != nil { t.Fatal(err) } gEarly := openGranted(t, s2, 20, EndedClaim{App: "ghost", SnapshotIDs: all2}) diff --git a/hub/internal/offsitekeys/ended_large_marker_test.go b/hub/internal/offsitekeys/ended_large_marker_test.go new file mode 100644 index 00000000..17dec949 --- /dev/null +++ b/hub/internal/offsitekeys/ended_large_marker_test.go @@ -0,0 +1,83 @@ +package offsitekeys + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// Security review of the review-fix-1 commit (2026-11-15). The hub remembered its first sight of a large request by +// SEARCHING the windows' refusal text — text that also held the box's own claimed names. A box could plant the marker +// in a claimed name a week ahead and so skip the wait, and the operator's mail with it. The first sight now lives in +// the hub's own table, written only by the hub, and a claimed name that is not an app name never reaches the ledger. +// RED-PROOF: documentation/audits/build-release-4-2026-11-15/A/redproof-review-s1.txt. +func TestD84_ABoxCannotPlantTheLargeRequestMarker(t *testing.T) { + s, events, all := ghostFixture(t) + ctx := context.Background() + // A week ahead: a window in which the box claims „apps” whose NAMES are the texts the hub writes for a large + // request — every form of it the box can guess. + planted := []EndedClaim{ + {App: "more than half of the store (apps: ghost)", SnapshotIDs: []string{eid(0)}}, + {App: "more than half of the store", SnapshotIDs: []string{eid(1)}}, + {App: "the request is for 20 of 20 snapshots — more than half of the store (apps: ghost): it waits one week, and the operator is told", SnapshotIDs: []string{eid(2)}}, + {App: "large:ghost", SnapshotIDs: []string{eid(3)}}, + } + g0 := openGranted(t, s, 20, planted...) + if len(g0.EndedAllowed) != 0 { + t.Fatalf("fixture: the planted names were agreed: %v", shortIDs(g0.EndedAllowed)) + } + _ = s.CloseWindowFor(ctx, "c1", WindowResult{WindowID: g0.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) + // the box's words are not in the hub's ledger at all + w, _ := s.Store.GetOffsiteWindow(g0.WindowID) + for _, r := range w.EndedRefused { + if strings.Contains(r.App, "more than half") || strings.Contains(r.App, "large:") || strings.Contains(r.App, " ") { + t.Errorf("a claimed name that is not an app name was written into the window's row: %q", r.App) + } + } + aWeekLater(t, s, g0.WindowID) + *events = nil + // Now the real request for the whole store: it must wait and the operator must be mailed, as if nothing was planted. + g := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: all}) + if len(g.EndedAllowed) != 0 { + t.Fatalf("the box planted the marker a week ahead and the hub agreed to %d ids at once — no wait, no warning", len(g.EndedAllowed)) + } + if n := len(ofType(*events, EventEndedLargeAgreement)); n != 1 { + t.Fatalf("the operator was told %d time(s) about the large request, want once", n) + } + // and the hub's own first sight is what later ends the wait + if at := s.Store.LargeEndedRequestFirstSeen("c1", largeSetKey(map[string][]string{"ghost": nil}), time.Now().Add(-time.Hour)); at.IsZero() { + t.Fatal("the hub did not record its own first sight of the large request") + } + // a first sight older than the caller's memory is no first sight: the request waits again + if at := s.Store.LargeEndedRequestFirstSeen("c1", largeSetKey(map[string][]string{"ghost": nil}), time.Now().Add(time.Hour)); !at.IsZero() { + t.Fatal("a first sight older than `since` was returned") + } + if at := s.Store.LargeEndedRequestFirstSeen("other-household", largeSetKey(map[string][]string{"ghost": nil}), time.Now().Add(-time.Hour)); !at.IsZero() { + t.Fatal("another household reads this household's first sight") + } +} + +// The first sight goes with the household, and one household cannot fill the table. +func TestD84_TheLargeRequestRecordIsBoundedAndGoesWithTheHousehold(t *testing.T) { + s, _, _ := ghostFixture(t) + now := time.Now().UTC() + for i := 0; i < 500; i++ { + if err := s.Store.NoteLargeEndedRequest("c1", strings.Repeat("k", 10)+string(rune('a'+i%26))+strings.Repeat("x", i%7)+strings.Repeat("y", i/26), now); err != nil { + t.Fatal(err) + } + } + r, err := s.Store.CustomerResidue("c1") + if err != nil || r.OffsiteLargeRequests == 0 || r.OffsiteLargeRequests > 50 { + t.Fatalf("500 different requests left %d rows (limit 50): %v", r.OffsiteLargeRequests, err) + } + if err := s.Store.PurgeCustomerResidue("c1"); err != nil { + t.Fatal(err) + } + if r2, _ := s.Store.CustomerResidue("c1"); r2.OffsiteLargeRequests != 0 { + t.Fatalf("the record outlived the household: %d", r2.OffsiteLargeRequests) + } + var _ = store.EndedRefusal{} +} diff --git a/hub/internal/offsitekeys/ended_test.go b/hub/internal/offsitekeys/ended_test.go index 753d988f..fee33bca 100644 --- a/hub/internal/offsitekeys/ended_test.go +++ b/hub/internal/offsitekeys/ended_test.go @@ -70,7 +70,7 @@ func TestD84_AgreeEnded_TheHubsOwnRecordsDecide(t *testing.T) { {name: "the shares pseudo-app and the marker tag are never eligible, whatever the record says", presence: map[string]store.AppPresence{"_shares": {AbsentSince: days(90)}, "felhom-offbox": {AbsentSince: days(90)}}, listing: true, claims: []EndedClaim{boxSaysLongAgo("_shares", eid(1)), boxSaysLongAgo("felhom-offbox", eid(2))}, - refused: map[string]string{"_shares": "not an app", "felhom-offbox": "not an app"}}, + refused: map[string]string{notAnAppName: "not an app", "felhom-offbox": "not an app"}}, {name: "the box's own infrastructure stacks are never eligible", presence: map[string]store.AppPresence{"traefik": {AbsentSince: days(90)}, "Samba": {AbsentSince: days(90)}}, listing: true, claims: []EndedClaim{boxSaysLongAgo("traefik", eid(1)), boxSaysLongAgo("Samba", eid(2))}, diff --git a/hub/internal/store/customer_delete.go b/hub/internal/store/customer_delete.go index 114407e9..065c7b89 100644 --- a/hub/internal/store/customer_delete.go +++ b/hub/internal/store/customer_delete.go @@ -53,6 +53,8 @@ type CustomerResidue struct { OffsiteAbandonRequests int // D72: the hub's own record of which apps the household's box runs (offsite_ended.go). OffsiteAppPresence int + // The hub's own first sight of a large ended-app request (offsite_ended.go). + OffsiteLargeRequests int } // Total is the single number the "is there anything left at all?" decision hangs on. @@ -62,7 +64,7 @@ func (r *CustomerResidue) Total() int { } return r.Reports + r.AppTelemetry + r.AppLogTails + r.LogTailRequests + r.NotificationPrefs + r.SelfBindTokens + r.ApplianceRegistrations + r.AppLogIssueRefs + - r.OffsiteKeys + r.OffsiteWindows + r.OffsiteAbandonRequests + r.OffsiteAppPresence + r.OffsiteKeys + r.OffsiteWindows + r.OffsiteAbandonRequests + r.OffsiteAppPresence + r.OffsiteLargeRequests } // residueQueries is the ONE list both the counter and the purge walk, so a table can never be @@ -82,6 +84,7 @@ var residueQueries = []struct { {"offsite_windows", func(r *CustomerResidue) *int { return &r.OffsiteWindows }}, {"offsite_abandon_requests", func(r *CustomerResidue) *int { return &r.OffsiteAbandonRequests }}, {"offsite_app_presence", func(r *CustomerResidue) *int { return &r.OffsiteAppPresence }}, + {"offsite_large_requests", func(r *CustomerResidue) *int { return &r.OffsiteLargeRequests }}, } // CustomerResidue counts the customer's report-derived + binding residue. Read-only. diff --git a/hub/internal/store/offsite_ended.go b/hub/internal/store/offsite_ended.go index d441b67c..eb7d1b94 100644 --- a/hub/internal/store/offsite_ended.go +++ b/hub/internal/store/offsite_ended.go @@ -38,6 +38,15 @@ func (s *Store) migrateOffsiteEnded() error { s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_removed TEXT") // 1 on a row that exists ONLY because the box named the app in an end record (the hub never saw it installed). s.db.Exec("ALTER TABLE offsite_app_presence ADD COLUMN claim_born INTEGER NOT NULL DEFAULT 0") + if _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS offsite_large_requests ( + customer_id TEXT NOT NULL, + set_key TEXT NOT NULL, + first_seen TEXT NOT NULL, + PRIMARY KEY (customer_id, set_key) + );`); err != nil { + return fmt.Errorf("offsite_large_requests: %w", err) + } s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_apps TEXT") // agreed app → its agreed ids (the per-app record) return nil } @@ -382,28 +391,79 @@ func jsonStrings(v sql.NullString) []string { return out } -// LargeEndedRefusalMark is the text a window's refusal carries when the request was for more than half of the store -// (offsitekeys). FirstLargeEndedRefusal finds it on the window rows — the ledger is the memory, so it goes with the -// household and needs no second store. -const LargeEndedRefusalMark = "more than half of the store" +// THE HUB'S OWN FIRST SIGHT OF A LARGE REQUEST (security review of 2026-11-15). A request to remove more than half of +// a store for removed apps waits a week after the hub first saw it (offsitekeys). That first sight is kept HERE, in a +// table only the hub writes: (household, a key of the app set, first_seen). It was first found by searching the +// windows' refusal text — text that also held names a box had claimed, so a box could plant its own „first sight”. +// No box-supplied text is searched any more. -// FirstLargeEndedRefusal returns when the household's EARLIEST window since `since` refused a request for more than -// half of the store FOR THE SAME APPS (`needle` is the refusal's own text for them; zero time: none). A store that -// cannot answer reads as none — and then the request waits again. -func (s *Store) FirstLargeEndedRefusal(customerID, needle string, since time.Time) time.Time { +// maxLargeRequestRows bounds one household's rows: a box can ask for many app sets. +const maxLargeRequestRows = 50 + +// LargeEndedRequestFirstSeen returns when the hub first saw this household ask for this app set in a large request, +// if that was after `since` (zero time: never, or too long ago — the request then waits again). +func (s *Store) LargeEndedRequestFirstSeen(customerID, key string, since time.Time) time.Time { var v sql.NullString - err := s.db.QueryRow(`SELECT MIN(opened_at) FROM offsite_windows WHERE customer_id = ? AND opened_at >= ? AND instr(COALESCE(ended_refused, ''), ?) > 0`, - customerID, since.UTC().Format("2006-01-02 15:04:05"), needle).Scan(&v) - if err != nil || !v.Valid || v.String == "" { + if err := s.db.QueryRow(`SELECT first_seen FROM offsite_large_requests WHERE customer_id = ? AND set_key = ?`, customerID, key).Scan(&v); err != nil || !v.Valid { return time.Time{} } - return parseSQLiteTime(v.String) + t, err := time.Parse(time.RFC3339, v.String) + if err != nil || t.Before(since) { + return time.Time{} + } + return t } -// BackdateOffsiteWindowForTest moves a window's opened_at into the past. TEST-ONLY. -func (s *Store) BackdateOffsiteWindowForTest(id int64, by time.Duration) error { - _, err := s.db.Exec(`UPDATE offsite_windows SET opened_at = ? WHERE id = ?`, time.Now().UTC().Add(-by).Format("2006-01-02 15:04:05"), id) - return err +// NoteLargeEndedRequest records the hub's first sight (now) of this app set — replacing a sight that is older than +// the caller's memory is the caller's choice: it calls this only when LargeEndedRequestFirstSeen answered „never”. +// A household holds at most maxLargeRequestRows rows; the oldest go first. +func (s *Store) NoteLargeEndedRequest(customerID, key string, now time.Time) error { + if customerID == "" || key == "" { + return fmt.Errorf("NoteLargeEndedRequest: empty household or key") + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`INSERT INTO offsite_large_requests (customer_id, set_key, first_seen) VALUES (?, ?, ?) + ON CONFLICT(customer_id, set_key) DO UPDATE SET first_seen = excluded.first_seen`, customerID, key, now.UTC().Format(time.RFC3339)); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM offsite_large_requests WHERE customer_id = ? AND set_key NOT IN ( + SELECT set_key FROM offsite_large_requests WHERE customer_id = ? ORDER BY first_seen DESC, set_key LIMIT ?)`, customerID, customerID, maxLargeRequestRows); err != nil { + return err + } + return tx.Commit() +} + +// BackdateLargeEndedRequestsForTest moves a household's first sights into the past. TEST-ONLY. +func (s *Store) BackdateLargeEndedRequestsForTest(customerID string, by time.Duration) error { + rows, err := s.db.Query(`SELECT set_key, first_seen FROM offsite_large_requests WHERE customer_id = ?`, customerID) + if err != nil { + return err + } + type row struct{ key, at string } + var all []row + for rows.Next() { + var r row + if err := rows.Scan(&r.key, &r.at); err != nil { + rows.Close() + return err + } + all = append(all, r) + } + rows.Close() + for _, r := range all { + t, perr := time.Parse(time.RFC3339, r.at) + if perr != nil { + continue + } + if _, err := s.db.Exec(`UPDATE offsite_large_requests SET first_seen = ? WHERE customer_id = ? AND set_key = ?`, t.Add(-by).Format(time.RFC3339), customerID, r.key); err != nil { + return err + } + } + return nil } // CountCustomerSentSince counts the mails of one type the hub's own log says were SENT to a household since `since` diff --git a/hub/internal/store/orphan_rows.go b/hub/internal/store/orphan_rows.go index 419144f5..535e841d 100644 --- a/hub/internal/store/orphan_rows.go +++ b/hub/internal/store/orphan_rows.go @@ -28,6 +28,7 @@ var orphanTables = []struct{ table, where string }{ {"offsite_windows", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, {"offsite_abandon_requests", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, {"offsite_app_presence", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, + {"offsite_large_requests", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, } // PurgeOrphanRows removes, in ONE transaction, the rows of boxes and households that no longer exist from the six