diff --git a/documentation/audits/build-release-4-2026-11-15/A/redproof-hub-app-record-bounds.txt b/documentation/audits/build-release-4-2026-11-15/A/redproof-hub-app-record-bounds.txt new file mode 100644 index 00000000..5a64a780 --- /dev/null +++ b/documentation/audits/build-release-4-2026-11-15/A/redproof-hub-app-record-bounds.txt @@ -0,0 +1,67 @@ +== MUTATION: no limit per household + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_ABoxCannotFillIt (0.06s) + d72_app_record_bounds_test.go:56: report 0: the household holds 600 rows, limit 400 — a box can grow the table without end + --- FAIL: TestD72_AppRecord_RoomIsMadeFromUnclaimedAbsentRowsOnly (0.05s) + d72_app_record_bounds_test.go:119: no room was made: app-000 (absent, not claimed) is still there +== MUTATION: room is made from CLAIMED rows too (a real ended app loses its date) + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_ABoxCannotFillIt (0.68s) + d72_app_record_bounds_test.go:64: the real ended app lost its row or its date in the flood (want absent since 2026-06-02 00:00:00 +0000 UTC): {Present:false AbsentSince:0001-01-01 00:00:00 +0000 UTC} ok=false + --- FAIL: TestD72_AppRecord_RoomIsMadeFromUnclaimedAbsentRowsOnly (0.05s) + d72_app_record_bounds_test.go:119: no room was made: app-000 (absent, not claimed) is still there +== MUTATION: a new name above the limit is stored anyway + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_ABoxCannotFillIt (0.05s) + d72_app_record_bounds_test.go:56: report 0: the household holds 600 rows, limit 400 — a box can grow the table without end + --- FAIL: TestD72_AppRecord_RoomIsMadeFromUnclaimedAbsentRowsOnly (0.05s) + d72_app_record_bounds_test.go:130: rows = 402, want the limit 400 +== MUTATION: the limit is said in the log at every report + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_ABoxCannotFillIt (0.63s) + d72_app_record_bounds_test.go:60: the limit was said 40 time(s) in the log, want once for 40 reports: +== MUTATION: any name from the app list is stored + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_OnlyAppNamesAreStored (0.05s) + d72_app_record_bounds_test.go:170: "_shares" was stored + d72_app_record_bounds_test.go:170: "felhom-offbox" was stored + d72_app_record_bounds_test.go:170: "traefik" was stored +== MUTATION: any name from an end record is stored + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_OnlyAppNamesAreStored (0.04s) + d72_app_record_bounds_test.go:170: "_shares" was stored + d72_app_record_bounds_test.go:170: "felhom-offbox" was stored + d72_app_record_bounds_test.go:170: "traefik" was stored +== MUTATION: no length limit on a name + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_OnlyAppNamesAreStored (0.05s) + d72_app_record_bounds_test.go:170: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" was stored + d72_app_record_bounds_test.go:179: rows = 7, want 6: map[Immich:{false 2026-11-15 00:00:00 +0000 UTC} a.b:{false 2026-11-15 00:00:00 +0000 UTC} aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:{false 2026-11-15 00: +== MUTATION: a reserved name in another case is stored + *** NOT A RED-PROOF *** (mutant rc=1 built=False real rc=0) + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/store [build failed] +== MUTATION: a name may start with a dot, a dash or an underscore (_shares) + *** NOT A RED-PROOF *** (mutant rc=1 built=False real rc=0) + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/store [build failed] + internal/store/offsite_ended.go:79:6: declared and not used: i +== MUTATION: the reserved list does not hold the infrastructure stacks (hub agreement) + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AgreeEnded_TheHubsOwnRecordsDecide (0.00s) + ended_test.go:114: refusal for "traefik" = "", want it to say "not an app" (all: [{App:Samba Reason:not an app a household removes} {App: Reason:1 snapshot id(s) refused: listed under an app the hub refused} {App: Reason:1 snapsho + +10 mutations, 2 not a red-proof + +### the two mutations done again +== MUTATION: REDO (the first mutant did not compile: `strings` unused): a reserved name in another case is stored + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_OnlyAppNamesAreStored (0.04s) + d72_app_record_bounds_test.go:170: "Samba" was stored + d72_app_record_bounds_test.go:179: rows = 7, want 6: map[Immich:{false 2026-11-15 00:00:00 +0000 UTC} Samba:{true 0001-01-01 00:00:00 +0000 UTC} a.b:{false 2026-11-15 00:00:00 +0000 UTC} aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +== MUTATION: REDO (the first mutant did not compile: `i` unused): a name may start with a dot, a dash or an underscore (_shares) + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AppRecord_OnlyAppNamesAreStored (0.04s) + d72_app_record_bounds_test.go:170: "_shares" was stored + d72_app_record_bounds_test.go:170: "." was stored + d72_app_record_bounds_test.go:170: ".." was stored + +2 mutations, 0 not a red-proof diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 28a89e0f..734bfc0f 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -23,6 +23,13 @@ (`notify.householdReceiptEvents`): mailed to the household's own notification address once per press, whatever its group switches say, in its language, with no raw details; the operator is not mailed. A household with no address gets no mail. `mail.event.offsite_end_scheduled` in both bundles. + **Bounds on the app record, after the commit's own security review (2026-11-15):** a household holds at most 400 + names (`maxPresenceRows`); a new name above that takes the place of the oldest absent name that the same report + does not claim, or is not stored — said once in the hub's log, not per report; a name is stored only when it is + shaped like an app name (64 characters, letters, digits, `. _ -`) and is not reserved (`_shares`, the marker, the + box's infrastructure stacks). **No age limit on a row, on purpose:** an aged-out row would start a real ended + app's 30 days again. Making room can never make an app eligible earlier. Tests `TestD72_AppRecord_*`; 10 more + mutations seen red, 2 after a redo (`redproof-hub-app-record-bounds.txt`). **Not built here:** the bytes in the per-app record (the hub cannot see sizes). **A limit, said plainly:** the hub cannot read a snapshot's tags, so a box that is taken over and has one honestly ended app can name other files older than that app's end — bounded to files older than 30 days, not closed. diff --git a/hub/internal/offsitekeys/ended.go b/hub/internal/offsitekeys/ended.go index fbc3d03d..5787b235 100644 --- a/hub/internal/offsitekeys/ended.go +++ b/hub/internal/offsitekeys/ended.go @@ -3,7 +3,6 @@ package offsitekeys import ( "fmt" "sort" - "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -48,10 +47,11 @@ type EndedClaim struct { SnapshotIDs []string `json:"snapshot_ids"` } -// reservedEndedName: the shares pseudo-app, the tier's marker tag and anything shaped like them are never apps a -// household removes (`07` §8.2f, the edge-case table). +// reservedEndedName: the shares pseudo-app, the tier's marker tag, the box's own infrastructure stacks and anything +// that has not the shape of an app name are never apps a household removes (`07` §8.2f, the edge-case table). The +// same two checks decide what the hub's app record stores at all (store.NoteAppList). func reservedEndedName(app string) bool { - return app == "" || strings.HasPrefix(app, "_") || app == "felhom-offbox" || strings.ContainsAny(app, "/\\ \t\n") + return !store.ValidPresenceApp(app) || store.ReservedPresenceApp(app) } // agreeEnded is the PURE decision. presence is the hub's own record; listing the hub's own stored listing (ok=false diff --git a/hub/internal/offsitekeys/ended_test.go b/hub/internal/offsitekeys/ended_test.go index f4353cb5..753d988f 100644 --- a/hub/internal/offsitekeys/ended_test.go +++ b/hub/internal/offsitekeys/ended_test.go @@ -71,6 +71,10 @@ func TestD84_AgreeEnded_TheHubsOwnRecordsDecide(t *testing.T) { presence: map[string]store.AppPresence{"_shares": {AbsentSince: days(90)}, "felhom-offbox": {AbsentSince: days(90)}}, listing: true, claims: []EndedClaim{boxSaysLongAgo("_shares", eid(1)), boxSaysLongAgo("felhom-offbox", eid(2))}, refused: map[string]string{"_shares": "not an app", "felhom-offbox": "not an app"}}, + {name: "the box's own infrastructure stacks are never eligible", + presence: map[string]store.AppPresence{"traefik": {AbsentSince: days(90)}, "Samba": {AbsentSince: days(90)}}, listing: true, + claims: []EndedClaim{boxSaysLongAgo("traefik", eid(1)), boxSaysLongAgo("Samba", eid(2))}, + refused: map[string]string{"traefik": "not an app", "Samba": "not an app"}}, {name: "a snapshot with two apps' tags stays until BOTH have ended", presence: map[string]store.AppPresence{"vikunja": {AbsentSince: days(40)}, "immich": {Present: true}}, listing: true, claims: []EndedClaim{boxSaysLongAgo("vikunja", eid(1), eid(2)), boxSaysLongAgo("immich", eid(2))}, diff --git a/hub/internal/store/d72_app_record_bounds_test.go b/hub/internal/store/d72_app_record_bounds_test.go new file mode 100644 index 00000000..9178113f --- /dev/null +++ b/hub/internal/store/d72_app_record_bounds_test.go @@ -0,0 +1,181 @@ +package store + +import ( + "bytes" + "fmt" + "log" + "path/filepath" + "strings" + "testing" + "time" +) + +// Security review of 2026-11-15 on the hub's own app record (D72, offsite_ended.go): what a box sends is bounded — +// per household, per name — and no bound may move a real ended app's 30 days. +// RED-PROOFS: documentation/audits/build-release-4-2026-11-15/A/redproof-hub-app-record-bounds.txt. + +func presenceRows(t *testing.T, s *Store, cust string) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM offsite_app_presence WHERE customer_id = ?`, cust).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// A box that sends NEW invented names in many reports — as apps and as end records — cannot grow the table past the +// household's limit, is named in the log ONCE, and does not push out the household's real ended app. +func TestD72_AppRecord_ABoxCannotFillIt(t *testing.T) { + var logBuf bytes.Buffer + s, err := New(filepath.Join(t.TempDir(), "test.db"), log.New(&logBuf, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + t0 := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + // the real life of the box: vikunja was installed, then removed with „delete backup data” + if err := s.NoteAppList("c1", []string{"vikunja", "immich"}, nil, t0); err != nil { + t.Fatal(err) + } + left := t0.Add(24 * time.Hour) + if err := s.NoteAppList("c1", []string{"immich"}, []string{"vikunja"}, left); err != nil { + t.Fatal(err) + } + logBuf.Reset() + for r := 0; r < 40; r++ { + var apps, claims []string + for i := 0; i < 300; i++ { + apps = append(apps, fmt.Sprintf("fake-app-%d-%d", r, i)) + claims = append(claims, fmt.Sprintf("fake-end-%d-%d", r, i)) + } + claims = append([]string{"vikunja"}, claims...) // the real record is still pending, as the box reports it + if err := s.NoteAppList("c1", append([]string{"immich"}, apps...), claims, left.Add(time.Duration(r+1)*time.Hour)); err != nil { + t.Fatal(err) + } + if n := presenceRows(t, s, "c1"); n > maxPresenceRows { + t.Fatalf("report %d: the household holds %d rows, limit %d — a box can grow the table without end", r, n, maxPresenceRows) + } + } + if n := strings.Count(logBuf.String(), "is at its limit"); n != 1 { + t.Fatalf("the limit was said %d time(s) in the log, want once for 40 reports:\n%s", n, logBuf.String()) + } + m, _ := s.AppPresences("c1") + if p, ok := m["vikunja"]; !ok || p.Present || !p.AbsentSince.Equal(left) { + t.Fatalf("the real ended app lost its row or its date in the flood (want absent since %s): %+v ok=%v", left, p, ok) + } + if !m["immich"].Present { + t.Fatalf("the installed app lost its row in the flood: %+v", m["immich"]) + } + // another household is not touched by this one's limit + if err := s.NoteAppList("c2", []string{"kimai"}, nil, t0); err != nil || presenceRows(t, s, "c2") != 1 { + t.Fatalf("c2: %v rows=%d", err, presenceRows(t, s, "c2")) + } +} + +// Making room never makes an app eligible EARLIER: the row that goes is an absent one the report does not claim, the +// oldest first; a claimed row and a present row stay; and an app that lost its row starts a NEW clock at its next +// claim — it never gets its old date back, and never a date the box names. +func TestD72_AppRecord_RoomIsMadeFromUnclaimedAbsentRowsOnly(t *testing.T) { + s := newTestStore(t) + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // The household at its limit: 300 installed apps (one report's most) and 100 removed apps with a pending end + // record each. `olds` are claimed in every report below. + var all, olds []string + for i := 0; i < maxPresenceApps; i++ { + all = append(all, fmt.Sprintf("app-%03d", i)) + } + for i := 0; i < maxPresenceRows-maxPresenceApps; i++ { + olds = append(olds, fmt.Sprintf("old-%03d", i)) + } + claims := func(more ...string) []string { return append(append([]string{}, olds...), more...) } + if err := s.NoteAppList("c1", all, olds, t0); err != nil { + t.Fatal(err) + } + if n := presenceRows(t, s, "c1"); n != maxPresenceRows { + t.Fatalf("fixture: %d rows, want %d", n, maxPresenceRows) + } + // app-000 leaves first, app-001 a day later; app-001 has a pending end record, app-000 has none + if err := s.NoteAppList("c1", all[1:], claims(), t0.Add(24*time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.NoteAppList("c1", all[2:], claims("app-001"), t0.Add(48*time.Hour)); err != nil { + t.Fatal(err) + } + // NO AGE LIMIT: a year later, with no new name, both absent rows are still there with their dates. + if err := s.NoteAppList("c1", all[2:], claims("app-001"), t0.Add(400*24*time.Hour)); err != nil { + t.Fatal(err) + } + m, _ := s.AppPresences("c1") + if !m["app-000"].AbsentSince.Equal(t0.Add(24*time.Hour)) || !m["app-001"].AbsentSince.Equal(t0.Add(48*time.Hour)) { + t.Fatalf("an absent row aged out or moved: %+v %+v", m["app-000"], m["app-001"]) + } + // one new app at the limit: the unclaimed absent row (app-000) goes; the claimed ones stay. + later := t0.Add(401 * 24 * time.Hour) + if err := s.NoteAppList("c1", append(append([]string{}, all[2:]...), "newapp"), claims("app-001"), later); err != nil { + t.Fatal(err) + } + m, _ = s.AppPresences("c1") + if _, ok := m["app-000"]; ok { + t.Fatalf("no room was made: app-000 (absent, not claimed) is still there") + } + if !m["app-001"].AbsentSince.Equal(t0.Add(48*time.Hour)) || !m["old-000"].AbsentSince.Equal(t0) || !m["newapp"].Present || presenceRows(t, s, "c1") != maxPresenceRows { + t.Fatalf("after making room: app-001=%+v old-000=%+v newapp=%+v rows=%d", m["app-001"], m["old-000"], m["newapp"], presenceRows(t, s, "c1")) + } + // Only present and claimed rows are left. A further new name is NOT stored, and no claimed row is pushed out. + if err := s.NoteAppList("c1", append(append([]string{}, all[2:]...), "newapp", "another"), claims("app-001", "app-000"), later.Add(time.Hour)); err != nil { + t.Fatal(err) + } + m, _ = s.AppPresences("c1") + if n := presenceRows(t, s, "c1"); n != maxPresenceRows { + t.Fatalf("rows = %d, want the limit %d", n, maxPresenceRows) + } + if _, ok := m["app-000"]; ok { + t.Fatalf("a claim found room that was not there: %+v", m["app-000"]) + } + if !m["app-001"].AbsentSince.Equal(t0.Add(48*time.Hour)) || !m["old-099"].AbsentSince.Equal(t0) { + t.Fatalf("a claimed row was pushed out or moved: %+v %+v", m["app-001"], m["old-099"]) + } + // The evicted app is claimed when there IS room again (an installed app left and was not claimed): its clock + // starts at THAT claim — never at its old date, never at a date the box names. + claimAt := later.Add(24 * time.Hour) + if err := s.NoteAppList("c1", append(append([]string{}, all[3:]...), "newapp"), claims("app-001"), claimAt.Add(-time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.NoteAppList("c1", append(append([]string{}, all[3:]...), "newapp"), claims("app-001", "app-000"), claimAt); err != nil { + t.Fatal(err) + } + m, _ = s.AppPresences("c1") + if p, ok := m["app-000"]; !ok || !p.AbsentSince.Equal(claimAt) { + t.Fatalf("an app that lost its row must start a new clock at its claim (%s): %+v ok=%v", claimAt, p, ok) + } + if _, ok := m["app-002"]; ok { + t.Fatalf("app-002 (absent, not claimed) should have made the room: %+v", m["app-002"]) + } +} + +// A name is stored only when it is shaped like an app name and is not reserved — from the app list and from an end +// record alike. +func TestD72_AppRecord_OnlyAppNamesAreStored(t *testing.T) { + s := newTestStore(t) + bad := []string{"_shares", "felhom-offbox", "traefik", "cloudflared", "felhom-controller", "filebrowser", "samba", "Samba", + "", ".", "..", "a/b", "a b", "a\nb", "-dash", ".hidden", "ékezet", strings.Repeat("a", 65), "x'; DROP TABLE reports;--"} + good := []string{"vikunja", "paperless-ngx", "app_1", "Immich", "a.b", strings.Repeat("a", 64)} + now := time.Date(2026, 11, 15, 0, 0, 0, 0, time.UTC) + if err := s.NoteAppList("c1", append(append([]string{}, bad...), good[:3]...), append(append([]string{}, bad...), good[3:]...), now); err != nil { + t.Fatal(err) + } + m, _ := s.AppPresences("c1") + for _, b := range bad { + if _, ok := m[b]; ok { + t.Errorf("%q was stored", b) + } + } + for _, g := range good { + if _, ok := m[g]; !ok { + t.Errorf("%q is an app name and was not stored", g) + } + } + if len(m) != len(good) { + t.Errorf("rows = %d, want %d: %v", len(m), len(good), m) + } +} diff --git a/hub/internal/store/offsite_ended.go b/hub/internal/store/offsite_ended.go index 26d94f4b..2b44df00 100644 --- a/hub/internal/store/offsite_ended.go +++ b/hub/internal/store/offsite_ended.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "sort" + "strings" + "sync" "time" ) @@ -46,8 +48,55 @@ type AppPresence struct { AbsentSince time.Time } -// maxPresenceApps bounds what one report may add: a box's lists are the box's own word. -const maxPresenceApps = 300 +// A box's lists are the box's own word, so the record is bounded three ways (security review of 2026-11-15): +// +// 1. ONE REPORT may name at most maxPresenceApps apps in each list. +// 2. A NAME is stored only when it has the shape of an app name (ValidPresenceApp) and is not reserved +// (ReservedPresenceApp: the shares pseudo-app, the tier's marker, the box's own infrastructure stacks). +// 3. ONE HOUSEHOLD holds at most maxPresenceRows rows. A new name above that first takes the place of the oldest +// ABSENT row that the same report does not name in a pending end record; when there is no such row the new name +// is not stored, and the hub says so in its log once per household (not per report). +// +// THERE IS NO AGE LIMIT ON A ROW, on purpose. An absent row is what makes a real ended app eligible exactly 30 days +// after the hub saw it leave; a row that aged out would start that clock again at the next claim — LATER than the +// household was told. So a row leaves only with the household (PurgeCustomerResidue, PurgeOrphanRows), by an install +// of the app (it becomes present), or by rule 3. Rule 3 can never make an app eligible EARLIER: an evicted app has no +// row, a claim for it starts a new clock at that moment, and 30 days are counted from there. It can make one LATER — +// only for a household that has passed maxPresenceRows names AND whose report did not carry the claim (its off-site +// tier was off). A row is ~60 bytes; 400 rows per household is the whole cost. +const ( + maxPresenceApps = 300 + maxPresenceRows = 400 + maxPresenceName = 64 +) + +// ValidPresenceApp: the shape of an app (stack) name — 1..64 of letters, digits, `.`, `_`, `-`, starting with a +// letter or a digit. Anything else a box sends is not stored and can never be agreed. +func ValidPresenceApp(app string) bool { + if app == "" || len(app) > maxPresenceName { + return false + } + for i, r := range app { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case i > 0 && (r == '.' || r == '_' || r == '-'): + default: + return false + } + } + return true +} + +// reservedPresenceApps are names that are never an app a household removes: the tier's marker tag and the box's own +// infrastructure stacks (the controller's protected list: `controller.yaml.default` plus its always-protected +// `samba`). The shares pseudo-app `_shares` is refused by its shape already (ValidPresenceApp). +var reservedPresenceApps = map[string]bool{ + "felhom-offbox": true, "traefik": true, "cloudflared": true, "felhom-controller": true, "filebrowser": true, + "samba": true, "felhom-samba": true, +} + +// ReservedPresenceApp reports a reserved name (case does not matter, as on the box). +func ReservedPresenceApp(app string) bool { return reservedPresenceApps[strings.ToLower(app)] } // NoteAppList updates the record from ONE report: `deployed` is the report's app list, `claimed` the apps the box // names in pending off-site end records. now is the HUB's clock. @@ -75,17 +124,23 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t } stamp := now.UTC().Format(time.RFC3339) in := make(map[string]bool, len(deployed)) + isClaimed := make(map[string]bool, len(claimed)) + for _, app := range claimed { + isClaimed[app] = true + } type change struct { app string absent sql.NullString } - var changes []change + var changes, added []change // changes touch a row that exists; added need a NEW row (rule 3) for _, app := range deployed { - if app == "" || in[app] { + if !ValidPresenceApp(app) || ReservedPresenceApp(app) || in[app] { continue } in[app] = true - if p, ok := cur[app]; !ok || !p.Present { + if p, ok := cur[app]; !ok { + added = append(added, change{app: app}) + } else if !p.Present { changes = append(changes, change{app: app}) } } @@ -96,15 +151,45 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t } seen := map[string]bool{} for _, app := range claimed { - if app == "" || in[app] || seen[app] { + if !ValidPresenceApp(app) || ReservedPresenceApp(app) || in[app] || seen[app] { continue } seen[app] = true if _, ok := cur[app]; !ok { - changes = append(changes, change{app: app, absent: sql.NullString{String: stamp, Valid: true}}) + added = append(added, change{app: app, absent: sql.NullString{String: stamp, Valid: true}}) } } - if len(changes) == 0 { + // Rule 3: the household's row limit. Room is made only from absent rows this report does not claim, oldest first. + var evict []string + refusedNew := 0 + if over := len(cur) + len(added) - maxPresenceRows; over > 0 { + type old struct { + app string + since time.Time + } + var free []old + for app, p := range cur { + if !p.Present && !in[app] && !isClaimed[app] { + free = append(free, old{app, p.AbsentSince}) + } + } + sort.Slice(free, func(i, j int) bool { + if !free[i].since.Equal(free[j].since) { + return free[i].since.Before(free[j].since) + } + return free[i].app < free[j].app + }) + for i := 0; i < len(free) && i < over; i++ { + evict = append(evict, free[i].app) + } + if short := over - len(evict); short > 0 { + refusedNew = short + added = added[:len(added)-short] // the report's app list first, then its claims: the tail goes + } + } + s.notePresenceFull(customerID, refusedNew, len(evict)) + changes = append(changes, added...) + if len(changes) == 0 && len(evict) == 0 { return nil } tx, err := s.db.Begin() @@ -112,6 +197,11 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t return err } defer tx.Rollback() + for _, app := range evict { + if _, err := tx.Exec(`DELETE FROM offsite_app_presence WHERE customer_id = ? AND app = ?`, customerID, app); err != nil { + return fmt.Errorf("NoteAppList %s/%s: %w", customerID, app, err) + } + } for _, c := range changes { if _, err := tx.Exec(`INSERT INTO offsite_app_presence (customer_id, app, absent_since) VALUES (?, ?, ?) ON CONFLICT(customer_id, app) DO UPDATE SET absent_since = excluded.absent_since`, customerID, c.app, c.absent); err != nil { @@ -121,6 +211,30 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t return tx.Commit() } +// presenceFull remembers, per household, that the row limit was said in the log — so a box that sends new names in +// every report gives ONE line, not one per report. In memory: a hub restart says it once more. +var ( + presenceFullMu sync.Mutex + presenceFull = map[string]bool{} +) + +func (s *Store) notePresenceFull(customerID string, refusedNew, evicted int) { + presenceFullMu.Lock() + defer presenceFullMu.Unlock() + if refusedNew == 0 && evicted == 0 { + delete(presenceFull, customerID) + return + } + if presenceFull[customerID] { + return + } + presenceFull[customerID] = true + if s.logger != nil { + s.logger.Printf("[WARN] offsite app record for %s is at its limit of %d names: this report took the place of %d old absent name(s) and %d new name(s) were NOT stored. A box does not run that many apps — look at what it reports. (Said once; not repeated per report.)", + customerID, maxPresenceRows, evicted, refusedNew) + } +} + // AppPresences returns the hub's record for one household, by app. func (s *Store) AppPresences(customerID string) (map[string]AppPresence, error) { rows, err := s.db.Query(`SELECT app, absent_since FROM offsite_app_presence WHERE customer_id = ?`, customerID)