diff --git a/documentation/architecture/07-backup-architecture.md b/documentation/architecture/07-backup-architecture.md index e74fd713..76bbccd9 100644 --- a/documentation/architecture/07-backup-architecture.md +++ b/documentation/architecture/07-backup-architecture.md @@ -1881,7 +1881,8 @@ section adds one thing to that window. 5. **Local copies are unchanged.** They are deleted at once, as before. The dialog says both. **The allowance (D84)** `[FACT — built, not released]`. The window's cap (`max_remove`, half the store, at least 5) -and its rule are not touched: the guard still refuses a policy plan above it. The ended apps' ids are BESIDE it — the +and its rule are not touched: the guard still refuses a policy plan above it. **One window's allowance is at most the +size of that cap (a narrowing of D84 by CC, below).** The ended apps' ids are BESIDE it — the hub stores the ids it agreed to on the window's row, keeps from the box's result only those, and allows a fall of the cap plus those. `store.WindowCreditsBetween` gives the two parts apart. @@ -1915,6 +1916,10 @@ notification address gets no mail: the page is all it has.** A blocked household | The guard refuses the window (a future-dated snapshot, a plan above the cap) | The ended apps wait with everything else | `TestD84_OneGuard_AndTheAllowanceIsBesideTheCap` | | The hub does not know ended apps (an older hub), or the box does not (an older controller) | Nothing of an ended app is removed; each side is releasable alone | `TestD84_TheBoxRemovesOnlyWhatTheHubAgreedTo` | | The app has no copy in the store at all | The due record is closed with no history line; nothing is asked of the hub | `TestD72_ARecordWithNoCopiesIsClosedQuietly` | +| A take-back or an install while the window is being prepared (the hub answers, the dry run, the guard) | The window reads the records AGAIN immediately before the forget; an id whose record is gone or changed is not removed. A take-back during the forget itself is answered „not marked, or already deleted”. **Not covered:** an install during the forget | `TestD72_ATakeBackDuringTheWindowKeepsTheCopies`, `TestD72_AnInstallDuringTheWindowKeepsTheCopies`, `TestD72_ATakeBackDuringTheForgetIsNotAnsweredKept` | +| The app is installed again by an import, or by a restore that installs from a unit | The record is deleted at the ONE point every install passes (`stacks.SaveAppConfig`) | `TestD72_AnImportDeletesTheEndRecord`, `TestD72_ARestoreIntoAnInstallDeletesTheEndRecord`, `TestD72_TheInstallWritersAreListed` | +| A box names an app the hub never saw installed | The operator is mailed at once; the hub's 30 days start then | hub `TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce` | +| A request for more than half of the store | It waits one week after a mail to the operator; then at most the cap's size per window | hub `TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize`, `TestD84_TheRestGoesInTheNextWindow` | | A box sends invented app names in every report | The hub's app record holds at most 400 names per household; only names shaped like an app name are stored. No age limit on a row, on purpose: it would move a real app's 30 days | hub `TestD72_AppRecord_*` | **What differs from the 2026-11-14 design, and what is not built.** @@ -1931,10 +1936,39 @@ notification address gets no mail: the page is all it has.** A blocked household - *The receipt does not read the household's group switches.* D84's reason is a household whose login someone else uses; that person could switch a group off first. **The operator may reverse this.** -**A limit, said plainly.** The hub cannot read a snapshot's tags (the repository is encrypted; the password never -leaves the box — decision 68). A box that is taken over AND has one honestly ended app can name, under that app, -other snapshots that are older than the app's end. The upload-time rule bounds that to files older than 30 days; it -does not close it. The same box can already remove half the store per week inside the cap. +**A limit, said plainly (rewritten after the review of 2026-11-15).** The hub cannot read a snapshot's tags (the +repository is encrypted; the password never leaves the box — decision 68), and its record of which apps a box runs is +made from what the box reports. So **no honestly ended app is needed, only 30 days**: a box that is taken over can +name an app that never existed in a pending end record, or stop listing a living app, wait 30 days by the hub's +clock, and then name snapshots of other apps under it. What bounds that now, all on the hub: + +- **A name the hub never saw installed is told to the operator at once** — a mailed warning + (`offsite_end_claim_unseen`), once per household and app, when the box first names it: 30 days before the hub could + agree to anything. It names the household, the app and the day the hub would agree. At most 8 per household. +- **A request for more than half of the store waits one week.** The first window that sees it agrees to nothing for + removed apps and mails the operator (`offsite_ended_large_agreement`); only a window at least six days later + agrees. The week belongs to those apps: another app set waits its own week. +- **One window agrees to at most the size of its own cap** — and the cap the hub's OWN count of the store gives, not + the box's `count_before`. The oldest files go first; the rest waits for the next window. +- An id is agreed only if the hub's own listing holds it, uploaded before the hub saw the app leave — so nothing + younger than 30 days can go this way. +- A box that removes more than was agreed raises the drop alarm at the close and the vanished-files alarm at 07:10. + +**What is still open:** a box that stops listing a living app is NOT told at the start (it looks exactly like an +honest removal); the hub hears of it only in the window, and then only if the request is for more than half of the +store. Under that size, such a box can remove up to the cap's size per week beside the cap, without a mail — twice +what it could remove silently before this feature (R-822's accepted limit, `09` §3 decision 166). And one window can +still take the cap AND the cap's size again; after a large request that is a week after the operator was mailed. + +**A narrowing of D84 — decided by CC unattended on 2026-11-15; the operator may reverse it.** *Question:* may one +window remove every snapshot the hub agreed to for removed apps, however many? *Option A (built):* no — at most the +size of the window's own cap per window, and a request for more than half of the store waits one week after a mail to +the operator. **Cost:** a household that removes a very large app has its off-site copies deleted over two or more +weekly windows, and waits one week more when the app was more than half of its store; one more operator mail per such +removal. *Option B (D84 as ruled, 309):* every agreed id in the first window. **Cost:** a box that lies about its app +list for 30 days can have the whole store removed in one window, and the first mail arrives with the deletion. *Why A:* +the ruling's own reason for the allowance was that the hub's 30-day check does the cap's job; the review showed the +30-day check rests on the box's own reports, so it cannot do that job alone. **The texts, as built — NOT published; the operator reads them before a release.** The Hungarian date carries its own final dot (`2026. 12. 15.`), so the Hungarian sentences have no dot after it. diff --git a/documentation/architecture/08-alarm-ladder.md b/documentation/architecture/08-alarm-ladder.md index 4949dae1..60df6c23 100644 --- a/documentation/architecture/08-alarm-ladder.md +++ b/documentation/architecture/08-alarm-ladder.md @@ -472,6 +472,8 @@ own records. What that adds here: |---|---|---|---| | `offsite_end_scheduled` (from the box) | info | **the household** — a receipt: one mail per press to its own notification address, whatever its group switches say; never the operator (`notify.householdReceiptEvents`) | the household pressed „delete backup data” at a removal, or „delete in 30 days” on the restore page | | `offsite_ended_app_removed` (hub) | info | operator-only, recorded, not mailed | a window removed an app's agreed snapshots — one per app, with the count | +| `offsite_end_claim_unseen` (hub; review of 2026-11-15) | warning, **mailed** | operator-only, once per household and app, at most 8 per household | a box first names, in a pending end record, an app the hub never saw in its app list — 30 days before the hub could agree to anything. The text names the household, the app and the day | +| `offsite_ended_large_agreement` (hub; review of 2026-11-15) | warning, **mailed** | operator-only, once per app set | a window was asked to remove more than half of the store's snapshots for removed apps. That window removes none of them; a window a week later does, at most the cap's size | | `offsite_ended_app_refused` (hub) | warning | operator-only, once per app (details `subject`) | a box asked for an app the hub's records clearly do not show ended: the app is in its app list, or absent for less than 29 days. A box that asks less than a day early is refused quietly | The count detector above and the 07:10 listing check (`offsite_snapshots_vanished`) both know the allowance: a window diff --git a/documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt new file mode 100644 index 00000000..db8482e9 --- /dev/null +++ b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt @@ -0,0 +1,110 @@ +FINDING 1 — a box could invent an ended app (or stop listing a living one), wait 30 days, and have the hub agree to the whole store beyond the cap, with no operator signal. + +### RED: the new tests on the hub code as it was pushed (felhom.eu bb226f23), before the fix +(the first three tests had, at that point, the first shape of the rule: one window at most the cap's size; the rule then grew a week's wait — see the note below) +=== RUN TestD84_OneWindowAgreesAtMostTheCapsSize + ended_bound_test.go:55: one window agreed to 20 ended ids with a cap of 10 — the allowance must be at most the cap's size +--- FAIL: TestD84_OneWindowAgreesAtMostTheCapsSize (0.06s) +=== RUN TestD84_TheRestGoesInTheNextWindow + ended_bound_test.go:103: first window: 15 agreed, want the cap's size 10 +--- FAIL: TestD84_TheRestGoesInTheNextWindow (0.05s) +=== RUN TestD84_TheAllowanceIsBoundByTheHubsOwnCount + ended_bound_test.go:143: a box that claims 1000 snapshots got 20 ended ids agreed; the hub's own listing holds 20 → at most 10 +--- FAIL: TestD84_TheAllowanceIsBoundByTheHubsOwnCount (0.04s) +=== RUN TestD84_ASmallEndIsQuiet + ended_bound_test.go:159: offsite_ended_large_agreement must be operator-only +--- FAIL: TestD84_ASmallEndIsQuiet (0.04s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys 0.199s +FAIL +=== RUN TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce + d84_unseen_claim_test.go:52: want ONE warning for the app the hub never saw, got 0 +--- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.05s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.068s +FAIL + +### After the fix: mutations +== MUTATION: a request for more than half of the store does not wait + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.05s) + ended_bound_test.go:74: the window that first saw a request for the whole store agreed to 10 ids — it must agree to none and tell the operator first + --- FAIL: TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek (0.05s) + ended_bound_test.go:143: a large request for another app rode the first app's week: agreed 10, refused [{App: Reason:10 snapshot id(s) wait for the next window: one window removes at most 10 snapshots of removed apps (the size of +== MUTATION: the wait is not a week (two days later it is agreed) + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.09s) + ended_bound_test.go:105: two days after the warning a window agreed to 10 ids — the operator must have a week +== MUTATION: the operator is mailed at every window of one large request + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.10s) + ended_bound_test.go:125: the operator was told 2 time(s) about one large request, want once +== MUTATION: one window agrees to everything (no bound to the cap's size) + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.10s) + ended_bound_test.go:110: a week later: 20 ended ids agreed with a cap of 10 — want exactly the cap's size + --- FAIL: TestD84_TheRestGoesInTheNextWindow (0.05s) + ended_bound_test.go:163: a week later: 15 agreed, want the cap's size 10 +== MUTATION: the bound takes the box's own count (count_before) alone + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_TheAllowanceIsBoundByTheHubsOwnCount (0.04s) + ended_bound_test.go:206: a box that claims 1000 snapshots got 20 ended ids agreed; the hub's own listing holds 20 → at most 10 +== MUTATION: the newest files go first + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.11s) + ended_bound_test.go:118: the allowance did not take the oldest files first: a1b2c684 agreed +== MUTATION: „more than half” is „more than twice the store” + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize (0.05s) + ended_bound_test.go:74: the window that first saw a request for the whole store agreed to 10 ids — it must agree to none and tell the operator first + --- FAIL: TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek (0.05s) + ended_bound_test.go:143: a large request for another app rode the first app's week: agreed 10, refused [{App: Reason:10 snapshot id(s) wait for the next window: one window removes at most 10 snapshots of removed apps (the size of +== MUTATION: a large request for other apps rides an earlier request's week + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek (0.05s) + ended_bound_test.go:143: a large request for another app rode the first app's week: agreed 10, refused [{App: Reason:10 snapshot id(s) wait for the next window: one window removes at most 10 snapshots of removed apps (the size of + +8 mutations, 0 not a red-proof +== MUTATION: no limit on the never-seen warnings of one household + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.06s) + d84_unseen_claim_test.go:75: 40 invented names gave 41 operator warnings — a box must not be able to flood the operator (limit 8 per household) +== MUTATION: the never-seen mark is not stored (the limit is per report) + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.06s) + d84_unseen_claim_test.go:75: 40 invented names gave 9 operator warnings — a box must not be able to flood the operator (limit 8 per household) +== MUTATION: an app the hub SAW installed is told as never seen + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.05s) + d84_unseen_claim_test.go:45: an app the hub saw installed raised the never-seen warning: [{ID:1 CustomerID:c1 EventType:offsite_end_claim_unseen Severity:warning Message:Off-site: the box of c1 holds an end record for the app "vik + +3 mutations, 0 not a red-proof +== MUTATION: the operator is not told about a never-seen app + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.05s) + d84_unseen_claim_test.go:52: want ONE warning for the app the hub never saw, got 0 +== MUTATION: the message names today, not the day the hub would agree + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.05s) + d84_unseen_claim_test.go:56: the warning must be a warning and name the household, the app and the day the hub would agree (2026-12-15): {ID:1 CustomerID:c1 EventType:offsite_end_claim_unseen Severity:warning Message:Off-site: the +== MUTATION: the warning has no per-app subject (two apps in an hour are one mail) + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce (0.04s) + d84_unseen_claim_test.go:59: no per-app subject in the details: {"agrees_from":"2026-12-15","app":"ghost"} + +3 mutations, 0 not a red-proof +== MUTATION: the two warnings are not operator-only + RED on the mutant, GREEN on the real file + --- FAIL: TestD84_ASmallEndIsQuiet (0.04s) + ended_bound_test.go:222: offsite_ended_large_agreement must be operator-only + +1 mutations, 0 not a red-proof + +NOTE. The RED block above was taken with the first shape of the fix's tests (one window agrees to at most the cap's +size). While fixing, that bound alone still let one window take the cap AND the cap's size again — a store of 20 +emptied in one window with the operator's mail arriving at the same moment. So the rule grew: a request for more +than half of the store waits one week after the mail. The tests were rewritten for that rule +(TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize, TestD84_TheRestGoesInTheNextWindow, +TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek); the mutations above run against the rewritten tests. +The reviewer's own reproduction (scratchpad r4/review-A/zz_ghost_test.go: "all 20 snapshots agreed for an app the hub +never saw installed; cap was 10; no alarm") is the same defect as the first RED line. diff --git a/documentation/audits/build-release-4-2026-11-15/A/redproof-review-2.txt b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-2.txt new file mode 100644 index 00000000..f041aa64 --- /dev/null +++ b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-2.txt @@ -0,0 +1,52 @@ +FINDING 2 — a take-back or an install during the window still deleted. + +### RED: the new tests on the code as it was pushed (controller f628cb5), before the fix +=== RUN TestD72_ATakeBackDuringTheWindowKeepsTheCopies + offbox_ended_race_test.go:48: the page said „the off-site copies are kept”, no record exists, and the window removed kimai's copy ([k1-full v1-full v2-full v3-full]) +--- FAIL: TestD72_ATakeBackDuringTheWindowKeepsTheCopies (0.00s) +=== RUN TestD72_AnInstallDuringTheWindowKeepsTheCopies +=== RUN TestD72_AnInstallDuringTheWindowKeepsTheCopies/the_install_deleted_the_record + offbox_ended_race_test.go:91: the window removed [k1-full] of an app that was installed again while the hub answered +=== RUN TestD72_AnInstallDuringTheWindowKeepsTheCopies/the_app_is_living_again_(the_record_is_still_there) + offbox_ended_race_test.go:91: the window removed [k1-full] of an app that was installed again while the hub answered +=== RUN TestD72_AnInstallDuringTheWindowKeepsTheCopies/removed_again:_a_NEW_record_with_a_new_time + offbox_ended_race_test.go:91: the window removed [k1-full] of an app that was installed again while the hub answered +--- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies (0.00s) + --- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies/the_install_deleted_the_record (0.00s) + --- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies/the_app_is_living_again_(the_record_is_still_there) (0.00s) + --- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies/removed_again:_a_NEW_record_with_a_new_time (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup 0.023s +FAIL + +### After the fix: mutations +== MUTATION: the window does not look at the records again before the forget + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_ATakeBackDuringTheWindowKeepsTheCopies (0.00s) + offbox_ended_race_test.go:48: the page said „the off-site copies are kept”, no record exists, and the window removed kimai's copy ([k1-full v1-full v2-full v3-full]) + --- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies (0.00s) + offbox_ended_race_test.go:91: the window removed [k1-full] of an app that was installed again while the hub answered + +1 mutations, 0 not a red-proof +== MUTATION: a record written ANEW since the claim counts as the claimed one + *** NOT A RED-PROOF *** (mutant rc=1 built=False real rc=0) + FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup [build failed] + internal/backup/offbox_ended.go:281:11: declared and not used: e +== MUTATION: the ids are not checked against the records as they are now + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_ATakeBackDuringTheWindowKeepsTheCopies (0.00s) + offbox_ended_race_test.go:48: the page said „the off-site copies are kept”, no record exists, and the window removed kimai's copy ([k1-full v1-full v2-full v3-full]) + --- FAIL: TestD72_AnInstallDuringTheWindowKeepsTheCopies (0.00s) + offbox_ended_race_test.go:91: the window removed [k1-full] of an app that was installed again while the hub answered +== MUTATION: a take-back during the forget is answered „kept” + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_ATakeBackDuringTheForgetIsNotAnsweredKept (0.00s) + offbox_ended_race_test.go:119: asked=true kept=true — a take-back during the forget was answered „kept” while the copies were being removed + +3 mutations, 1 not a red-proof + +NOTE on the mutation that is "not a red-proof": it did not compile (`e` unused). It was NOT done again, because the +changed condition cannot be reached by a test: a record written anew is never due (its 30 days have just started), so +`dueOffsiteEnds` has already dropped it before the removed_at comparison. The comparison stays in the code as a second +line and is NOT claimed as proven. The case itself ("removed again: a NEW record with a new time") is in the RED block +above and is green after the fix. diff --git a/documentation/audits/build-release-4-2026-11-15/A/redproof-review-3.txt b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-3.txt new file mode 100644 index 00000000..a0d25e29 --- /dev/null +++ b/documentation/audits/build-release-4-2026-11-15/A/redproof-review-3.txt @@ -0,0 +1,65 @@ +FINDING 3 — an import installed the app and the off-site end record survived. + +### RED: the new tests on the code as it was (controller 3928c3d), before the fix +=== RUN TestD72_AnImportDeletesTheEndRecord + d72_every_install_path_test.go:59: import: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:27:55Z EndsAt:2026-11-16T13:27:55Z}) — the window would delete a living app's history +--- FAIL: TestD72_AnImportDeletesTheEndRecord (0.00s) +=== RUN TestD72_ARestoreIntoAnInstallDeletesTheEndRecord + d72_every_install_path_test.go:68: unit restore: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:27:55Z EndsAt:2026-11-16T13:27:55Z}) — the window would delete a living app's history +--- FAIL: TestD72_ARestoreIntoAnInstallDeletesTheEndRecord (0.00s) +=== RUN TestD72_ADriveFlipRecordWriteDeletesTheEndRecord + d72_every_install_path_test.go:80: drive flip: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:27:55Z EndsAt:2026-11-16T13:27:55Z}) — the window would delete a living app's history +--- FAIL: TestD72_ADriveFlipRecordWriteDeletesTheEndRecord (0.00s) +=== RUN TestD72_SaveAppConfigIsTheOnePoint + d72_every_install_path_test.go:96: SaveAppConfig(Deployed: true): the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:27:55Z EndsAt:2026-11-16T13:27:55Z}) — the window would delete a living app's history +--- FAIL: TestD72_SaveAppConfigIsTheOnePoint (0.00s) +=== RUN TestD72_TheInstallWritersAreListed + d72_every_install_path_test.go:155: a struct literal that says Deployed: true: + d72_every_install_path_test.go:164: SaveAppConfig does not call noteAppInstalled — no install path deletes the off-site end record +--- FAIL: TestD72_TheInstallWritersAreListed (0.52s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.573s +FAIL + +### After the fix: mutations +== MUTATION: SaveAppConfig is not the install point (only DeployStack deletes the record) + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AnImportDeletesTheEndRecord (0.00s) + d72_every_install_path_test.go:59: import: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:29:15Z EndsAt:2026-11-16T13:29:15Z}) — the window would delete a living app's history + --- FAIL: TestD72_ARestoreIntoAnInstallDeletesTheEndRecord (0.00s) + d72_every_install_path_test.go:68: unit restore: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:29:15Z EndsAt:2026-11-16T13:29:15Z}) — the window would delete a living app's hist +== MUTATION: a „not deployed” write deletes the record too + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_SaveAppConfigIsTheOnePoint (0.00s) + d72_every_install_path_test.go:91: a „not deployed” write deleted the end record +== MUTATION: one global hook: a write in another stacks directory reaches this box + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_SaveAppConfigIsTheOnePoint (0.00s) + d72_every_install_path_test.go:96: SaveAppConfig(Deployed: true): the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:29:27Z EndsAt:2026-11-16T13:29:27Z}) — the window would delete a +== MUTATION: living: an app being installed is not living + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_IsLivingApp (0.00s) + d72_every_install_path_test.go:201: being installed (Deploying): "gapp" does not read as living +== MUTATION: living: the protected list is not read + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_IsLivingApp (0.00s) + d72_every_install_path_test.go:201: a protected stack by the config, not in the list: "traefik" does not read as living + d72_every_install_path_test.go:201: always-protected samba: "samba" does not read as living +== MUTATION: living: an installed app the manager does not list is „removed” + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_IsLivingApp (0.00s) + d72_every_install_path_test.go:220: an INSTALLED app the manager does not list reads as removed — the window could end its copies +== MUTATION: living: everything is living (no end is ever carried out) + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_IsLivingApp (0.00s) + d72_every_install_path_test.go:181: a catalog app that is NOT installed reads as living — its end could never be carried out + +7 mutations, 0 not a red-proof +== MUTATION: the manager does not register the install point + RED on the mutant, GREEN on the real file + --- FAIL: TestD72_AnImportDeletesTheEndRecord (0.00s) + d72_every_install_path_test.go:59: import: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:29:56Z EndsAt:2026-11-16T13:29:56Z}) — the window would delete a living app's history + --- FAIL: TestD72_ARestoreIntoAnInstallDeletesTheEndRecord (0.00s) + d72_every_install_path_test.go:68: unit restore: the app is installed again and its off-site end record survives ({App:gapp RemovedAt:2026-10-17T13:29:56Z EndsAt:2026-11-16T13:29:56Z}) — the window would delete a living app's hist + +1 mutations, 0 not a red-proof diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 734bfc0f..4c671ed2 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,24 @@ ## Unreleased (after v0.182.0) +- **Review fix 1 (2026-11-15): a box that lies about its app list can no longer have the whole store agreed in one + window with no signal.** The hub's 30-day check rests on what the box reports, so a box could name an app that + never existed (or stop listing a living one), wait 30 days, and get every snapshot agreed beside the cap. Now: + (a) **one window agrees to at most the size of its own cap**, taken from the hub's OWN count of the store + (`endedCap`, `boundEndedToCap`), oldest files first; the rest waits for the next window. *A narrowing of D84, + decided by CC unattended — the operator may reverse it (`07` §8.2f).* (b) **`offsite_end_claim_unseen`** (warning, + mailed, operator-only): the moment a box first names, in `offsite.ended_apps`, an app the hub never saw in its app + list — once per household and app, at most 8 per household (`store.NoteAppListClaims`, column `claim_born`). + (c) **`offsite_ended_large_agreement`** (warning, mailed, operator-only): a request for more than half of the store + **waits one week** — the window that first sees it agrees to nothing for removed apps; a window at least six days + later agrees. Neither event is in `allowedEventTypes`: a box cannot POST one. **Still open, said in `07`:** a box + that stops listing a living app is not told at the start, and under half of the store it can remove up to the cap's + size per week beside the cap without a mail. **Not done:** catching, at the close, a box that names agreed ids and + removed others — it needs the hub to list the repository inside the box-facing close (a new provider call there and + a second writer of the listing baseline); it is caught at 07:10, as before. + Tests `TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize`, `TestD84_TheRestGoesInTheNextWindow`, + `TestD84_TheAllowanceIsBoundByTheHubsOwnCount`, `TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek`, + `TestD84_ASmallEndIsQuiet`, `TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce` — seen red on the pushed code; 15 + mutations seen red (`documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt`). - **The off-site copies of a removed app end after 30 days — the hub's half (R-1374 part 1; D72, D84; `09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f). Built, NOT deployed; no box has run it.** (1) **The hub's own app record.** `store.NoteAppList` (new table `offsite_app_presence`, fed by every controller diff --git a/hub/internal/api/d84_unseen_claim_test.go b/hub/internal/api/d84_unseen_claim_test.go new file mode 100644 index 00000000..bf0b1612 --- /dev/null +++ b/hub/internal/api/d84_unseen_claim_test.go @@ -0,0 +1,100 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/notify" + "gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// Review of 2026-11-15, finding 1(b): the moment a box FIRST names, in a pending end record, an app the hub never saw +// in its app list, the operator is warned — once per (household, app), 30 days before the hub could agree to +// anything. An app the hub saw installed and then saw leave is the ordinary case and raises nothing. +// RED-PROOF: documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt. +func TestD84_AnAppTheHubNeverSawIsToldToTheOperatorOnce(t *testing.T) { + h, st, _ := newTestHandler(t) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "pp"}) + post := func(body string) { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/api/v1/report", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer ckey") + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("report → %d (%s)", rr.Code, rr.Body.String()) + } + } + told := func() []store.Event { + t.Helper() + evs, err := st.GetEventsByType("c1", offsitekeys.EventEndedClaimUnseen, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + return evs + } + post(`{"customer_id":"c1","stacks":{"deployed":["vikunja","immich"]}}`) + // vikunja is removed with „delete backup data”: the hub SAW it installed — the ordinary case, no warning. + post(`{"customer_id":"c1","stacks":{"deployed":["immich"]},"offsite":{"enabled":true,"ended_apps":[{"app":"vikunja"}]}}`) + if e := told(); len(e) != 0 { + t.Fatalf("an app the hub saw installed raised the never-seen warning: %+v", e) + } + // „ghost” was never in the app list. + body := `{"customer_id":"c1","stacks":{"deployed":["immich"]},"offsite":{"enabled":true,"ended_apps":[{"app":"vikunja"},{"app":"ghost","removed_at":"2024-01-01T00:00:00Z","ends_at":"2024-01-31T00:00:00Z"}]}}` + post(body) + e := told() + if len(e) != 1 { + t.Fatalf("want ONE warning for the app the hub never saw, got %d", len(e)) + } + day := time.Now().UTC().Add(offsitekeys.EndedAppAfter).Format("2006-01-02") + if e[0].Severity != "warning" || !strings.Contains(e[0].Message, "c1") || !strings.Contains(e[0].Message, `"ghost"`) || !strings.Contains(e[0].Message, day) { + t.Fatalf("the warning must be a warning and name the household, the app and the day the hub would agree (%s): %+v", day, e[0]) + } + if !strings.Contains(e[0].DetailsJSON, `"subject":"ghost"`) { + t.Fatalf("no per-app subject in the details: %s", e[0].DetailsJSON) + } + // the same record in every later report: told ONCE + for i := 0; i < 5; i++ { + post(body) + } + if e := told(); len(e) != 1 { + t.Fatalf("the warning was repeated: %d", len(e)) + } + // a box that invents many names: the marks are capped per household + var many []string + for i := 0; i < 40; i++ { + many = append(many, fmt.Sprintf(`{"app":"inv-%d"}`, i)) + } + post(`{"customer_id":"c1","stacks":{"deployed":["immich"]},"offsite":{"enabled":true,"ended_apps":[` + strings.Join(many, ",") + `]}}`) + if n := len(told()); n > 8 { + t.Fatalf("40 invented names gave %d operator warnings — a box must not be able to flood the operator (limit 8 per household)", n) + } + // … and in the NEXT report 40 more new names: the limit is the household's, not one report's + many = many[:0] + for i := 0; i < 40; i++ { + many = append(many, fmt.Sprintf(`{"app":"more-%d"}`, i)) + } + post(`{"customer_id":"c1","stacks":{"deployed":["immich"]},"offsite":{"enabled":true,"ended_apps":[` + strings.Join(many, ",") + `]}}`) + if n := len(told()); n != 8 { + t.Fatalf("80 invented names over two reports gave %d operator warnings, want exactly the household's limit 8", n) + } + // a name that is not an app name is never told (and never stored) + post(`{"customer_id":"c1","stacks":{"deployed":["immich"]},"offsite":{"enabled":true,"ended_apps":[{"app":"x"},{"app":"_shares"},{"app":"traefik"}]}}`) + for _, ev := range told() { + if strings.Contains(ev.Message, "") || strings.Contains(ev.Message, "_shares") || strings.Contains(ev.Message, "traefik") { + t.Fatalf("an invalid or reserved name reached an operator message: %s", ev.Message) + } + } + // hub-made: no box can POST it, the household never gets it, and each app is its own mail + if allowedEventTypes[offsitekeys.EventEndedClaimUnseen] || allowedEventTypes[offsitekeys.EventEndedLargeAgreement] { + t.Fatal("a box must not be able to POST the hub's own warnings") + } + if !notify.IsOperatorOnly(offsitekeys.EventEndedClaimUnseen) { + t.Fatal("offsite_end_claim_unseen must be operator-only") + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 45a64bb5..2a431981 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -27,6 +27,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/mailhold" "gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" + "gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) @@ -3699,7 +3700,29 @@ func (h *Handler) noteAppList(customerID string, body []byte) { claimed = append(claimed, e.App) } } - if err := h.store.NoteAppList(customerID, p.Stacks.Deployed, claimed, time.Now()); err != nil { + unseen, err := h.store.NoteAppListClaims(customerID, p.Stacks.Deployed, claimed, time.Now()) + if err != nil { h.logger.Printf("[WARN] [api] report from %s: the hub's own app record could not be updated: %v", customerID, err) + return + } + // Review of 2026-11-15: an app the hub NEVER saw in this box's app list, named in an end record for the first + // time. It is the honest shape of „removed before the hub kept this record” — and the shape of a box that invents + // an app to have its copies agreed. The operator hears it now, 30 days before the hub could agree to anything. + // The store returns each app once and at most a few per household; the name has the shape of an app name. + for _, u := range unseen { + day := u.AbsentSince.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02") + msg := fmt.Sprintf("Off-site: the box of %s holds an end record for the app %q (its off-site copies are to be deleted 30 days after „delete backup data”), "+ + "and the hub never saw that app in this box's app list. The hub's own 30 days start now: from %s it would agree, in a clean-up window, "+ + "to remove that app's off-site snapshots — at most the window's cap per week. If the household did not remove an app of that name, "+ + "look at the box before that day and switch the weekly windows off for it.", customerID, u.App, day) + details, _ := json.Marshal(map[string]any{"subject": u.App, "app": u.App, "agrees_from": day}) + h.logger.Printf("[WARN] [api] report from %s: an end record names %q, an app the hub never saw installed — operator told; the hub would agree from %s", customerID, u.App, day) + if _, eerr := h.store.SaveEvent(customerID, offsitekeys.EventEndedClaimUnseen, "warning", msg, string(details), "hub"); eerr != nil { + h.logger.Printf("[WARN] [api] %s event save FAILED for %s: %v", offsitekeys.EventEndedClaimUnseen, customerID, eerr) + continue // the audit row first: a mail without its event row lies + } + if h.dispatcher != nil { + go h.dispatcher.ProcessEvent(customerID, offsitekeys.EventEndedClaimUnseen, "warning", msg, string(details), "hub") + } } } diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 543932eb..9bdde952 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -642,6 +642,9 @@ var perSubjectCooldownEvents = map[string]bool{ "proxmox_trust_problem": true, // D72/D84: one refusal per app (details `subject` = the app) — two apps refused in one window are two mails. "offsite_ended_app_refused": true, + // one never-seen app is one mail (subject = the app); one window's large request is one mail (subject = the window) + "offsite_end_claim_unseen": true, + "offsite_ended_large_agreement": true, } // cooldownSubjectSuffix returns ":"+subject for an allow-listed type whose details carry a non-empty `subject`. @@ -906,6 +909,10 @@ var operatorOnlyEvents = map[string]bool{ // an end the hub's own records do not show (warning). Hub-origin only: NOT in api.allowedEventTypes. "offsite_ended_app_removed": true, "offsite_ended_app_refused": true, + // Review of 2026-11-15: a box named an app the hub never saw installed (told once per household and app, 30 days + // before the hub could agree), and a window was asked for more than half of the store. Hub-origin only. + "offsite_end_claim_unseen": true, + "offsite_ended_large_agreement": true, // OS updates (hub v0.130.0, `11` §8 step 2): run failures, rings, switches and approvals are operator facts. // os_update_applied is deliberately NOT here — it is the household's one line (info: recorded, never mailed). "os_update_failed": true, diff --git a/hub/internal/offsitekeys/ended.go b/hub/internal/offsitekeys/ended.go index 5787b235..fa8620a1 100644 --- a/hub/internal/offsitekeys/ended.go +++ b/hub/internal/offsitekeys/ended.go @@ -3,6 +3,7 @@ package offsitekeys import ( "fmt" "sort" + "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -33,6 +34,14 @@ const EndedAppAfter = 30 * 24 * time.Hour const ( EventEndedAppRemoved = "offsite_ended_app_removed" // info: the off-site copies of a removed app are gone (per app) EventEndedAppRefused = "offsite_ended_app_refused" // warning: a box claimed an end the hub's records do not show + // Review of 2026-11-15 — two more operator warnings, both mailed, both hub-made (no box can POST one): + EventEndedClaimUnseen = "offsite_end_claim_unseen" // a box first names an app the hub never saw installed + EventEndedLargeAgreement = "offsite_ended_large_agreement" // one window was asked for more than half of the store + + // A request for more than half of the store waits largeEndedWait after the window that first saw it; that first + // sight is remembered for largeEndedMemory (a new large request after that waits again). + largeEndedWait = 6 * 24 * time.Hour + largeEndedMemory = 30 * 24 * time.Hour maxEndedClaims = 50 // apps in one request maxEndedClaimIDs = 5000 // snapshot ids in one request, all apps together @@ -190,7 +199,7 @@ func clearDisagreement(now time.Time, presence map[string]store.AppPresence, r s // agreeEndedFor runs the decision for one household against the store, and tells the operator about a clear // disagreement. It never fails a window: with a store that cannot answer, nothing is agreed. -func (s *Service) agreeEndedFor(customerID string, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) { +func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) { if len(claims) == 0 { return nil, nil, nil } @@ -205,6 +214,40 @@ func (s *Service) agreeEndedFor(customerID string, claims []EndedClaim) (ids []s } listing, have := s.Store.GetOffsiteSnapshotListing(customerID) ids, byApp, refused = agreeEnded(now, presence, listing, have, claims) + // Review of 2026-11-15 — what bounds a box that invents an ended app, or stops listing a living one, and 30 days + // later names much of the store under it: + // + // (c) A request for more than HALF of the store's snapshots WAITS ONE WEEK: the first window that sees it agrees + // to nothing for removed apps and tells the operator (a mail); a window at least largeEndedWait later agrees. + // So the operator always has a week between the mail and the first removal. An honest household that + // removed its one big app looks the same and loses one week. + // (a) ONE window agrees to at most the cap's own size (boundEndedToCap); the rest waits for the next window. + if total := len(listing.Entries); have && len(ids)*2 > total { + // The week belongs to THESE apps: a large request for other apps waits its own week and gets its own mail. + mark := store.LargeEndedRefusalMark + " (apps: " + appNames(byApp) + ")" + first := s.Store.FirstLargeEndedRefusal(customerID, mark, now.Add(-largeEndedMemory)) + if first.IsZero() || now.Sub(first) < largeEndedWait { + if first.IsZero() { + s.event(customerID, EventEndedLargeAgreement, "warning", + fmt.Sprintf("Off-site clean-up window %d: the box of %s asked to remove %d of %d off-site snapshot(s) as copies of removed apps (%s) — more than half of the store. "+ + "NOTHING of them is removed in this window. The hub's own records agree to each app, so from %s a window removes them, at most %d per window. "+ + "If the household did not remove these apps, switch the weekly windows off for it before that day and look at the box.", + windowID, customerID, len(ids), total, appNames(byApp), now.Add(largeEndedWait).UTC().Format("2006-01-02"), endedCap(maxRemove, total)), + map[string]any{"subject": "large:" + appNames(byApp), "window_id": windowID, "asked": len(ids), "listed": total}) + } + refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("the request is for %d of %d snapshots — %s: it waits one week, and the operator is told", + len(ids), total, mark)}) + ids, byApp = nil, nil + } + } + if have && len(ids) > 0 { + var waits int + ids, byApp, waits = boundEndedToCap(ids, byApp, listing, endedCap(maxRemove, len(listing.Entries))) + if waits > 0 { + refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("%d snapshot id(s) wait for the next window: one window removes at most %d snapshots of removed apps (the size of its own cap)", + waits, endedCap(maxRemove, len(listing.Entries)))}) + } + } s.logf("[INFO] offsitekeys: ended apps %s: %d app(s) claimed, %d app(s) with agreed snapshots, %d snapshot id(s) agreed, %d refusal(s)", customerID, len(claims), len(byApp), len(ids), len(refused)) for _, r := range refused { @@ -235,3 +278,63 @@ func endedRemovedByApp(claims map[string][]string, removed []string) map[string] } return out } + +// endedCap is the most ended-app snapshots ONE window may agree to: the size of the window's own cap, and never more +// than the cap the hub's OWN count of the store would give (the window's cap comes from the box's count_before, which +// is the box's word). Decided by CC unattended on 2026-11-15, a narrowing of D84 — the operator may reverse it +// (`07-backup-architecture.md` §8.2f). +func endedCap(maxRemove, listed int) int { + c := MaxRemove(listed) + if maxRemove > 0 && maxRemove < c { + c = maxRemove + } + return c +} + +// boundEndedToCap keeps at most limit of the agreed ids — the OLDEST uploads first (the newest copies of a removed +// app are the last to go) — and returns how many wait. +func boundEndedToCap(ids []string, byApp map[string][]string, listing store.OffsiteSnapshotListing, limit int) ([]string, map[string][]string, int) { + if len(ids) <= limit { + return ids, byApp, 0 + } + sorted := append([]string(nil), ids...) + sort.Slice(sorted, func(i, j int) bool { + ti, tj := listing.Entries[sorted[i]], listing.Entries[sorted[j]] + if !ti.Equal(tj) { + return ti.Before(tj) + } + return sorted[i] < sorted[j] + }) + keep := make(map[string]bool, limit) + for _, id := range sorted[:limit] { + keep[id] = true + } + var out []string + for _, id := range ids { + if keep[id] { + out = append(out, id) + } + } + apps := map[string][]string{} + for app, list := range byApp { + for _, id := range list { + if keep[id] { + apps[app] = append(apps[app], id) + } + } + } + return out, apps, len(ids) - limit +} + +// appNames lists the agreed apps for a message (names already checked: store.ValidPresenceApp). +func appNames(byApp map[string][]string) string { + names := make([]string, 0, len(byApp)) + for app := range byApp { + names = append(names, app) + } + sort.Strings(names) + if len(names) > 10 { + return strings.Join(names[:10], ", ") + fmt.Sprintf(" and %d more", len(names)-10) + } + return strings.Join(names, ", ") +} diff --git a/hub/internal/offsitekeys/ended_bound_test.go b/hub/internal/offsitekeys/ended_bound_test.go new file mode 100644 index 00000000..0cef4ebd --- /dev/null +++ b/hub/internal/offsitekeys/ended_bound_test.go @@ -0,0 +1,225 @@ +package offsitekeys + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/notify" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// Review of 2026-11-15, finding 1: a box could invent an ended app (or stop listing a living one), wait 30 days, and +// have the hub agree to snapshots of the WHOLE store beyond the cap, with no operator signal. Three things now hold: +// the ended allowance of ONE window is at most the cap's own size (the rest waits for the next window); the operator +// is warned when a window is asked for more than half of the store; and (api, store) the operator is warned the +// moment a box first names an app the hub never saw installed — 30 days before anything can go. +// RED-PROOF: documentation/audits/build-release-4-2026-11-15/A/redproof-review-1.txt. + +// ghostFixture: 20 snapshot files (40 days old) and a box that has only ever NAMED „ghost” in an end record, 31 days +// ago; immich is installed all along. +func ghostFixture(t *testing.T) (*Service, *[]evt, []string) { + t.Helper() + s, _, events := watchFixture(t) + now := time.Now().UTC() + l := store.OffsiteSnapshotListing{ListedAt: now.Add(-20 * time.Hour), Entries: map[string]time.Time{}} + var all []string + for i := 0; i < 20; i++ { + l.Entries[eid(i)] = now.Add(-time.Duration(40*24+i) * time.Hour) // eid(19) is the oldest + all = append(all, eid(i)) + } + if err := s.Store.SetOffsiteSnapshotListing("c1", l); err != nil { + t.Fatal(err) + } + if err := s.Store.NoteAppList("c1", []string{"immich"}, []string{"ghost"}, now.Add(-31*24*time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.Store.NoteAppList("c1", []string{"immich"}, []string{"ghost"}, now.Add(-time.Hour)); err != nil { + t.Fatal(err) + } + *events = nil + return s, events, all +} + +// aWeekLater moves every window of the household eight days into the past (the ledger is stamped by SQLite's clock). +func aWeekLater(t *testing.T, s *Service, ids ...int64) { + t.Helper() + for _, id := range ids { + if err := s.Store.BackdateOffsiteWindowForTest(id, 8*24*time.Hour); err != nil { + t.Fatal(err) + } + } +} + +func hasReason(refused []store.EndedRefusal, frag string) bool { + for _, r := range refused { + if strings.Contains(r.Reason, frag) { + return true + } + } + return false +} + +// THE WORST CASE: the box names EVERY snapshot under the ghost. The first window agrees to NOTHING and the operator +// is mailed; if the box removes the store anyway, the drop alarm and the 07:10 audit both fire. A week later a window +// agrees — to at most the cap's own size, the oldest files first — and says that the rest waits. +func TestD84_ALargeRequestWaitsAWeekAndOneWindowAgreesAtMostTheCapsSize(t *testing.T) { + s, events, all := ghostFixture(t) + g := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: all}) + if g.MaxRemove != MaxRemove(20) { + t.Fatalf("the cap's own rule moved: %d, want %d", g.MaxRemove, MaxRemove(20)) + } + if len(g.EndedAllowed) != 0 { + t.Fatalf("the window that first saw a request for the whole store agreed to %d ids — it must agree to none and tell the operator first", len(g.EndedAllowed)) + } + if !hasReason(g.EndedRefused, "waits one week") { + t.Fatalf("the window does not say that the request waits: %+v", g.EndedRefused) + } + big := ofType(*events, EventEndedLargeAgreement) + if len(big) != 1 || big[0].sev != "warning" || !strings.Contains(big[0].msg, "c1") || !strings.Contains(big[0].msg, "20 of 20") || !strings.Contains(big[0].msg, "ghost") { + t.Fatalf("want ONE operator warning naming the household, the app and „20 of 20”: %+v", *events) + } + // the box removes the WHOLE store anyway and names all 20: nothing was agreed, so both alarms fire + *events = nil + if err := s.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: g.WindowID, CountBefore: 20, CountAfter: 0, Removed: 20, Outcome: "pruned", RemovedForEndedApps: all}); err != nil { + t.Fatal(err) + } + if len(ofType(*events, EventWindowDrop)) != 1 { + t.Fatalf("no drop alarm for a store emptied with nothing agreed: %+v", *events) + } + if v, _ := s.CheckSnapshotListing("c1", SnapshotListing{Entries: map[string]time.Time{}}, time.Now().UTC().Add(time.Minute)); v == 0 { + t.Fatal("the audit explained a store emptied beyond what the hub agreed to") + } + + // A second household state, where the box waits: two days later still nothing; a week later the cap's size. + s2, events2, all2 := ghostFixture(t) + g1 := openGranted(t, s2, 20, EndedClaim{App: "ghost", SnapshotIDs: all2}) + _ = s2.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: g1.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) + if err := s2.Store.BackdateOffsiteWindowForTest(g1.WindowID, 2*24*time.Hour); err != nil { + t.Fatal(err) + } + gEarly := openGranted(t, s2, 20, EndedClaim{App: "ghost", SnapshotIDs: all2}) + _ = s2.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: gEarly.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) + if len(gEarly.EndedAllowed) != 0 { + t.Fatalf("two days after the warning a window agreed to %d ids — the operator must have a week", len(gEarly.EndedAllowed)) + } + aWeekLater(t, s2, g1.WindowID, gEarly.WindowID) + g2 := openGranted(t, s2, 20, EndedClaim{App: "ghost", SnapshotIDs: all2}) + if len(g2.EndedAllowed) != g2.MaxRemove || g2.MaxRemove != 10 { + t.Fatalf("a week later: %d ended ids agreed with a cap of %d — want exactly the cap's size", len(g2.EndedAllowed), g2.MaxRemove) + } + for _, id := range g2.EndedAllowed { // the OLDEST go first: eid(10)..eid(19) + ok := false + for i := 10; i < 20; i++ { + ok = ok || id == eid(i) + } + if !ok { + t.Fatalf("the allowance did not take the oldest files first: %s agreed", shortID(id)) + } + } + if !hasReason(g2.EndedRefused, "wait for the next window") { + t.Fatalf("the window does not say that the rest waits: %+v", g2.EndedRefused) + } + if n := len(ofType(*events2, EventEndedLargeAgreement)); n != 1 { + t.Fatalf("the operator was told %d time(s) about one large request, want once", n) + } + if w, _ := s2.Store.GetOffsiteWindow(g2.WindowID); len(w.EndedAgreed) != 10 { + t.Fatalf("the window's row holds %d agreed ids", len(w.EndedAgreed)) + } +} + +// The week belongs to the apps that were told: a large request for ANOTHER app, a week after the first one's warning, +// waits its own week and gets its own mail. +func TestD84_ALargeRequestForOtherAppsWaitsItsOwnWeek(t *testing.T) { + s, events, all := ghostFixture(t) + now := time.Now().UTC() + _ = s.Store.NoteAppList("c1", []string{"immich"}, []string{"ghost", "second"}, now.Add(-31*24*time.Hour-time.Minute)) + g0 := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: all}) + _ = s.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: g0.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) + aWeekLater(t, s, g0.WindowID) + g1 := openGranted(t, s, 20, EndedClaim{App: "second", SnapshotIDs: all}) + if len(g1.EndedAllowed) != 0 || !hasReason(g1.EndedRefused, "waits one week") { + t.Fatalf("a large request for another app rode the first app's week: agreed %d, refused %+v", len(g1.EndedAllowed), g1.EndedRefused) + } + if n := len(ofType(*events, EventEndedLargeAgreement)); n != 2 { + t.Fatalf("two large requests for different apps gave %d operator warning(s), want 2", n) + } +} + +// AN HONEST LARGE END IS NOT LOST: it waits a week, then goes — the cap's size per window, the rest in the next one. +func TestD84_TheRestGoesInTheNextWindow(t *testing.T) { + s, _, all := ghostFixture(t) + ctx := context.Background() + big := all[5:] // 15 ids of a big removed app + g0 := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: big}) + if len(g0.EndedAllowed) != 0 { + t.Fatalf("the first window agreed to %d", len(g0.EndedAllowed)) + } + _ = s.CloseWindowFor(ctx, "c1", WindowResult{WindowID: g0.WindowID, CountBefore: 20, CountAfter: 20, Outcome: "nothing"}) + aWeekLater(t, s, g0.WindowID) + g1 := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: big}) + if len(g1.EndedAllowed) != 10 { + t.Fatalf("a week later: %d agreed, want the cap's size 10", len(g1.EndedAllowed)) + } + if err := s.CloseWindowFor(ctx, "c1", WindowResult{WindowID: g1.WindowID, CountBefore: 20, CountAfter: 10, Removed: 10, Outcome: "pruned", RemovedForEndedApps: g1.EndedAllowed}); err != nil { + t.Fatal(err) + } + gone := map[string]bool{} + for _, id := range g1.EndedAllowed { + gone[id] = true + } + var rest []string + for _, id := range big { + if !gone[id] { + rest = append(rest, id) + } + } + // the next day's audit: the hub's own listing now holds the 10 that are left, and raises nothing + left := SnapshotListing{Entries: map[string]time.Time{}} + for _, id := range all { + if !gone[id] { + left.Entries[id] = time.Now().UTC().Add(-40 * 24 * time.Hour) + } + } + if v, _ := s.CheckSnapshotListing("c1", left, time.Now().UTC().Add(time.Minute)); v != 0 { + t.Fatalf("the agreed removals were judged as vanished (%d)", v) + } + g2 := openGranted(t, s, 10, EndedClaim{App: "ghost", SnapshotIDs: rest}) + if len(rest) != 5 || strings.Join(g2.EndedAllowed, ",") != strings.Join(rest, ",") { + t.Fatalf("the next window: agreed %v, want the 5 that waited %v (refused: %+v)", shortIDs(g2.EndedAllowed), shortIDs(rest), g2.EndedRefused) + } +} + +// The bound uses the hub's OWN count of the store, not only the box's word: a box that says „1000 snapshots” gets a +// large cap for its policy, and still no larger ended allowance than half of what the hub's listing holds. +func TestD84_TheAllowanceIsBoundByTheHubsOwnCount(t *testing.T) { + s, _, all := ghostFixture(t) + g0 := openGranted(t, s, 1000, EndedClaim{App: "ghost", SnapshotIDs: all}) + _ = s.CloseWindowFor(context.Background(), "c1", WindowResult{WindowID: g0.WindowID, CountBefore: 1000, CountAfter: 1000, Outcome: "nothing"}) + aWeekLater(t, s, g0.WindowID) + g := openGranted(t, s, 1000, EndedClaim{App: "ghost", SnapshotIDs: all}) + if g.MaxRemove != MaxRemove(1000) { + t.Fatalf("cap = %d", g.MaxRemove) + } + if len(g.EndedAllowed) != MaxRemove(20) { + t.Fatalf("a box that claims 1000 snapshots got %d ended ids agreed; the hub's own listing holds 20 → at most %d", len(g.EndedAllowed), MaxRemove(20)) + } +} + +// A normal end (3 of 20) raises no warning and waits for nothing. +func TestD84_ASmallEndIsQuiet(t *testing.T) { + s, events, all := ghostFixture(t) + g := openGranted(t, s, 20, EndedClaim{App: "ghost", SnapshotIDs: all[:3]}) + if len(g.EndedAllowed) != 3 || len(g.EndedRefused) != 0 { + t.Fatalf("agreed=%d refused=%+v", len(g.EndedAllowed), g.EndedRefused) + } + if e := ofType(*events, EventEndedLargeAgreement); len(e) != 0 { + t.Fatalf("a small end raised the large-agreement warning: %+v", e) + } + for _, typ := range []string{EventEndedLargeAgreement, EventEndedClaimUnseen} { + if !notify.IsOperatorOnly(typ) { + t.Fatalf("%s must be operator-only", typ) + } + } +} diff --git a/hub/internal/offsitekeys/service.go b/hub/internal/offsitekeys/service.go index a192db10..0aa78ba5 100644 --- a/hub/internal/offsitekeys/service.go +++ b/hub/internal/offsitekeys/service.go @@ -378,7 +378,7 @@ func (s *Service) OpenWindowFor(ctx context.Context, customerID string, countBef g := WindowGrant{Granted: true, WindowID: id, NewestAllowed: now.UTC(), MaxRemove: maxRemove} // D84: the ended apps. Decided AFTER the window exists and only for a granted window, so a refused or not-due // window agrees to nothing; a ledger write that fails agrees to nothing either (the box then removes none). - if agreed, byApp, refused := s.agreeEndedFor(customerID, ended); len(agreed) > 0 || len(refused) > 0 { + if agreed, byApp, refused := s.agreeEndedFor(customerID, id, maxRemove, ended); len(agreed) > 0 || len(refused) > 0 { if werr := s.Store.SetOffsiteWindowEnded(id, agreed, byApp, refused); werr != nil { s.logf("[WARN] offsitekeys: window %d: the ended apps' agreement could not be recorded (%v) — nothing agreed", id, werr) g.EndedRefused = []store.EndedRefusal{{Reason: "the hub could not record its agreement"}} diff --git a/hub/internal/store/offsite_ended.go b/hub/internal/store/offsite_ended.go index 2b44df00..c25cddbe 100644 --- a/hub/internal/store/offsite_ended.go +++ b/hub/internal/store/offsite_ended.go @@ -36,6 +36,8 @@ func (s *Store) migrateOffsiteEnded() error { s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_agreed TEXT") s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_refused TEXT") s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_removed TEXT") + // 1 on a row that exists ONLY because the box named the app in an end record (the hub never saw it installed). + s.db.Exec("ALTER TABLE offsite_app_presence ADD COLUMN claim_born INTEGER NOT NULL DEFAULT 0") s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN ended_apps TEXT") // agreed app → its agreed ids (the per-app record) return nil } @@ -46,6 +48,9 @@ type AppPresence struct { Present bool // AbsentSince is the hub's time of the first report without the app (zero while Present). AbsentSince time.Time + // ClaimBorn: the hub never saw this app in the box's app list — the row exists only because the box named the + // app in a pending end record. The operator was warned when it appeared (NoteAppListClaims). + ClaimBorn bool } // A box's lists are the box's own word, so the record is bounded three ways (security review of 2026-11-15): @@ -68,8 +73,18 @@ const ( maxPresenceApps = 300 maxPresenceRows = 400 maxPresenceName = 64 + // maxUnseenClaimsTold: how many never-seen apps of ONE household the operator is warned about (one mail each). + // A box that invents more names is already eight warnings loud; the rest are stored and not mailed. + maxUnseenClaimsTold = 8 ) +// UnseenClaim is an app a box named in a pending end record that the hub never saw in the box's app list — told to +// the operator once, at the moment the row is made. +type UnseenClaim struct { + App string + AbsentSince time.Time // the hub's clock starts here; it could agree EndedAppAfter later +} + // ValidPresenceApp: the shape of an app (stack) name — 1..64 of letters, digits, `.`, `_`, `-`, starting with a // letter or a digit. Anything else a box sends is not stored and can never be agreed. func ValidPresenceApp(app string) bool { @@ -109,8 +124,16 @@ func ReservedPresenceApp(app string) bool { return reservedPresenceApps[strings. // // It writes only when something changes, so a steady box costs one read per report. func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now time.Time) error { + _, err := s.NoteAppListClaims(customerID, deployed, claimed, now) + return err +} + +// NoteAppListClaims is NoteAppList, and returns the never-seen apps this report named for the FIRST time (review of +// 2026-11-15): the caller warns the operator about each, 30 days before the hub could agree to anything for it. An +// app is returned once — when its row is made — and at most maxUnseenClaimsTold such rows per household are told. +func (s *Store) NoteAppListClaims(customerID string, deployed, claimed []string, now time.Time) (unseen []UnseenClaim, err error) { if customerID == "" { - return fmt.Errorf("NoteAppList: empty customer_id") + return nil, fmt.Errorf("NoteAppList: empty customer_id") } if len(deployed) > maxPresenceApps { deployed = deployed[:maxPresenceApps] @@ -120,7 +143,7 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t } cur, err := s.AppPresences(customerID) if err != nil { - return err + return nil, err } stamp := now.UTC().Format(time.RFC3339) in := make(map[string]bool, len(deployed)) @@ -129,8 +152,9 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t isClaimed[app] = true } type change struct { - app string - absent sql.NullString + app string + absent sql.NullString + claimBorn bool } var changes, added []change // changes touch a row that exists; added need a NEW row (rule 3) for _, app := range deployed { @@ -156,7 +180,7 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t } seen[app] = true if _, ok := cur[app]; !ok { - added = append(added, change{app: app, absent: sql.NullString{String: stamp, Valid: true}}) + added = append(added, change{app: app, absent: sql.NullString{String: stamp, Valid: true}, claimBorn: true}) } } // Rule 3: the household's row limit. Room is made only from absent rows this report does not claim, oldest first. @@ -188,27 +212,54 @@ func (s *Store) NoteAppList(customerID string, deployed, claimed []string, now t } } s.notePresenceFull(customerID, refusedNew, len(evict)) + // The never-seen apps to tell: the rows made now for a claim, while the household holds fewer than the limit. + bornBefore := 0 + evicted := make(map[string]bool, len(evict)) + for _, app := range evict { + evicted[app] = true + } + for app, p := range cur { + if p.ClaimBorn && !evicted[app] { + bornBefore++ + } + } + for _, c := range added { + if c.claimBorn && bornBefore+len(unseen) < maxUnseenClaimsTold { + unseen = append(unseen, UnseenClaim{App: c.app, AbsentSince: now.UTC()}) + } + } changes = append(changes, added...) if len(changes) == 0 && len(evict) == 0 { - return nil + return nil, nil } tx, err := s.db.Begin() if err != nil { - return err + return nil, err } defer tx.Rollback() for _, app := range evict { if _, err := tx.Exec(`DELETE FROM offsite_app_presence WHERE customer_id = ? AND app = ?`, customerID, app); err != nil { - return fmt.Errorf("NoteAppList %s/%s: %w", customerID, app, err) + return nil, fmt.Errorf("NoteAppList %s/%s: %w", customerID, app, err) } } for _, c := range changes { - if _, err := tx.Exec(`INSERT INTO offsite_app_presence (customer_id, app, absent_since) VALUES (?, ?, ?) - ON CONFLICT(customer_id, app) DO UPDATE SET absent_since = excluded.absent_since`, customerID, c.app, c.absent); err != nil { - return fmt.Errorf("NoteAppList %s/%s: %w", customerID, c.app, err) + // claim_born is set only on a row made for a claim, and cleared the moment a report lists the app: an app + // the hub has seen installed is an ordinary app from then on. + born := 0 + if c.claimBorn { + born = 1 + } else if p, ok := cur[c.app]; ok && p.ClaimBorn && c.absent.Valid { + born = 1 + } + if _, err := tx.Exec(`INSERT INTO offsite_app_presence (customer_id, app, absent_since, claim_born) VALUES (?, ?, ?, ?) + ON CONFLICT(customer_id, app) DO UPDATE SET absent_since = excluded.absent_since, claim_born = excluded.claim_born`, customerID, c.app, c.absent, born); err != nil { + return nil, fmt.Errorf("NoteAppList %s/%s: %w", customerID, c.app, err) } } - return tx.Commit() + if err := tx.Commit(); err != nil { + return nil, err + } + return unseen, nil } // presenceFull remembers, per household, that the row limit was said in the log — so a box that sends new names in @@ -237,7 +288,7 @@ func (s *Store) notePresenceFull(customerID string, refusedNew, evicted int) { // AppPresences returns the hub's record for one household, by app. func (s *Store) AppPresences(customerID string) (map[string]AppPresence, error) { - rows, err := s.db.Query(`SELECT app, absent_since FROM offsite_app_presence WHERE customer_id = ?`, customerID) + rows, err := s.db.Query(`SELECT app, absent_since, claim_born FROM offsite_app_presence WHERE customer_id = ?`, customerID) if err != nil { return nil, err } @@ -246,10 +297,11 @@ func (s *Store) AppPresences(customerID string) (map[string]AppPresence, error) for rows.Next() { var app string var absent sql.NullString - if err := rows.Scan(&app, &absent); err != nil { + var born sql.NullInt64 + if err := rows.Scan(&app, &absent, &born); err != nil { return nil, err } - p := AppPresence{Present: !absent.Valid || absent.String == ""} + p := AppPresence{Present: !absent.Valid || absent.String == "", ClaimBorn: born.Int64 == 1} if !p.Present { // A date that cannot be read stays the zero time, and the caller REFUSES an absent app with a zero // AbsentSince: an unreadable date is never "absent long ago". @@ -329,3 +381,27 @@ func jsonStrings(v sql.NullString) []string { } return out } + +// LargeEndedRefusalMark is the text a window's refusal carries when the request was for more than half of the store +// (offsitekeys). FirstLargeEndedRefusal finds it on the window rows — the ledger is the memory, so it goes with the +// household and needs no second store. +const LargeEndedRefusalMark = "more than half of the store" + +// FirstLargeEndedRefusal returns when the household's EARLIEST window since `since` refused a request for more than +// half of the store FOR THE SAME APPS (`needle` is the refusal's own text for them; zero time: none). A store that +// cannot answer reads as none — and then the request waits again. +func (s *Store) FirstLargeEndedRefusal(customerID, needle string, since time.Time) time.Time { + var v sql.NullString + err := s.db.QueryRow(`SELECT MIN(opened_at) FROM offsite_windows WHERE customer_id = ? AND opened_at >= ? AND instr(COALESCE(ended_refused, ''), ?) > 0`, + customerID, since.UTC().Format("2006-01-02 15:04:05"), needle).Scan(&v) + if err != nil || !v.Valid || v.String == "" { + return time.Time{} + } + return parseSQLiteTime(v.String) +} + +// BackdateOffsiteWindowForTest moves a window's opened_at into the past. TEST-ONLY. +func (s *Store) BackdateOffsiteWindowForTest(id int64, by time.Duration) error { + _, err := s.db.Exec(`UPDATE offsite_windows SET opened_at = ? WHERE id = ?`, time.Now().UTC().Add(-by).Format("2006-01-02 15:04:05"), id) + return err +}