diff --git a/documentation/architecture/05-hub-architecture.md b/documentation/architecture/05-hub-architecture.md index a27bad90..0e31fc25 100644 --- a/documentation/architecture/05-hub-architecture.md +++ b/documentation/architecture/05-hub-architecture.md @@ -437,6 +437,32 @@ repository's fake provider (`hetznerapi.Fake`); 9 mutations seen red, one of the added (`audits/night-build-2026-11-14/B/redproof-hub-offsite-record.txt`). +### 14.2 What a deleted box or household leaves in the hub [FACT — built, not released: hub `main` after v0.181.0, 2026-11-14; D73, `09` §3 decision 293, R-1376] + +The endings list found thirteen hub tables with no delete statement anywhere. Seven of them hold rows that name a box +or a household. The ruling: a delete removes them, with one exception — the delete journal. + +| Table | Named by | Goes when | How | +|---|---|---|---| +| `os_host_settings`, `os_reports`, `os_kernel_notices` | the box | the box is deleted | `store.DeleteHost`, in its one transaction | +| the hub's own alarm marks that name the box (`hub_settings`, `os_alarm::…`) | the box | the box is deleted | the same transaction; matched on the whole id between colons, so a box whose id only starts with this id keeps its marks | +| `offsite_keys`, `offsite_windows`, `offsite_abandon_requests` | the household | the household is deleted (the purge leg, after the outside legs) | `store.PurgeCustomerResidue`; counted in the delete dialog's residue | +| `operator_actions` (the operator's own action log) | the household | **one year after the deletion**, with `events` and `notification_log` | `store.PruneDeletedCustomerAudit`. *Decided by CC unattended — the operator may reverse:* it is an audit record of the same kind as the other two, and the one-year rule (decision 181) is the design's rule for those; the other option, at once, loses „who did what” on the day of the deletion | +| **`host_deletions`, `customer_resets`** — the delete journal | both | **never** | the ruling's one exception: an id and timestamps, no personal data. The privacy text must say so (proposed, not published) | + +**Rows left by boxes and households deleted before this release** (Peti's box, the test boxes of 2026-11): the daily +prune runs `store.PurgeOrphanRows` — the six tables of the first and third rows, for a box with no `hosts` row and for +a household with neither a config row nor a box. A household in the middle of its delete (boxes gone, config still +there) and a „ghost” that still has a box lose nothing. It finds something once. + +**Not in this:** the six tables that name no box or household (`api_tokens`, `os_candidates`, `os_releases`, +`r172_probe` and the two journals). The update reports of a LIVE box still grow with no end: an age limit there needs +its own design, because the kernel lane reads a box's old reports to know which kernel it must never offer again. + +**Not proven:** on a deployed hub; no hub database was read. Tests on the real store with look-alike ids +(`TestD73_*`), 16 mutations seen red (`audits/night-build-2026-11-14/B/redproof-hub-d73.txt`). + + ## 15. Customer e-mails — what the hub writes to a household [hub v0.118.0, R-558] **The section this document did not have.** The hub composes every sentence a household reads before diff --git a/documentation/audits/night-build-2026-11-14/B/redproof-hub-d73.txt b/documentation/audits/night-build-2026-11-14/B/redproof-hub-d73.txt new file mode 100644 index 00000000..6704df23 --- /dev/null +++ b/documentation/audits/night-build-2026-11-14/B/redproof-hub-d73.txt @@ -0,0 +1,82 @@ +== MUTATION: a deleted box keeps its update reports (the old behaviour) + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:92: os_reports: 2 row(s) left for the deleted box (want 0) +== MUTATION: a deleted box keeps its update settings + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:92: os_host_settings: 1 row(s) left for the deleted box (want 0) +== MUTATION: a deleted box keeps its kernel notices + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:92: os_kernel_notices: 1 row(s) left for the deleted box (want 0) +== MUTATION: the alarm marks are matched by LIKE on the box's id (a look-alike box loses its marks) + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:92: alarm marks: 0 row(s) left for the deleted box (want 10) + d73_gone_rows_test.go:107: another box (box-a1b) has 0 alarm mark(s) left, want its 5 + d73_gone_rows_test.go:107: another box (x-box-a1) has 0 alarm mark(s) left, want its 5 +== MUTATION: the alarm marks of a deleted box stay + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:92: alarm marks: 15 row(s) left for the deleted box (want 10) +== MUTATION: the box delete also removes the HOUSEHOLD's off-site key record + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_DeleteHostRemovesTheBoxesRows (0.05s) + d73_gone_rows_test.go:97: offsite_keys: 1 → 0 — a box delete must leave the HOUSEHOLD's rows (the household still exists) + +6 mutations, 0 not a red-proof +== MUTATION: a deleted household keeps its off-site key record + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeRemovesTheHouseholdsRows_JournalStays (0.04s) + d73_gone_rows_test.go:137: the preview's residue does not count the three tables: &{Reports:0 AppTelemetry:0 AppLogTails:0 LogTailRequests:0 NotificationPrefs:0 SelfBindTokens:0 ApplianceRegistrations:0 AppLogIssueRefs:0 OffsiteKe +== MUTATION: a deleted household keeps its abandon requests + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeRemovesTheHouseholdsRows_JournalStays (0.04s) + d73_gone_rows_test.go:137: the preview's residue does not count the three tables: &{Reports:0 AppTelemetry:0 AppLogTails:0 LogTailRequests:0 NotificationPrefs:0 SelfBindTokens:0 ApplianceRegistrations:0 AppLogIssueRefs:0 OffsiteKe +== MUTATION: the three tables are purged and not counted in the preview + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeRemovesTheHouseholdsRows_JournalStays (0.04s) + d73_gone_rows_test.go:137: the preview's residue does not count the three tables: &{Reports:0 AppTelemetry:0 AppLogTails:0 LogTailRequests:0 NotificationPrefs:0 SelfBindTokens:0 ApplianceRegistrations:0 AppLogIssueRefs:0 OffsiteKe + +3 mutations, 0 not a red-proof +== MUTATION: the operator's action log of a deleted household is never deleted + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeRemovesTheHouseholdsRows_JournalStays (0.04s) + d73_gone_rows_test.go:171: operator_actions a year after the deletion: 1, want 0 +== MUTATION: the one-year prune also empties the delete journal + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeRemovesTheHouseholdsRows_JournalStays (0.05s) + d73_gone_rows_test.go:178: journal host_deletions after the one-year prune: 0 row(s), want 1 + +2 mutations, 0 not a red-proof +== MUTATION: the sweep removes a LIVE box's reports (no owner check) + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeOrphanRows_OnlyTheGone (0.05s) + d73_gone_rows_test.go:214: live / os_reports: 0 row(s), want 2 + d73_gone_rows_test.go:214: ghost / os_reports: 0 row(s), want 2 + d73_gone_rows_test.go:219: the sweep reported map[offsite_abandon_requests:1 offsite_keys:1 offsite_windows:1 os_host_settings:2 os_kernel_notices:2 os_reports:8] (total 15), want 11 rows: 8 box rows of mid and gone, 3 household r +== MUTATION: the sweep removes the off-site rows of a household in the middle of its delete + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeOrphanRows_OnlyTheGone (0.04s) + d73_gone_rows_test.go:214: mid / offsite_windows: 0 row(s), want 1 + d73_gone_rows_test.go:219: the sweep reported map[offsite_abandon_requests:1 offsite_keys:1 offsite_windows:2 os_host_settings:2 os_kernel_notices:2 os_reports:4] (total 12), want 11 rows: 8 box rows of mid and gone, 3 household r +== MUTATION: the sweep removes the off-site rows of a household that still has a box + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeOrphanRows_OnlyTheGone (0.05s) + d73_gone_rows_test.go:214: ghost / offsite_keys: 0 row(s), want 1 + d73_gone_rows_test.go:219: the sweep reported map[offsite_abandon_requests:1 offsite_keys:2 offsite_windows:1 os_host_settings:2 os_kernel_notices:2 os_reports:4] (total 12), want 11 rows: 8 box rows of mid and gone, 3 household r +== MUTATION: the sweep leaves the kernel notices of a gone box + RED on the mutant, GREEN on the real file + --- FAIL: TestD73_PurgeOrphanRows_OnlyTheGone (0.05s) + d73_gone_rows_test.go:214: mid / os_kernel_notices: 1 row(s), want 0 + d73_gone_rows_test.go:214: gone / os_kernel_notices: 1 row(s), want 0 + d73_gone_rows_test.go:219: the sweep reported map[offsite_abandon_requests:1 offsite_keys:1 offsite_windows:1 os_host_settings:2 os_reports:4] (total 9), want 11 rows: 8 box rows of mid and gone, 3 household rows of gone + +4 mutations, 0 not a red-proof +== MUTATION: the daily prune does not call the sweep + RED on the mutant, GREEN on the real file + --- FAIL: TestR1383_MainWiresTheSilentFailureChecks (0.00s) + r1383_wiring_test.go:74: main.go: orphan sweep in the daily prune — found 0 time(s), want 1 + +1 mutations, 0 not a red-proof diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index f06e3af3..5d7c46a5 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -355,7 +355,7 @@ stopping line that lies. | **R-1345** | Hub & operator | P4 | **An access token's scopes cannot be changed after it is made — widening or narrowing one means a new token (and a new secret).** The operator, 2026-10-29: to give CC `hosts:read` + `hosts:write` for the live proofs he had to make `CC_HOSTS` (now in `sec`) beside `cc-releases-2`. Wanted: edit a token's scopes in the session (an audit line per change), so a scope can also be taken away after a session without retiring the secret. | OPEN | — | design + build the scope edit (session only), red-first test that an unticked scope is refused at once | CC | | **R-1353** | Hub & operator | P4 | **Statements still over 1 s, named by the new slow-statement line (hub 0.178.0) in its first hour:** a background deadline/restore-test check reading days of full host-report JSON (`GetHostReportsSince`, `store.go`; 1.5–9.2 s cold, from `monitor/deadline.go` and `monitor/restoretest.go` — no page waits on it); the start-up prune of `events` and `app_log_issues` (1.1 s and 1.8 s; no index on `created_at` / `last_seen`); the customer page's newest-report read (`GetCustomer`, 1.3 s cold in the pod probe). With the 1 GiB limit most of this should come from the cache once warm. Evidence: the hub log 2026-10-30 09:22–10:05 UTC; `audits/release-2026-10-30/p2-r1346/`. | OPEN **2026-11-02 read (`audits/security-2026-11-02/part4/hub-slow-lines-2026-11-01-02.txt`; the log covers only the 31 h since DooPlex restarted on 2026-11-01 16:11 CET, the lines from 10-30 are gone with the old pod):** 35 lines, 15 statements. What repeats: `VACUUM INTO` 5× (avg 110 s, the 6-hourly snapshot, expected); `GetHostReportsSince` 9× (avg 7.9 s, max 26.6 s); `GetCustomer`'s newest report 3× (1.1 s). Once, at the cold start: the prune of `events` (10.4 s) and `app_log_issues` (21 s, again 04:30), the OS-release and host lists (2–22 s). No fix before the full week (11-06). **2026-11-10 read (the hub's start after a deploy, `audits/release-space-fixes-2026-11-10/hub/10-watch.txt`):** 11 slow lines in the first 2 minutes of the new pod, the longest 19.1 s (`GetHostReportsSince`), then the OS-report reads (3–4 s) and the two prunes (2.7 s, 4.2 s); none in the 28 minutes after. The 13:02 UTC snapshot's `VACUUM INTO` took 127.8 s. The same statements as the 11-02 read; no new one. | — | read a week of slow-statement lines; index or narrow what still repeats (the history read needs only a few fields, not the whole JSON) | CC | | **R-1375** | Hub & operator | P2 | **The outside part of three endings is done by hand or not at all — the worst: an off-site account can stay at the provider with no record in the hub.** Found 2026-11-10 (the endings list, `documentation/audits/endings-list-2026-11-10/TABLE.md` endings EH06, EH01, EA21 and the ending "a tester ends"; read in source and documents; no account was called). (1) **Unticking the off-site tier** of a household takes nothing down and leaves no reminder (`hub/internal/web/configs.go:1737-1743`). A later RESET takes the account down only when the tier is ticked (`hub/internal/web/customer_reset.go:203-245`), so the sub-account and its data stay, are paid for, and the hub holds no record that they exist. (2) **Switching the second off-site tier off** takes nothing down: storage entry, token and copies stay; the source calls the teardown 'a deliberate future op' (agent `internal/pbsdr/manager.go:274`). (3) **A tester ends**: no handler and no runbook; done by hand once (2026-09-25); no list says what to collect. **Members (answers marked N):** EH06-4, EH01-4, EA21-4, and 'a tester ends' in the table's head. Related: R-810 (a household leaves), R-32. | NARROWED — part 1 (D71) is built, not released. **2026-11-14 (night build):** the hub keeps the record of an off-site account after the tier is unticked; RESET and delete take the account down when a record exists; an untick alone takes nothing down (pinned by a test); the config form says that the account still exists (`05` §14.1). The runbook „a household or a tester leaves” is written (D74, `runbooks/household-leaves.md`). **For the operator, once, after the release:** at the provider's page read each sub-account's label `felhom-customer=`; for a household whose tier is unticked in the hub, tick it again and save — the account's record comes back. LEFT: part (2) has no fault (the second tier keeps its record, read in source and its test); part (3) is the runbook; a household mail at leaving stays R-810. OPEN — filed 2026-11-10 (endings list) **2026-11-10 15:19 — RULED (D71 = yes, D74 = yes; `09` §3 decisions 291, 294).** Not built. | — | Hub (D71): keep the account record after an untick; RESET and delete take the account down, ticked or not. Docs (D74): the runbook 'a household or a tester leaves', from the five code doors, in a later documents-only session. Parts (2) and (3) of this row have no ruling of their own. | CC | -| **R-1376** | Hub & operator | P3 | **Records of a thing that is gone stay in six places.** Found 2026-11-10 (the endings list, `documentation/audits/endings-list-2026-11-10/TABLE.md` endings EH00, EH02, EH21, EA00, EA11, EA27; read in source, `documentation/audits/endings-list-2026-11-10/evidence/hub-delete-statements.txt`; no hub database was read). (1) **Hub:** thirteen tables have no delete statement anywhere. Deleting a box leaves its update settings, its update reports and its kernel notices; deleting a household leaves its off-site key rows, off-site window rows and abandon requests. Most are small; the update reports of each box grow with time. (2) **Host uninstall** leaves the mount units and host mounts of household drives (said when R-179 closed; no open row carried it). (3) **A drive wiped by a signed job** keeps a record and a mount unit that name the old filesystem (agent `internal/signedjobs/wipe.go:61-110`). (4) **The token of a destroyed guest** stays valid until a new guest takes its number: the token store has no remove (agent `internal/localapi/tokenstore.go:127-210`). **Members (answers marked N):** EH00-4, EH02-4, EH21-4, EA00-4, EA11-4, EA27-4. | OPEN — filed 2026-11-10 (endings list) **2026-11-10 15:19 — RULED (D73 = yes, with one exception; `09` §3 decision 293).** Not built. | — | Hub (D73): add the tables with no delete to the two purges and to the daily clean-up; the delete journal (`host_deletions`, `customer_resets`) stays on purpose, and the privacy text says so. Agent: remove a guest's token when the guest is destroyed. Parts (2) and (3) have no ruling of their own. | CC | +| **R-1376** | Hub & operator | P3 | **Records of a thing that is gone stay in six places.** Found 2026-11-10 (the endings list, `documentation/audits/endings-list-2026-11-10/TABLE.md` endings EH00, EH02, EH21, EA00, EA11, EA27; read in source, `documentation/audits/endings-list-2026-11-10/evidence/hub-delete-statements.txt`; no hub database was read). (1) **Hub:** thirteen tables have no delete statement anywhere. Deleting a box leaves its update settings, its update reports and its kernel notices; deleting a household leaves its off-site key rows, off-site window rows and abandon requests. Most are small; the update reports of each box grow with time. (2) **Host uninstall** leaves the mount units and host mounts of household drives (said when R-179 closed; no open row carried it). (3) **A drive wiped by a signed job** keeps a record and a mount unit that name the old filesystem (agent `internal/signedjobs/wipe.go:61-110`). (4) **The token of a destroyed guest** stays valid until a new guest takes its number: the token store has no remove (agent `internal/localapi/tokenstore.go:127-210`). **Members (answers marked N):** EH00-4, EH02-4, EH21-4, EA00-4, EA11-4, EA27-4. | NARROWED — part 1 (the hub, D73) is built, not released. **2026-11-14 (night build):** deleting a box removes its update settings, update reports, kernel notices and the hub's alarm marks that name it; deleting a household removes its off-site key, window and abandon rows; the operator's action log of a deleted household goes one year after the deletion with the other audit rows (decided by CC unattended — the operator may reverse); the delete journal stays; a daily sweep removes what earlier deletes left (`05` §14.2). The privacy sentence is proposed in the report, not published. LEFT: the agent part (a destroyed guest's token), parts (2) and (3), and an age limit for a LIVE box's update reports (needs a design: the kernel lane reads old reports). OPEN — filed 2026-11-10 (endings list) **2026-11-10 15:19 — RULED (D73 = yes, with one exception; `09` §3 decision 293).** Not built. | — | Hub (D73): add the tables with no delete to the two purges and to the daily clean-up; the delete journal (`host_deletions`, `customer_resets`) stays on purpose, and the privacy text says so. Agent: remove a guest's token when the guest is destroyed. Parts (2) and (3) have no ruling of their own. | CC | | **R-1386** | Hub & operator | P3 | **Timed jobs with no time limit and no measured duration (25 answers).** Found 2026-11-11 (the Clock list, class W4, `documentation/audits/clock-list-2026-11-11/TABLE.md`; read in source; the hub was not measured, by ruling). The hub's 60-second sweep runs its checks one after the other with no limit for each: one slow check delays all the others, the liveness check too (CH19-3, CH19-8, CH01-3, CH08-3, CH08-8, CH10-3, CH13-8, CH18-3, CH06-8, CH02-3); the 07:10 audit has 10 minutes, measured with three boxes only (CH11-3, CH11-8). On the box: the interval jobs have no limit of their own (CC11-3, CC12-3, CC13-3, CC14-3, CC15-3, CC16-3, CC20-3, CA06-3, CA10-3, CA10-8, CA13-8, CA15-3, CA19-3). Every measured duration is from a small test store. | OPEN — filed 2026-11-11 (clock list) | — | Hub: log each sweep step's duration and alarm when one sweep takes longer than its interval; then read a week of numbers and set limits. Box: the same for the controller's interval jobs. | CC | | **R-1393** | Hub & operator | P4 | **For about 8 minutes after a hub restart its database statements take 1 to 32 seconds; the System page took 12.7 s and a box's enrolment 11.4 s.** Measured 2026-11-14 at the 0.181.0 deploy (`audits/release-1-dates-2026-11-14/hub/05-watch.txt`): 24 `slow statement` lines from 09:58 to 10:06 UTC, none in the 20 minutes after. The database file is 678 MB; the slow ones read `host_reports`, `os_reports` and `events` (whole report JSON per row). One 32-second statement ran while CC copied the database out of the pod for a read (CC's own load). No report was lost that CC could see; not measured: whether a box's report timed out in that window. | OPEN — filed 2026-11-14 | — | measure a restart with no copy running; then decide: an index or a shorter keep on `host_reports`/`os_reports`, or accept | CC | diff --git a/documentation/runbooks/household-leaves.md b/documentation/runbooks/household-leaves.md index 8f85ed81..acf933b5 100644 --- a/documentation/runbooks/household-leaves.md +++ b/documentation/runbooks/household-leaves.md @@ -71,8 +71,8 @@ The hub then runs, in this order: the journal entry → every box deleted (door measured (`runbooks/ep0-datastore-copy.md`). - The purge of the hub's rows: PROVEN once (482 reports, 1,036 telemetry rows, 1 log tail gone; journal #20). **Known gap, R-1376:** rows in 13 tables stay (the box's update settings and reports, its kernel notices, the - off-site key and window rows, the abandon requests). Ruled to be removed (D73, `09` §3 decision 293); read the - row for whether that is released — until then they stay. + off-site key and window rows, the abandon requests). Built to be removed, not released (hub `main`, 2026-11-14; + D73, `05` §14.2) — until that release they stay. ### Step 5 — if a leg fails: press again. The dialog offers Resume. A leg that is done is not done again. Nothing is half forgotten: the journal names the leg. diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index b509aabf..f9c17e81 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -3,6 +3,12 @@ - `bundlefiles`: the line „bundle X holds the same files as the vouched Y” is logged once per pair (INFO), not at every view of the System page (it was 6 DEBUG lines per view, seen at the 0.181.0 deploy). No behaviour change. - **Release 3 („the hub tells you”), built 2026-11-14 night, NOT deployed. Every new alarm is operator-only.** +- **A deleted box or household leaves no rows in the tables that had no delete (R-1376 part 1, D73, `09` §3 decision + 293; `05` §14.2).** `DeleteHost` also removes the box's `os_host_settings`, `os_reports`, `os_kernel_notices` and the + hub's alarm marks that name it; `PurgeCustomerResidue` also removes the household's `offsite_keys`, + `offsite_windows`, `offsite_abandon_requests` (and the delete dialog counts them); `operator_actions` goes with the + audit rows one year after the deletion. **The delete journal stays** (`host_deletions`, `customer_resets`). + `PurgeOrphanRows` in the daily prune removes what earlier deletes left. Tests `TestD73_*`; 16 mutations seen red. - **Two end dates warn 14 days before (R-1379, D77, `09` §3 decision 298; `08` §12).** `monitor.EndDateChecker`, hourly on the sweep: `operator_token_ends_soon` for an operator access token that is not revoked, and `web_cert_ends_soon` for a web certificate a box reports in `web_certs` (controllers after 0.352.1). Told once at 14 days and once more, diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 6eb7a38b..df40c267 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -1214,6 +1214,13 @@ func pruneAll(s *store.Store, maxDays int, logger *log.Logger) { } else if n > 0 { logger.Printf("[INFO] Pruned %d stale app issues", n) } + // D73 (`09` §3 decision 293, R-1376): the rows a box or household deleted BEFORE this release left in the six + // tables that had no delete. A delete now leaves none, so this finds something only once. + if orphans, err := s.PurgeOrphanRows(); err != nil { + logger.Printf("[ERROR] Purge rows of deleted boxes and households: %v", err) + } else if orphans.Total() > 0 { + logger.Printf("[INFO] Purged %d row(s) of boxes and households that no longer exist: %v", orphans.Total(), map[string]int64(orphans)) + } // R-901 (`09` §3 decision 181): a deleted customer's audit rows go 1 year after the deletion. if ev, nl, err := s.PruneDeletedCustomerAudit(time.Now().UTC(), store.DeletedCustomerAuditKeep); err != nil { logger.Printf("[ERROR] Prune deleted customers' audit rows: %v", err) diff --git a/hub/cmd/hub/r1383_wiring_test.go b/hub/cmd/hub/r1383_wiring_test.go index b7bab31a..85b04d64 100644 --- a/hub/cmd/hub/r1383_wiring_test.go +++ b/hub/cmd/hub/r1383_wiring_test.go @@ -60,6 +60,8 @@ func TestR1383_MainWiresTheSilentFailureChecks(t *testing.T) { seen["lost-mail scan"]++ case recv == "signedJobWaitChecker" && name == "Check": seen["signed-job sweep"]++ + case name == "PurgeOrphanRows" && fn.Name.Name == "pruneAll": + seen["orphan sweep in the daily prune"]++ // D73, R-1376 case recv == "endDateChecker" && name == "Check": seen["end-date sweep"]++ // R-1379 } @@ -67,7 +69,7 @@ func TestR1383_MainWiresTheSilentFailureChecks(t *testing.T) { return true }) } - for _, k := range []string{"ledger built", `watched "deadline-check"`, `watched "offsite-key-audit"`, "lost-mail scan", "signed-job sweep", "end-date sweep"} { + for _, k := range []string{"ledger built", `watched "deadline-check"`, `watched "offsite-key-audit"`, "lost-mail scan", "signed-job sweep", "end-date sweep", "orphan sweep in the daily prune"} { if seen[k] != 1 { t.Errorf("main.go: %s — found %d time(s), want 1", k, seen[k]) } diff --git a/hub/internal/store/customer_delete.go b/hub/internal/store/customer_delete.go index 8c03d95b..fb72d4eb 100644 --- a/hub/internal/store/customer_delete.go +++ b/hub/internal/store/customer_delete.go @@ -24,6 +24,13 @@ import "fmt" // What deliberately SURVIVES (unchanged from every other tier): `events` (the audit stream), // `notification_log` (the send-attempt audit), `host_deletions` and `customer_resets` (F-14 // provenance). The audit trail outlives every lifecycle tier — that rule is not relaxed here. +// +// D73 (`09` §3 decision 293, R-1376): „deleting a household or a box deletes its rows in the tables that have no +// delete today” — with ONE exception, the delete journal (`host_deletions`, `customer_resets`), which stays for ever +// and which the privacy text must name. So: the three off-site tables go with this purge; the box's three update +// tables go with DeleteHost; and `operator_actions` — the operator's own action log, an audit record like `events` +// — goes with the audit rows, one year after the deletion (PruneDeletedCustomerAudit; decided by CC unattended, +// the operator may reverse). Rows left by households and boxes deleted BEFORE this: PurgeOrphanRows, daily. // CustomerResidue counts what a customer has left behind OUTSIDE the config row: the report-derived // state that keeps a deleted customer visible and alerting, plus the credential-bearing bindings. @@ -39,6 +46,11 @@ type CustomerResidue struct { // AppLogIssueRefs (R-244, v0.147.0): shared app_log_issues rows naming the customer. De-referenced // (never blindly deleted) by the purge — see app_log_residue.go. AppLogIssueRefs int + // D73 (`09` §3 decision 293, R-1376): the three household-keyed tables that had no delete statement anywhere — + // the off-site key record, the off-site clean-up windows and the abandon requests. They outlived the household. + OffsiteKeys int + OffsiteWindows int + OffsiteAbandonRequests int } // Total is the single number the "is there anything left at all?" decision hangs on. @@ -47,7 +59,8 @@ func (r *CustomerResidue) Total() int { return 0 } return r.Reports + r.AppTelemetry + r.AppLogTails + r.LogTailRequests + - r.NotificationPrefs + r.SelfBindTokens + r.ApplianceRegistrations + r.AppLogIssueRefs + r.NotificationPrefs + r.SelfBindTokens + r.ApplianceRegistrations + r.AppLogIssueRefs + + r.OffsiteKeys + r.OffsiteWindows + r.OffsiteAbandonRequests } // residueQueries is the ONE list both the counter and the purge walk, so a table can never be @@ -63,6 +76,9 @@ var residueQueries = []struct { {"customer_notifications", func(r *CustomerResidue) *int { return &r.NotificationPrefs }}, {"selfbind_tokens", func(r *CustomerResidue) *int { return &r.SelfBindTokens }}, {"appliance_registrations", func(r *CustomerResidue) *int { return &r.ApplianceRegistrations }}, + {"offsite_keys", func(r *CustomerResidue) *int { return &r.OffsiteKeys }}, + {"offsite_windows", func(r *CustomerResidue) *int { return &r.OffsiteWindows }}, + {"offsite_abandon_requests", func(r *CustomerResidue) *int { return &r.OffsiteAbandonRequests }}, } // CustomerResidue counts the customer's report-derived + binding residue. Read-only. diff --git a/hub/internal/store/d73_gone_rows_test.go b/hub/internal/store/d73_gone_rows_test.go new file mode 100644 index 00000000..c43f87a1 --- /dev/null +++ b/hub/internal/store/d73_gone_rows_test.go @@ -0,0 +1,225 @@ +package store + +import ( + "testing" + "time" +) + +// d73Seed gives a box rows in the three box-keyed tables that had no delete, and its household rows in the three +// household-keyed ones — plus the hub's own alarm marks that name the box. +func d73Seed(t *testing.T, s *Store, hostID, customerID string) { + t.Helper() + exec := func(q string, a ...any) { + t.Helper() + if _, err := s.db.Exec(q, a...); err != nil { + t.Fatalf("seed %q: %v", q, err) + } + } + exec(`INSERT INTO os_host_settings (host_id, ring, enabled) VALUES (?, 0, 1)`, hostID) + exec(`INSERT INTO os_reports (host_id, layer, outcome, report_json) VALUES (?, 'host', 'nothing', '{}')`, hostID) + exec(`INSERT INTO os_reports (host_id, layer, outcome, report_json) VALUES (?, 'kernel', 'applied', '{}')`, hostID) + exec(`INSERT INTO os_kernel_notices (host_id, kver, sent_at) VALUES (?, '7.0.14-22-pve', datetime('now'))`, hostID) + exec(`INSERT INTO offsite_keys (customer_id, fingerprint) VALUES (?, 'SHA256:x')`, customerID) + exec(`INSERT INTO offsite_windows (customer_id, closes_by) VALUES (?, datetime('now'))`, customerID) + exec(`INSERT INTO offsite_abandon_requests (customer_id, path, due_at) VALUES (?, '.orphaned-1', datetime('now'))`, customerID) + exec(`INSERT INTO operator_actions (customer_id, host_id, action, requested_at) VALUES (?, ?, 'rerun_backup', datetime('now'))`, customerID, hostID) + for _, k := range []string{"box_clock:" + hostID + ":ahead", "pve_trust:" + hostID + ":ca_copy_does_not_verify", + "guest_power:" + hostID + ":9201", "signed_job_wait:" + hostID + ":job1", "wg_stale:" + hostID} { + if err := s.SetOSAlarmRaised(k, time.Now()); err != nil { + t.Fatal(err) + } + } +} + +func d73Count(t *testing.T, s *Store, q string, args ...any) int { + t.Helper() + var n int + if err := s.db.QueryRow(q, args...).Scan(&n); err != nil { + t.Fatalf("count %q: %v", q, err) + } + return n +} + +func d73Counts(t *testing.T, s *Store, hostID, customerID string) map[string]int { + t.Helper() + return map[string]int{ + "os_host_settings": d73Count(t, s, `SELECT COUNT(*) FROM os_host_settings WHERE host_id = ?`, hostID), + "os_reports": d73Count(t, s, `SELECT COUNT(*) FROM os_reports WHERE host_id = ?`, hostID), + "os_kernel_notices": d73Count(t, s, `SELECT COUNT(*) FROM os_kernel_notices WHERE host_id = ?`, hostID), + "alarm marks": d73Count(t, s, `SELECT COUNT(*) FROM hub_settings WHERE key LIKE 'os_alarm:%' AND instr(key, ?) > 0`, hostID), + "offsite_keys": d73Count(t, s, `SELECT COUNT(*) FROM offsite_keys WHERE customer_id = ?`, customerID), + "offsite_windows": d73Count(t, s, `SELECT COUNT(*) FROM offsite_windows WHERE customer_id = ?`, customerID), + "offsite_abandon_requests": d73Count(t, s, `SELECT COUNT(*) FROM offsite_abandon_requests WHERE customer_id = ?`, customerID), + "operator_actions": d73Count(t, s, `SELECT COUNT(*) FROM operator_actions WHERE customer_id = ?`, customerID), + } +} + +var d73BoxTables = []string{"os_host_settings", "os_reports", "os_kernel_notices", "alarm marks"} +var d73HouseholdTables = []string{"offsite_keys", "offsite_windows", "offsite_abandon_requests"} + +// D73 (`09` §3 decision 293, R-1376): deleting a BOX deletes its rows in the three box-keyed tables that had no +// delete statement, and the hub's alarm marks that name it. The household's rows stay (the household is still +// there), the other box is untouched — also one whose id only STARTS with this box's id — and the delete journal has +// its row. COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-d73.txt). +func TestD73_DeleteHostRemovesTheBoxesRows(t *testing.T) { + s := newTestStore(t) + for _, h := range []struct{ host, cust string }{{"box-a1", "cust-a"}, {"box-a1b", "cust-b"}, {"x-box-a1", "cust-c"}} { + if err := s.UpsertHost(&Host{HostID: h.host, CustomerID: h.cust, APIKey: "k-" + h.host}); err != nil { + t.Fatal(err) + } + d73Seed(t, s, h.host, h.cust) + } + before := d73Counts(t, s, "box-a1", "cust-a") + for table, n := range before { + if n == 0 { + t.Fatalf("fixture gap: %s has no row", table) + } + } + otherB, otherC := d73Counts(t, s, "box-a1b", "cust-b"), d73Counts(t, s, "x-box-a1", "cust-c") + if err := s.DeleteHost("box-a1", true); err != nil { + t.Fatal(err) + } + after := d73Counts(t, s, "box-a1", "cust-a") + // "alarm marks" counts by a substring, so the two look-alike boxes' marks (box-a1b, x-box-a1) are in it: those + // must be all that is left. + lookAlikes := 2 * 5 + for _, table := range d73BoxTables { + want := 0 + if table == "alarm marks" { + want = lookAlikes + } + if after[table] != want { + t.Errorf("%s: %d row(s) left for the deleted box (want %d)", table, after[table], want) + } + } + for _, table := range append(d73HouseholdTables, "operator_actions") { + if after[table] != before[table] { + t.Errorf("%s: %d → %d — a box delete must leave the HOUSEHOLD's rows (the household still exists)", table, before[table], after[table]) + } + } + for name, pair := range map[string][2]map[string]int{"box-a1b": {otherB, d73Counts(t, s, "box-a1b", "cust-b")}, "x-box-a1": {otherC, d73Counts(t, s, "x-box-a1", "cust-c")}} { + for _, table := range []string{"os_host_settings", "os_reports", "os_kernel_notices", "offsite_keys", "operator_actions"} { + if pair[0][table] != pair[1][table] { + t.Errorf("another box (%s) lost rows in %s: %d → %d", name, table, pair[0][table], pair[1][table]) + } + } + if n := d73Count(t, s, `SELECT COUNT(*) FROM hub_settings WHERE key LIKE 'os_alarm:%' AND (instr(key, ':'||?||':') > 0 OR key LIKE '%:'||?)`, name, name); n != 5 { + t.Errorf("another box (%s) has %d alarm mark(s) left, want its 5", name, n) + } + } + if n := d73Count(t, s, `SELECT COUNT(*) FROM host_deletions WHERE host_id = 'box-a1'`); n != 1 { + t.Errorf("the delete journal has %d row(s) for the box, want 1 — the journal STAYS", n) + } +} + +// Deleting a HOUSEHOLD (the purge leg) deletes its rows in the three household-keyed tables, counts them in the +// preview's residue, and leaves the two journals alone. The operator's action log is an audit record: it stays at the +// purge and goes with the audit rows one year after the deletion. +func TestD73_PurgeRemovesTheHouseholdsRows_JournalStays(t *testing.T) { + s := newTestStore(t) + for _, c := range []string{"cust-a", "cust-b"} { + if err := s.SaveCustomerConfig(&CustomerConfig{CustomerID: c, APIKey: "k-" + c, RetrievalPassword: "p"}); err != nil { + t.Fatal(err) + } + d73Seed(t, s, "box-"+c, c) + } + id, err := s.StartCustomerReset("cust-a", false) + if err != nil { + t.Fatal(err) + } + _ = s.UpdateResetLeg(id, "customer_delete", "ok") + _ = s.FinishCustomerReset(id) + if _, err := s.db.Exec(`INSERT INTO host_deletions (host_id, customer_id, escrow_acked) VALUES ('box-cust-a', 'cust-a', 0)`); err != nil { + t.Fatal(err) + } + res, err := s.CustomerResidue("cust-a") + if err != nil || res.OffsiteKeys != 1 || res.OffsiteWindows != 1 || res.OffsiteAbandonRequests != 1 || res.Total() < 3 { + t.Fatalf("the preview's residue does not count the three tables: %+v (%v)", res, err) + } + if err := s.PurgeCustomerResidue("cust-a"); err != nil { + t.Fatal(err) + } + after, other := d73Counts(t, s, "box-cust-a", "cust-a"), d73Counts(t, s, "box-cust-b", "cust-b") + for _, table := range d73HouseholdTables { + if after[table] != 0 { + t.Errorf("%s: %d row(s) outlived the household", table, after[table]) + } + if other[table] != 1 { + t.Errorf("%s: the OTHER household has %d row(s), want 1", table, other[table]) + } + } + if after["operator_actions"] != 1 { + t.Errorf("operator_actions: %d — the action log is an audit record and stays at the purge", after["operator_actions"]) + } + for _, j := range []string{"customer_resets", "host_deletions"} { + if n := d73Count(t, s, `SELECT COUNT(*) FROM `+j+` WHERE customer_id = 'cust-a'`); n != 1 { + t.Errorf("journal %s: %d row(s), want 1 — the delete journal stays (the ruling's one exception)", j, n) + } + } + // the action log goes with the audit rows: not at 100 days, gone after a year + now := time.Now().UTC() + if _, _, err := s.PruneDeletedCustomerAudit(now.Add(100*24*time.Hour), DeletedCustomerAuditKeep); err != nil { + t.Fatal(err) + } + if n := d73Counts(t, s, "box-cust-a", "cust-a")["operator_actions"]; n != 1 { + t.Errorf("operator_actions after 100 days: %d, want 1", n) + } + if _, _, err := s.PruneDeletedCustomerAudit(now.Add(400*24*time.Hour), DeletedCustomerAuditKeep); err != nil { + t.Fatal(err) + } + if n := d73Counts(t, s, "box-cust-a", "cust-a")["operator_actions"]; n != 0 { + t.Errorf("operator_actions a year after the deletion: %d, want 0", n) + } + if n := d73Counts(t, s, "box-cust-b", "cust-b")["operator_actions"]; n != 1 { + t.Errorf("the OTHER household's action log: %d, want 1", n) + } + for _, j := range []string{"customer_resets", "host_deletions"} { + if n := d73Count(t, s, `SELECT COUNT(*) FROM `+j+` WHERE customer_id = 'cust-a'`); n != 1 { + t.Errorf("journal %s after the one-year prune: %d row(s), want 1", j, n) + } + } +} + +// The daily sweep is for what was left BEFORE: rows of a box or household that no longer exists. A live box, a live +// household, and a household in the middle of its delete (its boxes are gone, its config is still there) lose nothing. +func TestD73_PurgeOrphanRows_OnlyTheGone(t *testing.T) { + s := newTestStore(t) + // live: a household with a box + _ = s.SaveCustomerConfig(&CustomerConfig{CustomerID: "live", APIKey: "k1", RetrievalPassword: "p"}) + _ = s.UpsertHost(&Host{HostID: "live-box", CustomerID: "live", APIKey: "hk1"}) + d73Seed(t, s, "live-box", "live") + // mid-delete: the config is there, the box is already gone (so its box rows ARE orphans, its household rows are not) + _ = s.SaveCustomerConfig(&CustomerConfig{CustomerID: "mid", APIKey: "k2", RetrievalPassword: "p"}) + d73Seed(t, s, "mid-box", "mid") + // a ghost with a box: the config row is gone, a box row is left → the household still „exists” + _ = s.UpsertHost(&Host{HostID: "ghost-box", CustomerID: "ghost", APIKey: "hk3"}) + d73Seed(t, s, "ghost-box", "ghost") + // gone: neither config nor box (Peti's box, the test boxes of 2026-11) + d73Seed(t, s, "gone-box", "gone") + + got, err := s.PurgeOrphanRows() + if err != nil { + t.Fatal(err) + } + want := map[string]map[string]int{ + "live": {"os_host_settings": 1, "os_reports": 2, "os_kernel_notices": 1, "offsite_keys": 1, "offsite_windows": 1, "offsite_abandon_requests": 1, "operator_actions": 1}, + "mid": {"os_host_settings": 0, "os_reports": 0, "os_kernel_notices": 0, "offsite_keys": 1, "offsite_windows": 1, "offsite_abandon_requests": 1, "operator_actions": 1}, + "ghost": {"os_host_settings": 1, "os_reports": 2, "os_kernel_notices": 1, "offsite_keys": 1, "offsite_windows": 1, "offsite_abandon_requests": 1, "operator_actions": 1}, + "gone": {"os_host_settings": 0, "os_reports": 0, "os_kernel_notices": 0, "offsite_keys": 0, "offsite_windows": 0, "offsite_abandon_requests": 0, "operator_actions": 1}, + } + for cust, tables := range want { + c := d73Counts(t, s, cust+"-box", cust) + for table, n := range tables { + if c[table] != n { + t.Errorf("%s / %s: %d row(s), want %d", cust, table, c[table], n) + } + } + } + if got.Total() != 11 || got["os_reports"] != 4 || got["offsite_keys"] != 1 { + t.Errorf("the sweep reported %v (total %d), want 11 rows: 8 box rows of mid and gone, 3 household rows of gone", got, got.Total()) + } + again, _ := s.PurgeOrphanRows() + if again.Total() != 0 { + t.Errorf("a second sweep removed %v — it must find nothing", again) + } +} diff --git a/hub/internal/store/deleted_customer_audit.go b/hub/internal/store/deleted_customer_audit.go index 7afb64b7..191ab15b 100644 --- a/hub/internal/store/deleted_customer_audit.go +++ b/hub/internal/store/deleted_customer_audit.go @@ -65,6 +65,11 @@ func (s *Store) PruneDeletedCustomerAudit(now time.Time, keep time.Duration) (ev } n, _ = r.RowsAffected() notifications += n + // D73 (R-1376): the operator's action log of the deleted household is an audit record of the same kind — it had + // no delete at all; it goes with the other two. + if _, err = s.db.Exec(`DELETE FROM operator_actions WHERE customer_id = ? AND requested_at <= ?`, d.id, cutoff); err != nil { + return events, notifications, fmt.Errorf("PruneDeletedCustomerAudit %s: operator_actions: %w", d.id, err) + } } return events, notifications, nil } diff --git a/hub/internal/store/orphan_rows.go b/hub/internal/store/orphan_rows.go new file mode 100644 index 00000000..acc6ede4 --- /dev/null +++ b/hub/internal/store/orphan_rows.go @@ -0,0 +1,55 @@ +package store + +import "fmt" + +// OrphanRows counts what PurgeOrphanRows removed, per table. +type OrphanRows map[string]int64 + +// Total is the sum. +func (o OrphanRows) Total() int64 { + var n int64 + for _, v := range o { + n += v + } + return n +} + +// orphanTables are the tables D73 names (`09` §3 decision 293, R-1376) — the ones that had no delete statement — each +// with the column that names its owner and the query that says „the owner still exists”. +// +// A BOX exists while it has a `hosts` row. A HOUSEHOLD exists while it has a config row OR a box: a „ghost” (the +// config row gone, residue left) is exactly what this sweep is for, but a household in the middle of its delete +// cascade (boxes gone, config still there) is not an orphan. +var orphanTables = []struct{ table, where string }{ + {"os_host_settings", `host_id NOT IN (SELECT host_id FROM hosts)`}, + {"os_reports", `host_id NOT IN (SELECT host_id FROM hosts)`}, + {"os_kernel_notices", `host_id NOT IN (SELECT host_id FROM hosts)`}, + {"offsite_keys", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, + {"offsite_windows", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, + {"offsite_abandon_requests", `customer_id NOT IN (SELECT customer_id FROM customer_configs) AND customer_id NOT IN (SELECT customer_id FROM hosts)`}, +} + +// PurgeOrphanRows removes, in ONE transaction, the rows of boxes and households that no longer exist from the six +// tables that had no delete statement (D73). It is the daily clean-up for what was left BEFORE DeleteHost and +// PurgeCustomerResidue learned those tables (Peti's box, the test boxes of 2026-11): from now on a delete leaves +// nothing for it. It never touches the delete journal (`host_deletions`, `customer_resets`), `events`, +// `notification_log` or `operator_actions` (those go by the one-year rule). Idempotent. +// Pinned by TestD73_PurgeOrphanRows_OnlyTheGone. +func (s *Store) PurgeOrphanRows() (OrphanRows, error) { + tx, err := s.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + out := OrphanRows{} + for _, t := range orphanTables { + res, err := tx.Exec(`DELETE FROM ` + t.table + ` WHERE ` + t.where) + if err != nil { + return nil, fmt.Errorf("PurgeOrphanRows %s: %w", t.table, err) + } + if n, _ := res.RowsAffected(); n > 0 { + out[t.table] = n + } + } + return out, tx.Commit() +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 95acd58d..682a33d1 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -3322,6 +3322,11 @@ func (s *Store) DeleteHost(hostID string, deleteEscrow bool) error { `DELETE FROM log_bundle_requests WHERE scope_id = ?`, `DELETE FROM log_bundles WHERE scope_id = ?`, `DELETE FROM wg_peers WHERE host_id = ?`, + // D73 (`09` §3 decision 293, R-1376): the three box-keyed tables that had no delete at all. A deleted box + // kept its update settings, every update report it ever sent, and its kernel notices — for ever. + `DELETE FROM os_host_settings WHERE host_id = ?`, + `DELETE FROM os_reports WHERE host_id = ?`, + `DELETE FROM os_kernel_notices WHERE host_id = ?`, `DELETE FROM hosts WHERE host_id = ?`, } for _, q := range stmts { @@ -3329,6 +3334,12 @@ func (s *Store) DeleteHost(hostID string, deleteEscrow bool) error { return fmt.Errorf("DeleteHost %s: %q: %w", hostID, q, err) } } + // D73: the hub's own small alarm marks that NAME the box (`os_alarm::[:…]` — a clock alarm, a trust + // cause, a guest the watchdog gave up on, a signed job). instr, not LIKE: a host id may hold `_` or `%`. + if _, err := tx.Exec(`DELETE FROM hub_settings WHERE key LIKE 'os_alarm:%' AND + (instr(key, ':' || ?1 || ':') > 0 OR substr(key, -(length(?1) + 1)) = ':' || ?1)`, hostID); err != nil { + return fmt.Errorf("DeleteHost %s: alarm marks: %w", hostID, err) + } return tx.Commit() }