From 5277d859d50ab6c048771458498d927b47385c37 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Fri, 23 Oct 2026 19:40:44 +0200 Subject: [PATCH] hub 0.168.0: seal the last plain tokens in config_json (R-975, R-1241's Felhom half); decide row R-1243 for the hub's plain Longhorn copies (register 188 -> 189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cloudflare tunnel token, Cloudflare API token and git token inside customer_configs.config_json now hold the 05 §16.2 seal (enc:v1:, OFFSITE_SECRET_KEY). Old rows sealed once at start-up without a config_version bump, then VACUUM + WAL checkpoint (an UPDATE leaves the old plaintext in a freed block). -unseal-box-secrets opens them too. Eight tests, each red-proved. Built, not deployed. Audit: documentation/audits/hub-token-seal-2026-10-23/. --- CONTEXT.md | 12 + STATUS.md | 7 +- .../architecture/05-hub-architecture.md | 10 + .../hub-token-seal-2026-10-23/README.md | 104 +++++++ documentation/backlog/OPEN-ITEMS.md | 7 +- hub/CHANGELOG.md | 19 +- hub/cmd/hub/main.go | 8 +- hub/internal/configgen/r975_tokens_test.go | 36 +++ hub/internal/store/r879_box_seal.go | 18 +- hub/internal/store/r975_config_token_seal.go | 249 +++++++++++++++ .../store/r975_config_token_seal_test.go | 287 ++++++++++++++++++ hub/internal/store/store.go | 17 +- hub/internal/web/r975_form_tokens_test.go | 70 +++++ 13 files changed, 830 insertions(+), 14 deletions(-) create mode 100644 documentation/audits/hub-token-seal-2026-10-23/README.md create mode 100644 hub/internal/configgen/r975_tokens_test.go create mode 100644 hub/internal/store/r975_config_token_seal.go create mode 100644 hub/internal/store/r975_config_token_seal_test.go create mode 100644 hub/internal/web/r975_form_tokens_test.go diff --git a/CONTEXT.md b/CONTEXT.md index 203ce83d..970ddb85 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -16,6 +16,18 @@ > and holds nothing of its own; this file does hold its own content, namely the standing rulings below. +> **2026-10-23 19:22 — operator rulings by starting the weekend-night brief (R-1241's Felhom half; the upload watch).** +> **Part 1 = R-1241's Felhom half:** seal R-975 (done in hub 0.168.0, not deployed — `05` §16.2, +> `audits/hub-token-seal-2026-10-23/`); the hub's plain Longhorn copies are decide row R-1243 (CC's pick: take the hub +> volume out of the Longhorn backup; ep0's encrypted copy is the backup). **Part 2 = the upload watch, shaped by the +> operator:** the box measures its own upload line (first from what it already sees — the off-site backup's upload +> rate — a short test at a quiet hour only if needed); on a sustained high upload it attributes the bytes to an app's +> container and ASKS THE APP (qBittorrent's seeding, Plex's remote sessions); a mismatch is the alarm +> (`app_unexplained_upload`, the same shape for CPU, `app_unexplained_cpu`) — operator only. qBittorrent gets a default +> upload limit from the measured line; a limit the household set is respected. **Open for the operator, CC's pick in +> STATUS:** the default limit's size (CC: 50 % of the measured upload) and who is told (CC: the family only about the +> qBittorrent limit they can change; the unexplained alarms to the operator only — the R-388 rule). + > **2026-10-23 ~12:00 — operator rulings by starting the D39 brief (`09` §3 decision 255; R-1239 closed).** **D39 = B:** a > `security_fix` ladder step — box walk on 9202 required, bench optional, taken by the night leg. **No operator word per > incident:** a public advisory id (CVE or GHSA) + its https:// notice is the authority; the session that writes the step diff --git a/STATUS.md b/STATUS.md index 66cc6849..5b0f5d93 100644 --- a/STATUS.md +++ b/STATUS.md @@ -18,9 +18,8 @@ ## What is broken - **P1:** any hacked app on DooPlex can read the hub's nightly backups, and maybe the signing keys (R-1241, homelab session); about 12 homelab logins still use a password that was public, and the public Gitea repo is not decided (R-925, yours). -- Tester 2 is off since 2026-10-04: old emergency password, no off-site check, no bootstrap (R-1139, R-1104, R-862). +- Tester 2 is off since 2026-10-04: old emergency password, no off-site check, no bootstrap (R-1139, R-1104, R-862). The scratch box cannot use agent features (R-1083). - DooPlex: the failed disk is still in the box, its alarms silenced (R-934); something sets the shared checkout to „bare" (R-935). -- The scratch box cannot use agent features (R-1083). ## What we work on - Dated checks, run by CC. 2026-10-24: the 8 GB box's first night copy on 0.340.0 (R-1214). 2026-10-26: each hub daily job ran once on the clock-change day (R-1183). 2026-10-27: the weekly clean-up window (R-435; the logs of the last one were lost to restarts). 2026-10-30: D35, website and installer to Cloudflare, starts (R-1228, ruled yes). @@ -31,6 +30,7 @@ Newest first. CC's pick is in the „Pick" column. Each row also has its full te | # | Row | Question | Pick (why) | The other option costs | If you do nothing | Blocked | |---|---|---|---|---|---|---| +| D40 | R-1243 | The hub's backups in DooPlex's shared folder are not encrypted (two there now hold the tunnel tokens). Encrypt the hub's disk, or stop that backup? | **Stop it** (the hub's own copy to ep0 is encrypted, every 6 h, tested weekly) | encrypt: 5–10 min hub downtime and one more key to keep | the plain copies stay readable by any hacked DooPlex app | a homelab change | | D38 | R-1235 | Put the newest golden (637 MB) into the nightly release copy on ep0 too? | **Yes** (a new install after a DooPlex loss needs it; it fits under the 3 GB cap) | leave it out: after a loss, no new install until a golden is rebuilt and vouched | the same | — | | D37 | R-1236 | Raise Gitea's memory limit from 1 GiB to 2 GiB? It was killed for memory 4 times in 17 days, once during today's release. | **Yes** (DooPlex has the RAM; a DooPlex change, so yours) | keep 1 GiB: a restart in release hours fails box updates and CI for a minute | the same | — | | D9 | R-1062 | Split the N100's USB drive in two, for multi-drive tests? | **No — keep one drive** (the lived-in box keeps its nightly whole-box backup) | Split: the box loses its local whole-box backup, or you buy a second drive first | it stays one drive; multi-drive tests stay on Tester 1 | — | @@ -51,7 +51,8 @@ Newest first. CC's pick is in the „Pick" column. Each row also has its full te 1. Print the break-glass sheet (`documentation/runbooks/break-glass-sheet.md`, the part from „The sheet”, one A4), fill it by hand, keep it outside the home (D32; R-923, R-924). If not: after a DooPlex loss no off-site copy opens. 2. On the hub's Hosts page: Discard appliance 49, and delete host Tester4-16fdef. If not: two dead entries alarm. 3. D37 and D38 in the table (two small yes/no). If not: Gitea may restart in release hours; no golden in the release copy. -4. Give the homelab session R-1241 and R-1242 and the hand-over list in `documentation/audits/plex2shell-felhom-impact-2026-10-23/README.md` (7 items: the backup folder is still open to every app, Longhorn needs no login, an unknown Longhorn delete at 13:24, ~1.5 TB sent out by Plex, lost logs). If not: the next hacked app on DooPlex can read the hub's backups. +4. Give the homelab session R-1241, R-1242 and the 7-item list at the end of `documentation/audits/plex2shell-felhom-impact-2026-10-23/README.md`. If not: the next hacked app on DooPlex can read the hub's backups. +5. Monday: deploy hub 0.168.0 — the last plain tokens locked (R-975). If not: a copy of the hub database still holds them. **The rest** - System poster needs a refresh: the new 01:40 „release copy” arrow from DooPlex to ep0 (D33); the hub-database label already says every 6 hours. If not: the drawing misses one copy. - Pull the failed disk Z6A07P2G out of DooPlex (R-934). If not: its alarms return on 11 November. Since 2026-10-12. diff --git a/documentation/architecture/05-hub-architecture.md b/documentation/architecture/05-hub-architecture.md index 6f4e7223..3359c9d9 100644 --- a/documentation/architecture/05-hub-architecture.md +++ b/documentation/architecture/05-hub-architecture.md @@ -555,6 +555,16 @@ record unreadable: its serve paths answer 500, a save refuses it (never blanks i to a hub before v0.138.0** needs the columns opened first: `felhom-hub -unseal-box-secrets` (all or nothing; exit 1 = nothing changed) — run in the running pod, then put the old image back. `guests.api_key` is an unused, always-empty column and is not sealed. *Decided by CC unattended — operator may reverse (`09` §3 decision 132).* +**Since v0.168.0 (R-975, R-1241) the last plain tokens are sealed too:** the Cloudflare tunnel token, the Cloudflare API +token and the git token live INSIDE `customer_configs.config_json` (`infrastructure.cf_tunnel_token`, +`infrastructure.cf_api_token`, `git.token`). Each of those JSON leaves holds the same `enc:v1:` seal; the rest of the JSON +stays readable. `SaveCustomerConfig` seals, every reader opens through `openCustomerSecrets` (a leaf that does not open is +blanked and marks the record unreadable, as above). Legacy rows are sealed at start-up (`SealLegacyConfigTokens`, without +a `config_version` bump — the box's config is byte-for-byte the same, so no box re-pulls or restarts), followed ONCE by +`VACUUM` + a WAL checkpoint, because an `UPDATE` leaves the old plaintext row in a freed block of the file +(`TestR975_DBFileWithoutKeyRevealsNoToken`). `-unseal-box-secrets` opens these leaves too, so the roll-back above also +covers a return to a hub before v0.168.0. The full list of secret-bearing columns, measured on a copy of the live database +2026-10-23: `audits/hub-token-seal-2026-10-23/README.md`. **And the key is the other half:** a backup of the database restores a hub that can open the sealed columns only with the same `OFFSITE_SECRET_KEY`; today that key exists only on DooPlex (the k8s Secret, and the GPG secrets export on the same machine). The off-site plan for the database and its key: `runbooks/RUNBOOK-hub-db-offsite-backup.md` diff --git a/documentation/audits/hub-token-seal-2026-10-23/README.md b/documentation/audits/hub-token-seal-2026-10-23/README.md new file mode 100644 index 00000000..cfba6763 --- /dev/null +++ b/documentation/audits/hub-token-seal-2026-10-23/README.md @@ -0,0 +1,104 @@ +# The hub's last plain tokens, sealed (R-975, R-1241's Felhom half) — 2026-10-23 night + +Class: implementation (hub 0.168.0, built, **not deployed**) + a design for the backup copies (Part 1.4). +Architecture read: `architecture/05-hub-architecture.md` §16.2–§16.3 (sealing, the snapshot), `13-threat-model.md` +(attacker j, R-1241), `audits/plex2shell-felhom-impact-2026-10-23/README.md`. Names and places only — no value appears +here or in any file of this session. Every count below came from a read-only copy of the hub's own 14:00Z snapshot +(`VACUUM INTO`, so it holds every committed write), copied to a 0700 scratch folder and deleted after. + +## 1. Every hub column that holds a secret (measured, 2026-10-23) + +| Where | What | Before 0.168.0 | 0.168.0 | +|---|---|---|---| +| `hosts.api_key` | each box's hub key | sealed (R-879, 0.138.0) — 4/4 | same | +| `customer_configs.api_key`, `retrieval_password` | controller key, owner passphrase | sealed (R-879) — 5/5 each | same | +| `host_pbs_secrets.value` | PBS-DR token | sealed (R-879) — 4/4 | same | +| `one_time_secrets.value` | Storage Box sub-account password | sealed (R-821, 0.127.0) — 4/4 | same | +| `host_recovery.secret` | break-glass console password | sealed (R-133, 0.135.0) — 4/4 | same | +| **`customer_configs.config_json` → `infrastructure.cf_tunnel_token`** | Cloudflare tunnel token | **plain — 4 rows** | **sealed** | +| **`customer_configs.config_json` → `infrastructure.cf_api_token`** | Cloudflare API token | **plain — 4 rows** | **sealed** | +| **`customer_configs.config_json` → `git.token`** | git token (the form still accepts one) | none stored today | **sealed** when written | +| `host_escrow.blob`, `identity_blob`; `host_escrow_superseded.*` | escrow | age-wrapped by the household's recovery code — not the hub's to seal | same | +| `api_tokens.token_hash`, `selfbind_tokens.token_hash`, `appliance_registrations.token_hash`, `customer_claims.code_hash`, `hub_settings[operator_password_hash]` | operator tokens, bind tokens, claim codes, operator password | one-way hashes (SHA-256 / bcrypt) | same | +| `hosts.api_key_hash`, `customer_configs.api_key_hash` | lookup twins | unkeyed SHA-256 of a 256-bit random key (R-879) | same | +| `wg_peers.pubkey`, `wg_endpoints.server_pubkey`, `host_escrow.key_fingerprint`, `appliance_registrations.ssh_host_pubkeys`, `hub_settings[oob_operator_ssh_pubkey]` | public keys, fingerprints | not secret | same | +| `guests.api_key` | unused column | empty 7/7 | same | +| `appliance_registrations.pairing_code` | 6-character console code | plain, 31 rows | **NOT-A-FINDING:** it is printed on the box's screen and is only one of two factors; the other (the self-bind token) is stored as a hash. Pending registrations only. | + +**Do the token VALUES sit anywhere else?** Each of the 8 distinct Cloudflare values was searched (by value, in memory, +never printed) in every column of every other table: **0 hits**. The key NAME `cf_tunnel_token` appears in one +`reports.report_json` and one `app_telemetry.issues_json` row — names, not values. `events.details_json` carries +`repo_key` (literally the word „changed") and PBS `token_id`s (a name like `user@pbs!name`, not a secret). + +## 2. The change (hub 0.168.0) + +- **Same mechanism, same key** (`05` §16.2): each token leaf in `config_json` holds `enc:v1:` + AES-256-GCM under + `OFFSITE_SECRET_KEY`. The rest of the JSON stays readable. `internal/store/r975_config_token_seal.go`. +- **New writes sealed:** `SaveCustomerConfig` (the only writer of `config_json`). No key and a token → refused. +- **Every reader opens through the existing helper:** `openCustomerSecrets`, which the three readers + (`GetCustomerConfig`, `GetCustomerConfigByAPIKey`, `ListCustomerConfigs`) already call. A leaf that does not open is + blanked and sets `SecretsUnreadable` (R-879's rule: never served, a save refuses the record, `configgen.Generate` refuses). +- **Old rows sealed once at start-up:** `SealLegacyConfigTokens`, logged as `config tokens sealed at rest (N legacy + plaintext token(s) sealed now)` — counts only. **No `config_version` bump** — the box's config is byte-for-byte the + same, so no box re-pulls or restarts. Expected on the live hub: N = 8 on the first start, 0 after. +- **Then one `VACUUM` + WAL checkpoint** (only when N > 0): an `UPDATE` leaves the old plaintext row in a freed block of + `hub.db` (the SQLite driver's `secure_delete` is 0 — measured). Timed on a copy of the live 534 MB database: **2.8 s**, + well inside the liveness probe's ~95 s. The volume has 2.4 GB free for it. +- **Roll-back:** `felhom-hub -unseal-box-secrets` now also opens these leaves (all or nothing), so going back to + 0.167.0 stays the documented procedure (`05` §16.2). Without it an older hub would hand the box the sealed string as its + tunnel token. + +**Tests, each red-proved** (the rule broken on purpose → the test failed → restored): + +| Test | Pins | Red-proof (mutation → failed) | +|---|---|---| +| `TestR975_FreshWriteIsStoredSealed` | a fresh write is sealed; non-secret fields intact; no key → refused | write path unsealed → FAIL (with 5 others) | +| `TestR975_ReaderGetsTheSameValue` | all three readers + a load-modify-save return the same value | open on read removed → FAIL | +| `TestR975_MigrationSealsOnceThenNoOp` | 3 sealed, then 0; no `config_version`/`updated_at` change | migration no-op → FAIL; migration bumps the version → FAIL | +| `TestR975_DBFileWithoutKeyRevealsNoToken` | **a copy of `hub.db` + `-wal` + `-shm` greps clean**; positive control: the pre-0.168.0 rows DO show in the file | VACUUM removed → FAIL (the old row stays in a freed block) | +| `TestR975_WrongKeyBlanksAndRefusesSave` | wrong key → blanked, flagged, save refused, right key restores | not blanked → FAIL; flag ignores the tokens → FAIL | +| `TestR975_UnsealRestoresPlaintextForAnOlderHub` | roll-back opens the tokens, all or nothing | the config half skipped → FAIL | +| `TestR975_EveryFormSecretIsSealed` (web) | every leaf the operator form writes is either on a 2-name non-secret list or sealed | write path unsealed → FAIL | +| `TestR975_GeneratedConfigCarriesThePlainTunnelToken` (configgen) | the box still gets the REAL token in `controller.yaml` | open on read removed → FAIL | + +Two first mutations broke the build instead of a test (an unused variable) and were redone so a test, not the +compiler, failed. The first version of the file test passed with the VACUUM removed — the token row was the last cell on +its page, so the new row overwrote it; a second customer written after it (as on the live hub) made the test honest. + +**Measured on the live file (count of the key name only):** `cf_tunnel_token` occurs 12 times both in the live +`hub.db` and in the clean `VACUUM INTO` snapshot, so the live file holds no extra old copies today. + +## 3. The backup copies — design only (Part 1.4) + +**What exists today (measured):** + +| Copy | Where | Encrypted? | Holds the tokens? | +|---|---|---|---| +| Hub snapshots `/snapshots/` (2, 6-hourly) | the hub's Longhorn volume | no (the volume is `encrypted=false`) | after 0.168.0: sealed | +| **Longhorn backups of the hub volume** | `/mnt/5_hdd/backup/longhorn-pvc` (the NFS folder of R-1241) — **2 backups: 2026-10-18 03:00Z (weekly) and 2026-10-23 02:01Z (daily)**, ~4.1 GB each, `retain 1` each | **no** | **yes, in plaintext** — both predate 0.168.0 | +| The hub's own copy to ep0 (`felhom-hub-db-backup`, 6-hourly :30) | ep0 PBS, namespace `operator` | **yes, at rest:** 16 snapshots 2026-10-10 → 10-23, every `hubdb.pxar.didx` and catalog `crypt-mode: encrypt` (`index.json.blob` sign-only). The key `/etc/felhom-hub-backup/enc.key` is on DooPlex's **root disk (sdb1)**, not on the NFS disk, and on paper (S2). **In transit:** SSH port-forward `127.0.0.1:18007` + PBS TLS with a pinned fingerprint. Last push 14:31 CEST today, 7 s; the weekly restore-test last passed 2026-10-18 („4 sealed console password(s), 0 readable"). | sealed after 0.168.0 | + +**The two options:** + +| | (a) Encrypt the hub's Longhorn volume | (b) Take the hub volume out of the Longhorn NFS backup | +|---|---|---| +| What | A new Longhorn volume with `encrypted=true` (a StorageClass with a crypto Secret), copy `/data` across, point `manifests/hub.yaml` at it | Remove the volume from the `default` recurring-job group (one label on the Longhorn volume); the ep0 copy is the hub's only backup | +| Hub downtime | ~5–10 min (scale to 0, copy 1.5 GB, scale up) | none | +| Who | homelab (a StorageClass + Secret in `longhorn-system`) + a Felhom manifest change | homelab (the Longhorn volume label); no Felhom code | +| New key to keep | yes — the volume passphrase, on paper, and it lives in the cluster where the R-1241 attacker reaches the Longhorn API | none | +| What it fixes | the backups stay but are ciphertext; also the volume's freed blocks | no more hub copies in the NFS folder at all | +| What it loses | nothing | the hub's 90-day access log (`/data/access-log`, R-1140) and its local snapshots are no longer backed up anywhere; a lost volume is restored from ep0 (`RUNBOOK-hub-db-offsite-backup.md` §restore, restore-tested weekly) — at most ~6.5 h of hub data lost, the same as today's off-site promise | +| Does the encryption key sit next to the data? | yes (cluster Secret) | the PBS key is on another disk and on paper | + +**CC's pick: (b)**, because the ep0 copy is already encrypted at rest and in transit (measured above), off-site, every 6 +hours, and restore-tested weekly — the Longhorn copy adds a second, unencrypted copy in exactly the folder R-1241 found +open, and (a) would put its key in the same cluster. The access log's loss can be closed later by adding +`/data/access-log` to the ep0 push (small; not tonight). → decide row **R-1243**. + +**The plain copies already in the folder** (the two hub-volume backups above): they hold today's tokens in plaintext. +Deleting backups is the operator's decision. Without action they rotate out by themselves after 0.168.0 is deployed — +the daily one at the next 04:00 run, the weekly one at the next Sunday 05:00 run (`retain 1` each) — **but only if the +volume stays in the backup job** (option a, or no decision). With option (b) the last two copies stay until someone +deletes them. Either way the tokens in them should then be considered exposed only if R-1241's door is used — the +Plex2Shell audit measured it was not. **Note:** Longhorn backs up the volume's blocks, so a block freed by the VACUUM can +still hold old plaintext in a later Longhorn backup; only (a) or (b) removes that — not measured, reasoned. diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index 9e2ed692..aa8d2c03 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -218,11 +218,11 @@ stopping line that lies. | **R-1094** | Storage & devices | P4 | **The whole-disk format refusal tells the household to format the free part — which is refused while the other part is in use.** SEEN 2026-10-16 (controller 0.318.0): 409 „Ezen a lemezen egy használatban lévő meghajtó van, ezért a teljes lemez nem formázható. Csak a szabad részét formázhatod." — since agent 0.158.1 a part next to a mounted part can only be USED (the root wrapper trusts no „Felhom's own" claim). Evidence: `audits/storage-identity-buildB-2026-10-15/live/14-samedisk-refusal-offer.txt`. | READY | — | Reword `err.web.meghajto_hasznalatban_formazas` (hu + en): „… nem formázható. A szabad részét használatba veheted." Next controller release; the copy gates run. | CC | | **R-1213** | Storage & devices | P4 | **Operator detail still on two household pages:** the registered drive card's `ext4 · /dev/sdb1 · Model` line (`templates/storage.html`; W14 hid only the agent drive cards), and the heading „Távoli mentés (3. mentés) — titkosított, offsite" with the English „offsite". Found by the night helper 2026-10-21 (W11/W14 follow-up). | OPEN | — | Move the line behind „Részletek"; „…titkosított, házon kívüli" | CC | -## Security & access — 21 rows (P1 2, P2 2, P3 11, P4 6) +## Security & access — 22 rows (P1 2, P2 3, P3 11, P4 6) | ID | Category | Sev | What | State | Blocked on | Next action | Owner | |---|---|---|---|---|---|---|---| -| **R-1241** | Security & access | P1 | **A hacked app pod on DooPlex can read the hub's nightly database copies — with every customer's unsealed Cloudflare, tunnel and git token (R-975) — through the Longhorn NFS export, and by a published file-handle escape the operator signing keys on the same disk: whoever breaks one homelab app can take over the boxes' tunnels and sign agent updates.** MEASURED 2026-10-23 (Plex2Shell impact audit, `audits/plex2shell-felhom-impact-2026-10-23/`): export `/mnt/5_hdd/backup/longhorn-pvc` rw, `no_root_squash`, `no_subtree_check`, a sub-directory of sda1, which also holds `/mnt/5_hdd/felhom.eu/felhom-op-*` (no passphrase, R-924); `secure` is void in pods (containerd sets `ip_unprivileged_port_start=0`; a uid-1000 probe with no capabilities bound port 700); the four `felhom-system` volumes have `encrypted=false`; Longhorn's API on :9500 answers any pod with no login (backups, targets, volumes). In the Plex2Shell window neither door was used — verdict NOT REACHED (nfsd accepted 0 new connections 10-10 → 10-23, positive control; no off-schedule Longhorn backup or volume). 2026-10-23 ~15:18Z, operator yes: the export was narrowed from `*` to `10.42.0.0/16` — the home network and the tailnet are closed, pods are not (Longhorn connects from its pod address). Not exercised: the escape itself (Tier-2 machine). | OPEN | the homelab session (DooPlex host + cluster); the operator (R-924) | Homelab: a NetworkPolicy so only `longhorn-system` reaches :2049/:111 and :9500, or the backup store on its own filesystem. Felhom: seal R-975; a passphrase on the signing keys (R-924) | operator | +| **R-1241** | Security & access | P1 | **A hacked app pod on DooPlex can read the hub's nightly database copies — with every customer's unsealed Cloudflare, tunnel and git token (R-975) — through the Longhorn NFS export, and by a published file-handle escape the operator signing keys on the same disk: whoever breaks one homelab app can take over the boxes' tunnels and sign agent updates.** MEASURED 2026-10-23 (Plex2Shell impact audit, `audits/plex2shell-felhom-impact-2026-10-23/`): export `/mnt/5_hdd/backup/longhorn-pvc` rw, `no_root_squash`, `no_subtree_check`, a sub-directory of sda1, which also holds `/mnt/5_hdd/felhom.eu/felhom-op-*` (no passphrase, R-924); `secure` is void in pods (containerd sets `ip_unprivileged_port_start=0`; a uid-1000 probe with no capabilities bound port 700); the four `felhom-system` volumes have `encrypted=false`; Longhorn's API on :9500 answers any pod with no login (backups, targets, volumes). In the Plex2Shell window neither door was used — verdict NOT REACHED (nfsd accepted 0 new connections 10-10 → 10-23, positive control; no off-schedule Longhorn backup or volume). 2026-10-23 ~15:18Z, operator yes: the export was narrowed from `*` to `10.42.0.0/16` — the home network and the tailnet are closed, pods are not (Longhorn connects from its pod address). Not exercised: the escape itself (Tier-2 machine). | OPEN | the homelab session (DooPlex host + cluster); the operator (R-924) | Homelab: a NetworkPolicy so only `longhorn-system` reaches :2049/:111 and :9500, or the backup store on its own filesystem. Felhom: R-975 sealed in hub 0.168.0 (deploy pending); the hub's plain Longhorn copies → R-1243; a passphrase on the signing keys (R-924) | operator | | **R-1139** | Security & access | P2 | **Tester 2's break-glass root password was not rotated with the others (R-954, operator decision D1 = A, 2026-10-17): the box has been offline since 2026-10-04.** demo-hp, Tester 1 and demo-felhom were rotated and proven on 2026-10-17 (`audits/agent-allowlist-rotation-2026-10-17/P3/`). | OPEN — filed 2026-10-17 **2026-10-22: closes when Tester 2 is reinstalled (operator ruling 2026-10-22 14:11 — a fresh install sets and vaults a new console password).** | Tester 2 reporting again | When it reports: the installer's step 4b alone (`P3/rotate-step4b.sh` shape), then the vaulted value logs in (PVE API 200) and the old one is refused | CC | | **R-1042** | Security & access | P3 | **The boxes' Cloudflare keys cannot change zone settings, so Cloudflare's own „Always Use HTTPS” cannot be switched on.** Measured 2026-10-14 on all three zones (`audits/https-only-2026-10-14/``1A-1-zones-and-keys.txt`): each `cf_api_token` is valid and sees only its own zone, but GET/PATCH `settings/always_use_https`, `ssl`, `security_header` answer 403; the redirect and transform rule phases too. The https-only fix therefore lives on the box (decision 246). A second layer at Cloudflare needs „Zone Settings: Edit” (and, for headers, „Transform Rules”) added to each customer's key — the hub's zone-reach check stays. | **WAITING-ON-OPERATOR** — filed 2026-10-14 | a key permission only the operator can grant | Optional: widen each customer key, then switch „Always Use HTTPS” on per zone | operator | | **R-1242** | Security & access | P3 | **The demo NAS export `/mnt/5_hdd/felhom-data` → 192.168.0.162 (demo-felhom) is a sub-directory of DooPlex's sda1 with `no_subtree_check`, so a compromised demo-felhom host could walk the whole disk as uid 101000 and read every world-readable file there (the GPG secrets exports are 0644, every repo, the public keys).** Measured 2026-10-23 (`audits/plex2shell-felhom-impact-2026-10-23/` Part 1; `exportfs -v`: `all_squash,anonuid=101000`). 0600 files (the signing keys) stay out of reach. Not exercised. | OPEN | a DooPlex host change (fence) | Give the demo share its own filesystem, or `subtree_check`; or remove other-read from `/mnt/5_hdd/backup/secrets/exports` | operator | @@ -236,6 +236,7 @@ stopping line that lies. | **R-908** | Security & access | P3 | **An old Resend API key is still in the history of the `homelab-manifests` repository; it was replaced on 2026-06-29, but whether it is also revoked at Resend is unknown.** FOUND 2026-10-08 by the contact-mailer source search (`audits/mailer-source-2026-10-08/SEARCH.md`, „A side finding"): commit c9648cd (2026-02-05, „added mailer pod") put a key literal in the old `felhom-system/contact-mailer.yaml` comment; the file was deleted in ee93b50 but the history keeps it. Compared by hash only, never printed: it is NOT today's key (`Secret/resend-api`, rotated 2026-06-29, felhom.eu feea0606). If the old key still works at Resend, anyone with read access to that repository can send mail as felhom.eu. | **OPEN — owner: operator** **Note 2026-10-12 (security review):** gitleaks 8.24.3 with its default rules did NOT flag this key in the full history scan, so a scanner gate alone would not have caught it (R-986). | — | In the Resend console, check that the old key is revoked (revoke it if not); rewriting the repository's history is NOT needed once it is revoked | operator | | **R-525** | Security & access | P4 | **[P3-LOW] FileBrowser has its own login; putting it behind the dashboard session (traefik forwardAuth or Quantum proxy auth) is a new mechanism nobody has measured.** Filed 2026-09-15 by the P1-fixes task (B.5). R-513 closed the default-password hole with a generated password; a household still has two logins. **What it needs:** a spike on a scratch guest — forwardAuth to the controller session, and what FileBrowser Quantum does with a trusted header. | **READY — rank P3-LOW; owner: CC (spike)** **Re-ranked 2026-10-03: P3->P4: comfort feature needing a new unmeasured mechanism; the default password hole is closed.** | — | — | CC | | **R-925** | Security & access | P1 | **`manifests/felhom.secret.yaml` carries REAL secret values, and the Gitea repo that holds it is readable by anyone on the internet with no login.** FOUND 2026-10-09 by a background security review of the legal-pages commit, which flagged the served `` comments; chasing what those comments pointed at found this instead. **MEASURED, in this order:** (1) `scripts/manifest_bearer_gate.py` itself prints `manifests/felhom.secret.yaml:39 KNOWN-BACKLOG committed secret 65cee3c4…7a86`; (2) the file holds **live-shaped values, not placeholders** — `SECRET_KEY` (69 chars), `SUPERUSER_EMAIL`/`SUPERUSER_PASSWORD` (18), Umami `APP_SECRET` (66) and `POSTGRES_PASSWORD` (34), and a `username`/`password` pair (18); values were never printed, only measured by length; (3) `gitea.dooplex.hu` resolves to **37.191.56.193**, the same public address as the website; (4) an **anonymous** `curl` returns HTTP 200 and 1686 bytes for that exact file, and 200 for `hub/internal/store/store.go` and `manifests/hub.yaml`; (5) **the off-network control**, which is what makes (4) mean anything — fetched from outside the operator's network entirely: a real path returns the file's first line, a nonsense path returns 404, so the 200 is genuine anonymous read from the internet and not a LAN-only ACL. **This contradicts the project's own written model**: `documentation/runbooks/secrets.md:3-4` says *“Secret values are never committed to git. The manifests in `manifests/` carry only placeholders + comments that point here.”* That promise is false today, which is the P1 wording in this register's own scale — see the ranking note below. The same runbook already states the rule that governs the fix: *“The git-history copy stays alive until the value is ROTATED — de-git alone kills nothing.”* **Known neighbours, none of which cover this:** R-887 records that an outside crawler walks the public Gitea pages and leaves *“Gitea's exposure to the crawler”* as the operator's undone item; R-580 still owes a Gitea admin token rotation. Neither says a secret file is world-readable. **Why P2 and not P1, stated so the operator can overrule it:** the exposed credentials guard **analytics and an undeployed healthchecks instance, not household data** — measured: `umami-db` is a **ClusterIP** service on 5432 with no external IP, so the Postgres password is not reachable from the internet; the realistic harm is session forgery against the public `stats.felhom.eu` via `APP_SECRET`, and reuse of those passwords anywhere else. No customer box, hub token or escrow key is in this file. **CC did NOT change anything**: making the repo private could break the public day-0 path (the installer is fetched from a public tag, R-110), and rotation plus repository visibility are operator decisions on production infrastructure. **PARTLY FIXED 2026-10-09, and RE-RANKED P2 -> P1 on what it turned out to be.** The exposed value was not an analytics password: it was the **Gitea `admin` account password** (`is_admin: true`; `/api/v1/admin/users` answered 200), published in a repo `gitea.dooplex.hu` serves anonymously to the internet. That is push access to every repo — including the one whose `website/` is git-synced live and whose `scripts/` is published by tag to **every new box installer** (R-110), i.e. a supply-chain path onto customer hardware. My first ranking said P2 because I had only measured the analytics blast radius; the credential test is what corrected it. **DONE (CC, operator-instructed):** (1) `umami-config` `APP_SECRET` + `POSTGRES_PASSWORD` rotated — the password changed **inside Postgres** too (`ALTER USER`), because the env var is only read at first init; verified by a real beacon returning 200 with a bogus-site-id control still returning 400. (2) `healthchecks-config` `SECRET_KEY` + `SUPERUSER_PASSWORD` rotated (nothing consumes them — there is no healthchecks Deployment). (3) `gitea-creds` no longer holds the admin password at all: it holds a **scoped token** (`read:package` + `read:repository`). **Both scopes are load-bearing and the second was found by breaking it** — a package-only token made the hub log `Template fetch: unexpected status 403`, because the template fetcher reads a raw file out of the `felhom-controller` repo. (4) The Gitea **admin password** was changed and `gitea-system/gitea-admin` updated; **verified: new password 200, old published password 401 — the leak is dead.** (5) The file is removed from git and `.gitignore`'s `*secret*` now applies to it; `manifest_bearer_gate.py`'s `KNOWN_BACKLOG` carve-out is gone, red-proofed with a decoy (exit 1 with, 0 without). **AN INCIDENT CAUSED BY THE FIX, recorded because it is the useful part:** the `rollout restart` needed to pick up the new umami secret put umami into **CrashLoopBackOff and took `stats.felhom.eu` down (503)**. Cause was not the rotation — at `memory: 512Mi` the pod runs for months but **cannot restart**, because startup (Prisma + Next.js) peaks over the limit and is OOMKilled (exit 137). Raised to 1Gi, **in the manifest, not just live** (the `.claude/rules/manifests.md` rule: a bare `kubectl set` is reverted by the next sync and the fix silently disappears). Service restored and verified. New values are in a 0600 file on DooPlex (`~/rotated-secrets-2026-10-09.txt`), never echoed; the operator moves them to the password manager and deletes it. **WHAT REMAINS, and it is bigger than what was fixed:** that one password is still the admin password in about **12 other namespaces** — `nextcloud`, `paperless`, `bookstack` (a **database root** password), `tandoor`, `calibre`, `adventurelog`, `gokapi`, `qbittorrent` (x2), `servarr`, `homepage` (x2). Rotating Gitea does not touch them: each is its own login and each is still the string that was published. **Also owed:** a Gitea token for user `kisfenyo` sits in plaintext in the `origin` URL of the local `homelab-manifests` clone (`.git/config`); CC printed it to a session transcript while investigating, so it should be rotated regardless — this is R-580's shape (store the remote without credentials). **Not a finding:** `homelab-manifests` itself is private (404 anonymously) and does not contain the password; ArgoCD's repo credential is a separate token, so the rotation did not touch it. | **NARROWED 2026-10-09 — the felhom.eu half is DONE and verified (Gitea admin password rotated, old one now 401; umami + healthchecks rotated; file de-gitted; gate carve-out removed). What remains is the ~12 reused logins elsewhere in the homelab and the repo-visibility decision; owner: operator** **2026-10-10 (release session), two consequences measured:** (a) DooPlex's own Docker login for `gitea.dooplex.hu` (`~/.docker/config.json`) still holds the OLD admin password — the registry token endpoint answers 401 for it and 200 for the current one — so every image push from DooPlex fails `unauthorized` until it is refreshed; the 0.145.0/0.305.0 pushes used a one-off login in a scratch `DOCKER_CONFIG`, shredded after. (b) The `felhom` ArgoCD app reads OutOfSync for `Secret/gitea-creds`, `Secret/healthchecks-config`, `Secret/umami-config` and `Deployment/umami`: a whole-app sync would push git's (de-gitted) view over the rotated live Secrets — the hub deploy synced ONLY `Deployment/hub` (`audits/release-2026-10-10/hub-0.145.0-deploy.txt`). Refreshing the DooPlex login is a DooPlex change (operator's word). **2026-10-22 night (Part 6, read only, `audits/night-2026-10-22/P6-repos-private/01-readers.md`):** homelab-manifests is already private; felhom-controller can go private after one change — CI clones with a read token (check first that the hub's `gitea-creds` token user can read a private `admin` repo); felhom.eu must stay public now (the website git-sync has no credential), felhom-agent too (every install fetches 16 config files from `raw/tag` with no login), and the catalog too (every box clones it with no login, and a token on a customer box is readable by its owner). Making a repository private does NOT hide its images or packages: they follow the user `admin`, which is public — and boxes need that (NOT-A-FINDING). **Count corrected:** the served `` comments are 61 on four pages (adatkezeles 33, feltetelek 15, teszteles 7, en/testing 6), not „32 on /adatkezeles”. The visibility question is decision D29 in STATUS. **2026-10-22 evening — D29 = no (operator):** the controller repository stays public for now; this row's secret rotation is unchanged. | — | **OPERATOR RULED 2026-10-09: (a) he rotates the remaining services himself — CC's scope stopped at the Felhom boundary; (b) the repo STAYS anonymously readable for now**, because making it private breaks the website git-sync and the installer tag fetch, which both clone with no credentials (R-110). The standing consequence of (b): no secret may ever enter this repo again, which `manifest_bearer_gate.py` now enforces with no exemption; and if (b) is reversed, the 61 served `` comments (four pages, 33 on /adatkezeles) must be stripped in the same change. **The exact checklist of the 12 remaining secrets** (namespace / secret / key, re-measured after the rotation, with the two traps that bite — a DB password is not changed by editing the Secret, and a pod that has run for months may not restart) is in `documentation/runbooks/secrets.md`. Operator: **(1)** work that checklist, `bookstack-db/root-password` first because it is a database root password and every one of these hosts answers on the public internet; (`bookstack` first — it is a database root password); **(2)** rotate the `kisfenyo` Gitea token embedded in the local `homelab-manifests` remote URL, and store the remote without credentials; **(3)** decide whether `gitea.dooplex.hu` should answer anonymously at all — remembering the website git-sync clones it with no credentials and the installer is fetched from a public tag, so making it private breaks both unless they get credentials first, and the 61 served `` comments become a map of it; **(4)** move `~/rotated-secrets-2026-10-09.txt` into the password manager and delete it. If nothing is done: the published password keeps opening a dozen services, even though Gitea itself is now safe | operator | +| **R-1243** | Security & access | P2 | **The hub's Longhorn backups sit unencrypted in the NFS folder any DooPlex pod can read (R-1241) — the two there now (2026-10-18, 2026-10-23) hold every household's Cloudflare and tunnel tokens in plaintext, because they predate hub 0.168.0.** Measured 2026-10-23 night (`audits/hub-token-seal-2026-10-23/` §3): the hub volume is `encrypted=false` and in Longhorn's `default` backup group (daily 04:00 + weekly Sunday 05:00, retain 1 each); the hub's own copy to ep0 is encrypted at rest (16/16 snapshots `crypt-mode: encrypt`, key on DooPlex's root disk and on paper) and in transit (SSH tunnel + pinned PBS TLS), every 6 h, restore-tested weekly. DECIDE: (a) an encrypted Longhorn volume for the hub (~5–10 min downtime, a homelab StorageClass + key in the cluster) or (b) take the hub volume out of the Longhorn backup — ep0's copy is the backup; the access log then has no backup until it rides the ep0 push. **CC's pick: (b).** And: delete the two plain copies (the operator's call; with (b) they never rotate out). | WAITING-ON-OPERATOR | the operator's pick; a homelab change for either option | Operator: pick (a) or (b), and say whether to delete the two plain copies after hub 0.168.0 is live | operator | | **R-779** | Security & access | P4 | **[P3-LOW] Part A's "two outside addresses seen as two" is proven through the simulated tunnel only; on the REAL tunnel the second outside address (ep0, one request allowed) was refused by Cloudflare's edge with 403 and never reached the box.** Measured 2026-10-01 19:51 UTC (`audits/visitors-2026-10-01/A/L2-demo-hp-real-tunnel.txt`): no log line on demo-hp; demo-hp's box has no geo restriction in its settings, so a Cloudflare ZONE rule (country or bot, not read) refused a German datacenter address. DooPlex's own address on the real tunnel was seen as itself. **Needs:** one sign-in from a second Hungarian address (the operator's phone off wifi) while DooPlex is locked out — 2 minutes; and say which Cloudflare rule refused ep0. | **WAITING-ON-OPERATOR — rank P3-LOW; owner: operator (a phone), CC reads the logs** **Re-ranked 2026-10-03: P3→P4: a proof gap on the real tunnel; operator-only follow-up.** | — | — | CC + operator | | **R-904** | Security & access | P4 | **Cloudflare can read every household's app traffic; replacing it with our own relay is a later item.** Facts (reviewer discussion 2026-10-08; `01`): app traffic and the dashboard reach the box through the Cloudflare Tunnel (`01` §5 trust table, rows end-user ↔ apps and customer ↔ controller UI; §7), and the tunnel's public end is Cloudflare's edge, where TLS ends — so Cloudflare can technically read that traffic (the FAQ says so since 2026-10-08, R-900; „TLS ends at the edge" is not written in `01` — add it there). What Cloudflare gives today, free: inbound reach with no router setup, the CGNAT answer (`01` §4, §7); certificates (`01` §7, the free tier covers one level below a zone); the geo-WAF the hub enforces (`01` §5 last row, §7); flood protection (not in `01`). The alternative named: our own EU relay over WireGuard with TLS passthrough by SNI, certificates on the box, the geo-block on the relay. Its costs: one more machine the operator keeps up, and a single point of reach for every box; weaker flood protection; about a week of work after a spike. Operator ruling 2026-10-08 09:07 (`09` §3 decision 184): a later item. | **DEFERRED** — **2026-10-10: the asked sentence is in `01` §7** („TLS ends at Cloudflare's edge …”); the spike stays deferred. **DEFERRED — after the first customers (operator ruling 2026-10-08 09:07)** | — | A spike after the first customers (the relay's reach, cost and flood behaviour, measured) | operator | | **R-1040** | Security & access | P4 | **Until the household makes Sonarr's or Radarr's first admin, any container on the box can read the app's API key.** Measured on the bench 2026-10-14 with the R-1030 seed in place: `AuthenticationRequired=Enabled`, `AuthenticationMethod=None` — `/initialize.json` from another container answers 200 with `apiKey`. Outsiders are blocked by the box's setup gate in that window; a compromised other app is not. | **OPEN** — filed 2026-10-14 | — | Seed the API key at install (`SONARR__AUTH__APIKEY`, measured to work) so the box knows it, and close the window by telling the household in the first steps; or seed a Forms user | CC | @@ -290,7 +291,7 @@ stopping line that lies. | ID | Category | Sev | What | State | Blocked on | Next action | Owner | |---|---|---|---|---|---|---|---| -| **R-975** | Hub & operator | P3 | **The hub database keeps every customer's tunnel token, Cloudflare API token and git token unsealed in one column, so anyone with a copy of the database (or its nightly snapshots) gets them.** R-879 sealed the other secret columns but not this one. Fix: seal this column the same way. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). | OPEN | — | Seal with a migration and a test | CC | +| **R-975** | Hub & operator | P3 | **The hub database keeps every customer's tunnel token, Cloudflare API token and git token unsealed in one column, so anyone with a copy of the database (or its nightly snapshots) gets them.** R-879 sealed the other secret columns but not this one. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). 2026-10-23 night: **sealed in hub 0.168.0** — the three JSON leaves of `customer_configs.config_json` hold the `05` §16.2 seal, old rows sealed once at start-up + VACUUM, eight tests red-proved (`audits/hub-token-seal-2026-10-23/`). Built, not deployed. | WAITING-ON-OPERATOR | the deploy of hub 0.168.0 (operator yes) | Deploy; read the start-up line `config tokens sealed at rest (8 …)`, then a second start `(0 …)`; then close | operator | | **R-31** | Hub & operator | P3 | **[P2-HIGH] Offsite provisioning is synchronous with no status affordance.** Save runs the Hetzner sync in-request, so the request can hit the nginx 504 **while succeeding server-side**: the operator cannot tell failed from slow, and a retry races the first attempt. **MIGRATED FROM `ROADMAP.md` 2026-08-22 (R-369) — originally filed 2026-07-21, size M, roadmap state `idea`.** Moved verbatim; nothing added or reinterpreted. The roadmap keeps its copy as history, marked moved. | **OPEN — migrated from ROADMAP 2026-08-22, rank unchanged** **Re-ranked 2026-10-03: P2→P3: operator-only; a known workaround (click once, wait, verify) exists.** **2026-10-06 night: the race half fixed on felhom.eu main (hub, unreleased):** a second Save while the first still provisions is refused with 409 („already running — wait about a minute, then reload; do not save again"); nothing saved, nothing created; per customer, in memory. `TestProvision_R31_*`, red-proof `audits/night-burndown-2026-10-06/hub/R-31-red.txt`. LEFT: the async save with a status card (the escrow-card idiom). | — | Direction: make it async + a status card, reusing the proven **awaiting-card/poll idiom** (v0.138.0 escrow card). **Interim mitigation belongs in R-3 as an operator note: click once, wait, verify — do not re-click.** | CC | | **R-882** | Hub & operator | P3 | **Longhorn on DooPlex could not grow a volume online: its `instance-manager` (116 days up) called a host process that no longer existed** — `nsenter: cannot open /host/proc/196610/ns/mnt` on every expansion retry, and an offline growth was blocked by the expansion's own attachment ticket (found 2026-10-05 growing `hub-data` to 2 Gi). A restart of the instance-manager (operator-approved) fixed it: 77/77 volumes back `attached/healthy` in 110 s. **Why the cached PID went stale was not established** (likely a containerd/k3s or iscsid restart after the instance-manager started), so it will recur after the next such restart and stay invisible until a volume needs to grow. `audits/hub-db-offsite-2026-10-05/partA/step1-*.txt` | **OPEN** | — | Find which host process the PID was and whether Longhorn 1.10.x re-resolves it; until then, before growing any volume, check the instance-manager's age against the last k3s/containerd restart | operator | | **R-883** | Hub & operator | P3 | **8 DooPlex workloads run an image by a moving tag (`:latest` or none), so any pod restart is a silent upgrade.** Measured 2026-10-05: the Longhorn restart restarted zipline on `ghcr.io/diced/zipline:latest` (pull Always), which pulled 4.8.0; 4.8.0 refused its database (`cannot safely migrate from prisma to drizzle: expected migration 20260508022000 … was not applied`) and crash-looped. Fixed for zipline by pinning `4.7.0` (homelab-manifests `90f60e4`, `4c8ec7a`; 4.7.0 applied the four missing migrations; a dump from before is kept out-of-band). The other 7 were counted, not named or changed. **Checked from source 2026-10-05 (burn-down round 2):** homelab-manifests@87dfc29 still has moving tags (grep image lines without a numeric tag): admin-system/toolbox.yaml:12 nicolaka/netshoot:latest (a bare Pod); calibre-system/cwa.yaml:826 calibre-web-automated:dev; outline-system/outline.yaml:270 minio/minio:latest; tandoor-system/recipe-importer.yaml:26 gitea.dooplex.hu/admin/recipe-importer:latest (pull Always :27); adventurelog-system/adventurelog.yaml:100 and :256 adventurelog-backend/frontend:latest (pull Always :101,:257); jarrs-system/jarr- | **OPEN** | — | List the 7 (`kubectl get deploy,sts -A` images without a fixed tag), pin each to the running version, and let Renovate move them | operator | diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index d94e7350..53c2574d 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,4 +1,21 @@ -## v0.167.0 (unreleased — committed 2026-10-22/23 night, NOT built, NOT deployed) — D31 and the D30 alarm clear +## v0.168.0 (unreleased — built 2026-10-23 night, NOT deployed; the deploy is the operator's yes) — the last plain tokens sealed (R-975, R-1241) + +- **R-975 / R-1241's Felhom half.** The Cloudflare tunnel token, the Cloudflare API token and the git token lived in + plaintext inside `customer_configs.config_json`, so a copy of the hub database (a snapshot, a Longhorn backup of the + hub volume — any DooPlex pod could read those, R-1241) held every household's tunnel and DNS credentials. Each of + those JSON leaves now holds the same `enc:v1:` AES-256-GCM seal under `OFFSITE_SECRET_KEY` as every other hub + secret (`05` §16.2) — no second mechanism. `SaveCustomerConfig` seals; `openCustomerSecrets` opens for all three + readers; a leaf that does not open is blanked and flags the record unreadable (the R-879 rule). Old rows are sealed + once at start-up (`SealLegacyConfigTokens`, counts logged, no `config_version` bump — no box re-pulls), followed once by + `VACUUM` + a WAL checkpoint, because an `UPDATE` leaves the old plaintext in a freed block (measured: 2.8 s on a copy + of the live 534 MB database). `-unseal-box-secrets` opens the leaves too, so the roll-back to 0.167.0 stays one command. + Tests `TestR975_FreshWriteIsStoredSealed`, `…_ReaderGetsTheSameValue`, `…_MigrationSealsOnceThenNoOp`, + `…_DBFileWithoutKeyRevealsNoToken` (a grep of the DB files; positive control: the pre-0.168.0 file shows the tokens), + `…_WrongKeyBlanksAndRefusesSave`, `…_UnsealRestoresPlaintextForAnOlderHub`, `…_EveryFormSecretIsSealed` (web), + `…_GeneratedConfigCarriesThePlainTunnelToken` (configgen) — each red-proved + (`documentation/audits/hub-token-seal-2026-10-23/`). + +## v0.167.0 (2026-10-23) — DEPLOYED on the operator's yes (manifest commit `34979d5d`; this header still read 'NOT built' until 2026-10-23 night) — D31 and the D30 alarm clear - **D31 (operator 2026-10-22, R-1205).** The F-14 PBS auto-re-issue now reads the customer's newest deletion of a host that HELD A KEY (`store.LatestKeyHoldingHostDeletion`, `escrow_acked = 1`) instead of the newest deletion of any diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 7a30aae2..65db6154 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -247,6 +247,12 @@ func main() { } else { logger.Printf("[INFO] box secrets sealed at rest (%d legacy plaintext value(s) sealed now)", n) } + // R-975: the tunnel, Cloudflare API and git tokens inside config_json — same key, same seal. + if n, serr := dataStore.SealLegacyConfigTokens(); serr != nil { + logger.Printf("[ERROR] sealing legacy config tokens failed after %d value(s): %v", n, serr) + } else { + logger.Printf("[INFO] config tokens sealed at rest (%d legacy plaintext token(s) sealed now)", n) + } } logger.Printf("[INFO] Database opened at %s", dbPath) @@ -1147,6 +1153,6 @@ func runUnsealBoxSecrets(st *store.Store, logger *log.Logger) int { logger.Printf("[ERROR] unseal-box-secrets: %v", err) return 1 } - logger.Printf("[INFO] unseal-box-secrets: %d sealed value(s) opened back to plaintext; a hub <= 0.137.0 can run on this database now", n) + logger.Printf("[INFO] unseal-box-secrets: %d sealed value(s) opened back to plaintext; any older hub can run on this database now", n) return 0 } diff --git a/hub/internal/configgen/r975_tokens_test.go b/hub/internal/configgen/r975_tokens_test.go new file mode 100644 index 00000000..ea6add8f --- /dev/null +++ b/hub/internal/configgen/r975_tokens_test.go @@ -0,0 +1,36 @@ +package configgen + +import ( + "io" + "log" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-975: the tokens are sealed in the database, and the box still receives the REAL tunnel token in its +// generated controller.yaml (the consequence — a sealed string there would be a dead tunnel). +func TestR975_GeneratedConfigCarriesThePlainTunnelToken(t *testing.T) { + st, err := store.New(filepath.Join(t.TempDir(), "t.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + js := `{"infrastructure":{"cf_tunnel_token":"tunnel-canary-r975","cf_api_token":"cfapi-canary-r975"}}` + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", RetrievalPassword: "p", APIKey: "k1", ConfigJSON: js}); err != nil { + t.Fatal(err) + } + cfg, err := st.GetCustomerConfig("c1") + if err != nil { + t.Fatal(err) + } + out, err := Generate("hub: {}\n", cfg, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "cf_tunnel_token: tunnel-canary-r975") || strings.Contains(out, "enc:v1:") { + t.Fatalf("controller.yaml does not carry the plain tunnel token (or carries a sealed string):\n%s", out) + } +} diff --git a/hub/internal/store/r879_box_seal.go b/hub/internal/store/r879_box_seal.go index 59c6c840..3b7069a4 100644 --- a/hub/internal/store/r879_box_seal.go +++ b/hub/internal/store/r879_box_seal.go @@ -178,7 +178,9 @@ var r879Columns = []struct { // UnsealBoxSecrets is the ROLL-BACK primitive for R-879: it opens every sealed value in the four // columns back to plaintext, so a hub older than R-879 (which looks keys up with `WHERE api_key = ?` -// and serves the passphrase column as it is) works on this database again. api_key_hash is kept +// and serves the passphrase column as it is) works on this database again. Since R-975 (0.168.0) it +// also opens the sealed token leaves of customer_configs.config_json, which a hub older than 0.168.0 +// would otherwise hand to the box as an `enc:v1:` string (a dead tunnel). api_key_hash is kept // (an older hub ignores it; a newer one re-seals at its next start). // // All or nothing: every sealed value is opened FIRST; if any one does not open (missing/wrong key, @@ -216,6 +218,11 @@ func (s *Store) UnsealBoxSecrets() (int, error) { } all[i].plain = pt } + // R-975: the config_json token leaves too — a hub older than 0.168.0 serves them as they are stored. + cfgRows, err := s.unsealConfigTokenRows() + if err != nil { + return 0, err + } tx, err := s.db.Begin() if err != nil { return 0, err @@ -231,6 +238,15 @@ func (s *Store) UnsealBoxSecrets() (int, error) { n++ } } + for _, r := range cfgRows { + res, err := tx.Exec(`UPDATE customer_configs SET config_json = ? WHERE customer_id = ? AND config_json = ?`, r.plain, r.id, r.sealed) + if err != nil { + return 0, fmt.Errorf("customer_configs.config_json: %w", err) + } + if k, _ := res.RowsAffected(); k > 0 { + n += r.tokens + } + } if err := tx.Commit(); err != nil { return 0, err } diff --git a/hub/internal/store/r975_config_token_seal.go b/hub/internal/store/r975_config_token_seal.go new file mode 100644 index 00000000..a6ee1999 --- /dev/null +++ b/hub/internal/store/r975_config_token_seal.go @@ -0,0 +1,249 @@ +package store + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// ── R-975 / R-1241: the tokens inside customer_configs.config_json are sealed at rest too ────────────── +// +// R-879 sealed every secret COLUMN of the hub database but left three secrets that live inside the +// customer's config_json overrides: the Cloudflare tunnel token, the Cloudflare API token and the git +// token. A copy of hub.db (a nightly snapshot, a Longhorn backup of the hub volume — R-1241 measured +// that any pod on DooPlex could read those) therefore still held every household's tunnel and DNS +// credentials in the clear. +// +// Now each of those JSON leaves holds the SAME seal as every other hub secret (`enc:v1:`, AES-256-GCM, +// OFFSITE_SECRET_KEY, `05` §16.2) — no second mechanism, no second key. The rest of config_json stays +// readable (it is not secret, and the operator pages show it). Writers seal in SaveCustomerConfig; +// every reader opens through openCustomerSecrets, so the rest of the hub sees plaintext exactly as +// before. A sealed token that does not open is BLANKED and sets SecretsUnreadable, the R-879 rule: +// never served, never compared, and the record cannot be saved back. +// +// Pinned by: TestR975_FreshWriteIsStoredSealed, TestR975_MigrationSealsOnceThenNoOp, +// TestR975_ReaderGetsTheSameValue, TestR975_DBFileWithoutKeyRevealsNoToken, +// TestR975_WrongKeyBlanksAndRefusesSave, TestR975_UnsealRestoresPlaintextForAnOlderHub. + +// configTokenPaths are the secret leaves of config_json, as written by buildConfigJSON (web/configs.go). +// A new secret key there must be added here — TestR975_EveryFormSecretIsSealed reads the form's keys. +var configTokenPaths = [][]string{ + {"infrastructure", "cf_tunnel_token"}, + {"infrastructure", "cf_api_token"}, + {"git", "token"}, +} + +// decodeConfigJSON parses a config_json object, keeping numbers as written. ok=false for "" or a value +// that is not a JSON object (such a value holds no token leaf this file knows, and is left as it is). +func decodeConfigJSON(js string) (map[string]any, bool) { + if strings.TrimSpace(js) == "" { + return nil, false + } + dec := json.NewDecoder(strings.NewReader(js)) + dec.UseNumber() + var m map[string]any + if err := dec.Decode(&m); err != nil || m == nil { + return nil, false + } + return m, true +} + +func encodeConfigJSON(m map[string]any) (string, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(m); err != nil { + return "", err + } + return strings.TrimRight(buf.String(), "\n"), nil +} + +// mapConfigTokens calls fn on every non-empty string token leaf; fn returns the new value and whether +// it changed. The JSON is re-encoded only when something changed, so a config with no token keeps its +// exact bytes. +func mapConfigTokens(js string, fn func(path, v string) (string, bool, error)) (string, int, error) { + m, ok := decodeConfigJSON(js) + if !ok { + return js, 0, nil + } + n := 0 + for _, p := range configTokenPaths { + parent, ok := m[p[0]].(map[string]any) + if !ok { + continue + } + v, ok := parent[p[1]].(string) + if !ok || v == "" { + continue + } + nv, changed, err := fn(p[0]+"."+p[1], v) + if err != nil { + return js, n, err + } + if changed { + parent[p[1]] = nv + n++ + } + } + if n == 0 { + return js, 0, nil + } + out, err := encodeConfigJSON(m) + if err != nil { + return js, 0, err + } + return out, n, nil +} + +// sealConfigJSONTokens seals every plaintext token leaf. No key and a token to seal → ErrNoSealKey +// (fail-closed, §16.2); a config with no token needs no key. +func (s *Store) sealConfigJSONTokens(js string) (string, int, error) { + return mapConfigTokens(js, func(_, v string) (string, bool, error) { + if strings.HasPrefix(v, sealPrefix) { + return v, false, nil + } + sealed, err := s.sealSecret(v) + if err != nil { + return "", false, err + } + return sealed, true, nil + }) +} + +// openConfigJSONTokens opens every sealed token leaf. A leaf that does not open is blanked and ok=false; +// a legacy plaintext leaf passes through (the start-up sealing may not have run yet). +func (s *Store) openConfigJSONTokens(customerID, js string) (string, bool) { + if !strings.Contains(js, sealPrefix) { + return js, true + } + ok := true + out, _, err := mapConfigTokens(js, func(path, v string) (string, bool, error) { + if !strings.HasPrefix(v, sealPrefix) { + return v, false, nil + } + pt, good := s.openAtRest("customer_configs.config_json."+path, customerID, v) + if !good { + ok = false + return "", true, nil + } + return pt, true, nil + }) + if err != nil { + if s.logger != nil { + s.logger.Printf("[ERROR] store: config_json tokens for %s: %v", customerID, err) + } + return js, false + } + return out, ok +} + +// SealLegacyConfigTokens seals, in place, every config_json token leaf still in plaintext. Idempotent; +// returns how many token values it sealed. It does NOT bump config_version or updated_at: the config a +// box receives is byte-for-byte the same, so no box should re-pull and restart for it. Each row is +// written guarded by the value it read (a row saved meanwhile is left for the next start). Values are +// never logged. Called at start-up beside SealLegacyBoxSecrets (cmd/hub/main.go). +func (s *Store) SealLegacyConfigTokens() (int, error) { + if s.sealer == nil { + return 0, ErrNoSealKey + } + rows, err := s.db.Query(`SELECT customer_id, config_json FROM customer_configs`) + if err != nil { + return 0, err + } + type row struct{ id, js string } + var all []row + for rows.Next() { + var r row + if err := rows.Scan(&r.id, &r.js); err != nil { + rows.Close() + return 0, err + } + all = append(all, r) + } + rows.Close() + n := 0 + for _, r := range all { + sealed, k, err := s.sealConfigJSONTokens(r.js) + if err != nil { + return n, fmt.Errorf("customer_configs.config_json for %s: %w", r.id, err) + } + if k == 0 { + continue + } + res, err := s.db.Exec(`UPDATE customer_configs SET config_json = ? WHERE customer_id = ? AND config_json = ?`, sealed, r.id, r.js) + if err != nil { + return n, fmt.Errorf("customer_configs.config_json for %s: %w", r.id, err) + } + if a, _ := res.RowsAffected(); a > 0 { + n += k + } + } + if n > 0 { + // An UPDATE leaves the old row — the plaintext tokens — in a freed page of hub.db and in the -wal + // until something overwrites it, so a copy of the FILE (a Longhorn backup of the volume) would + // still hold them. VACUUM rewrites the file without free pages; the checkpoint empties the -wal. + // Once: a second start seals nothing and does not vacuum. Measured on a copy of the live + // database (534 MB): 2.8 s. TestR975_DBFileWithoutKeyRevealsNoToken pins it. + if _, err := s.db.Exec(`VACUUM`); err != nil { + return n, fmt.Errorf("vacuum after sealing config tokens: %w", err) + } + s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + } + return n, nil +} + +// configTokenUnseal is one customer_configs row whose sealed token leaves were opened. +type configTokenUnseal struct { + id, sealed, plain string + tokens int +} + +// unsealConfigTokenRows is UnsealBoxSecrets' half for config_json: for every row holding a sealed token +// it returns the row with every token opened — or an error if any one does not open (all or nothing; +// nothing is written here). +func (s *Store) unsealConfigTokenRows() ([]configTokenUnseal, error) { + rows, err := s.db.Query(`SELECT customer_id, config_json FROM customer_configs WHERE config_json LIKE '%enc:v1:%'`) + if err != nil { + return nil, err + } + var all []configTokenUnseal + for rows.Next() { + var r configTokenUnseal + if err := rows.Scan(&r.id, &r.sealed); err != nil { + rows.Close() + return nil, err + } + all = append(all, r) + } + rows.Close() + var out []configTokenUnseal + for _, r := range all { + plain, k, err := mapConfigTokens(r.sealed, func(path, v string) (string, bool, error) { + if !strings.HasPrefix(v, sealPrefix) { + return v, false, nil + } + pt, err := s.openSecret(v) + if err != nil { + return "", false, fmt.Errorf("customer_configs.config_json.%s for %s does not open — nothing was changed: %w", path, r.id, err) + } + return pt, true, nil + }) + if err != nil { + return nil, err + } + if k > 0 { + r.plain, r.tokens = plain, k + out = append(out, r) + } + } + return out, nil +} + +// RawConfigJSONForTest returns customer_configs.config_json exactly as stored (still sealed), for the +// web package's TestR975_EveryFormSecretIsSealed. Not used by the hub. +func (s *Store) RawConfigJSONForTest(customerID string) (string, error) { + var v string + err := s.db.QueryRow(`SELECT config_json FROM customer_configs WHERE customer_id = ?`, customerID).Scan(&v) + return v, err +} diff --git a/hub/internal/store/r975_config_token_seal_test.go b/hub/internal/store/r975_config_token_seal_test.go new file mode 100644 index 00000000..770f4f32 --- /dev/null +++ b/hub/internal/store/r975_config_token_seal_test.go @@ -0,0 +1,287 @@ +package store + +import ( + "bytes" + "encoding/json" + "io" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +// R-975: the tunnel, Cloudflare API and git tokens inside customer_configs.config_json are sealed at +// rest (r975_config_token_seal.go). Canaries only — never a real token. + +const ( + r975Tunnel = "tunnel-token-canary-eyJhIjoiYWJjIn0-r975" + r975CFAPI = "cf-api-token-canary-0f1e2d3c4b5a-r975" + r975Git = "git-token-canary-9a8b7c6d5e4f-r975" +) + +var r975Canaries = []string{r975Tunnel, r975CFAPI, r975Git} + +// r975ConfigJSON is the shape buildConfigJSON (web/configs.go) writes, plus a non-secret field and an +// offsite descriptor the offsite package merges in later. +func r975ConfigJSON() string { + return `{"git":{"token":"` + r975Git + `","username":"felhom-bot"},` + + `"infrastructure":{"cf_api_token":"` + r975CFAPI + `","cf_tunnel_token":"` + r975Tunnel + `"},` + + `"logging":{"level":"debug"},"offsite":{"quota_gb":100,"user":"u12345-sub3"}}` +} + +func r975Leaves(t *testing.T, js string) (tunnel, cfapi, git string) { + t.Helper() + var m struct { + Git struct{ Token string } `json:"git"` + Infrastructure struct { + CFAPIToken string `json:"cf_api_token"` + CFTunnelToken string `json:"cf_tunnel_token"` + } `json:"infrastructure"` + } + if err := json.Unmarshal([]byte(js), &m); err != nil { + t.Fatalf("config_json %q: %v", js, err) + } + return m.Infrastructure.CFTunnelToken, m.Infrastructure.CFAPIToken, m.Git.Token +} + +func r975RawConfig(t *testing.T, st *Store, id string) string { + t.Helper() + return r879Raw(t, st, `SELECT config_json FROM customer_configs WHERE customer_id = ?`, id) +} + +// A fresh write through SaveCustomerConfig stores each token sealed; the rest of the JSON stays readable. +func TestR975_FreshWriteIsStoredSealed(t *testing.T) { + st := sealTestStore(t) + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c1", RetrievalPassword: "p", APIKey: "k", ConfigJSON: r975ConfigJSON()}); err != nil { + t.Fatal(err) + } + raw := r975RawConfig(t, st, "c1") + for _, c := range r975Canaries { + if strings.Contains(raw, c) { + t.Errorf("config_json holds a plaintext token %q", c) + } + } + tun, api, git := r975Leaves(t, raw) + for name, v := range map[string]string{"cf_tunnel_token": tun, "cf_api_token": api, "git.token": git} { + if !strings.HasPrefix(v, sealPrefix) { + t.Errorf("%s is not sealed: %q", name, v) + } + } + for _, plain := range []string{`"level":"debug"`, `"user":"u12345-sub3"`, `"quota_gb":100`, `"username":"felhom-bot"`} { + if !strings.Contains(raw, plain) { + t.Errorf("non-secret field %s lost or changed in %q", plain, raw) + } + } + // No key → a config with a token is refused, never written in plaintext (§16.2). + st.sealer = nil + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c2", ConfigJSON: r975ConfigJSON()}); err != ErrNoSealKey { + t.Fatalf("SaveCustomerConfig with a token and no key = %v, want ErrNoSealKey", err) + } + // ...while a config with no token needs no key for its JSON (empty secrets stay empty). + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c3", ConfigJSON: `{"logging":{"level":"debug"}}`}); err != nil { + t.Fatalf("SaveCustomerConfig with no token and no key = %v", err) + } + if got := r975RawConfig(t, st, "c3"); got != `{"logging":{"level":"debug"}}` { + t.Fatalf("a token-free config_json was rewritten: %q", got) + } +} + +// Every reader gets back exactly the value that was written. +func TestR975_ReaderGetsTheSameValue(t *testing.T) { + st := sealTestStore(t) + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c1", RetrievalPassword: "p", APIKey: "cust-key-r975", ConfigJSON: r975ConfigJSON()}); err != nil { + t.Fatal(err) + } + check := func(where string, cfg *CustomerConfig) { + t.Helper() + if cfg == nil || cfg.SecretsUnreadable { + t.Fatalf("%s: %+v", where, cfg) + } + tun, api, git := r975Leaves(t, cfg.ConfigJSON) + if tun != r975Tunnel || api != r975CFAPI || git != r975Git { + t.Fatalf("%s: tokens read back wrong (tunnel ok=%v api ok=%v git ok=%v)", where, tun == r975Tunnel, api == r975CFAPI, git == r975Git) + } + } + cfg, err := st.GetCustomerConfig("c1") + if err != nil { + t.Fatal(err) + } + check("GetCustomerConfig", cfg) + byKey, err := st.GetCustomerConfigByAPIKey("cust-key-r975") + if err != nil { + t.Fatal(err) + } + check("GetCustomerConfigByAPIKey", byKey) + list, err := st.ListCustomerConfigs() + if err != nil || len(list) != 1 { + t.Fatalf("ListCustomerConfigs = %d, %v", len(list), err) + } + check("ListCustomerConfigs", &list[0]) + // Load-modify-save (every operator form does this): the value survives and is sealed again, not + // double-sealed. + cfg.CustomerName = "renamed" + if err := st.SaveCustomerConfig(cfg); err != nil { + t.Fatal(err) + } + again, _ := st.GetCustomerConfig("c1") + check("after a load-modify-save", again) + if raw := r975RawConfig(t, st, "c1"); strings.Contains(raw, r975Tunnel) { + t.Fatal("a load-modify-save wrote the tunnel token in plaintext") + } +} + +// The start-up migration seals rows a pre-R-975 hub wrote, once; a second start is a no-op. It does not +// bump config_version (the box's config is unchanged — no re-pull, no restart). +func TestR975_MigrationSealsOnceThenNoOp(t *testing.T) { + st := sealTestStore(t) + if _, err := st.db.Exec(`INSERT INTO customer_configs (customer_id, retrieval_password, api_key, config_json) VALUES ('c1', '', '', ?), ('c2', '', '', '{}')`, r975ConfigJSON()); err != nil { + t.Fatal(err) + } + before := r879Raw(t, st, `SELECT config_version || '|' || updated_at FROM customer_configs WHERE customer_id='c1'`) + n, err := st.SealLegacyConfigTokens() + if err != nil || n != 3 { + t.Fatalf("SealLegacyConfigTokens = %d, %v — want the three plaintext tokens", n, err) + } + raw := r975RawConfig(t, st, "c1") + for _, c := range r975Canaries { + if strings.Contains(raw, c) { + t.Fatalf("after the migration config_json still holds %q", c) + } + } + if got := r975RawConfig(t, st, "c2"); got != "{}" { + t.Fatalf("a token-free row was rewritten: %q", got) + } + if after := r879Raw(t, st, `SELECT config_version || '|' || updated_at FROM customer_configs WHERE customer_id='c1'`); after != before { + t.Fatalf("the migration bumped config_version/updated_at: %s -> %s (every box would re-pull and restart)", before, after) + } + if n, err := st.SealLegacyConfigTokens(); err != nil || n != 0 { + t.Fatalf("second SealLegacyConfigTokens = %d, %v — want a no-op", n, err) + } + cfg, _ := st.GetCustomerConfig("c1") + if tun, api, git := r975Leaves(t, cfg.ConfigJSON); tun != r975Tunnel || api != r975CFAPI || git != r975Git { + t.Fatal("tokens lost in the migration") + } + st.sealer = nil + if _, err := st.SealLegacyConfigTokens(); err != ErrNoSealKey { + t.Fatalf("SealLegacyConfigTokens without a key = %v, want ErrNoSealKey", err) + } +} + +// The consequence R-1241 cares about: a COPY OF THE DATABASE FILES, without the key, reveals no token. +// Positive control first: the rows exactly as a pre-R-975 hub wrote them DO show in the file — so the +// grep is able to see a token. Then the start-up migration runs and the same files hold none (the old +// plaintext must not survive in a freed page or in the -wal either). +func TestR975_DBFileWithoutKeyRevealsNoToken(t *testing.T) { + path := filepath.Join(t.TempDir(), "hub.db") + st, err := New(path, log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + // A second customer written AFTER c1, as on the live hub (five customers): c1's old row is then not + // the last cell on its page, so its sealed replacement does not land on top of it and the plaintext + // stays in a freed block unless the file is rewritten. + if _, err := st.db.Exec(`INSERT INTO customer_configs (customer_id, retrieval_password, api_key, config_json) VALUES ('c1', '', '', ?), ('c0', '', '', '{"logging":{"level":"debug"}}')`, r975ConfigJSON()); err != nil { + t.Fatal(err) + } + fileHolds := func(c string) bool { + t.Helper() + var all []byte + for _, f := range []string{path, path + "-wal", path + "-shm"} { + b, err := os.ReadFile(f) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + all = append(all, b...) + } + return bytes.Contains(all, []byte(c)) + } + for _, c := range r975Canaries { + if !fileHolds(c) { + t.Fatalf("positive control: the pre-R-975 file does not show %q — the grep cannot see a token", c) + } + } + if n, err := st.SealLegacyConfigTokens(); err != nil || n != 3 { + t.Fatalf("SealLegacyConfigTokens = %d, %v", n, err) + } + // A fresh write after the upgrade, too. + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c2", RetrievalPassword: "p", APIKey: "k2", ConfigJSON: strings.ReplaceAll(r975ConfigJSON(), "r975", "r975b")}); err != nil { + t.Fatal(err) + } + for _, c := range append(r975Canaries, strings.ReplaceAll(r975Tunnel, "r975", "r975b"), strings.ReplaceAll(r975Git, "r975", "r975b")) { + if fileHolds(c) { + t.Errorf("a copy of the database files still shows %q", c) + } + } +} + +// A wrong or missing key at a restart: the token is blanked, the record is flagged, and it cannot be +// saved back (which would erase the stored token). The right key brings it back intact. +func TestR975_WrongKeyBlanksAndRefusesSave(t *testing.T) { + st := sealTestStore(t) + // No passphrase and no API key: only the tokens are sealed, so the flag below can come from them alone. + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c1", ConfigJSON: r975ConfigJSON()}); err != nil { + t.Fatal(err) + } + if err := st.SetOffsiteSecretKey([]byte("another-key-of-exactly-32-bytes!")); err != nil { + t.Fatal(err) + } + cfg, err := st.GetCustomerConfig("c1") + if err != nil || cfg == nil || !cfg.SecretsUnreadable { + t.Fatalf("GetCustomerConfig with a wrong key = %+v, %v — want SecretsUnreadable", cfg, err) + } + if strings.Contains(cfg.ConfigJSON, sealPrefix) { + t.Fatalf("a sealed token that did not open is passed on: %q", cfg.ConfigJSON) + } + if tun, _, _ := r975Leaves(t, cfg.ConfigJSON); tun != "" { + t.Fatal("the tunnel token is not blanked") + } + if err := st.SaveCustomerConfig(cfg); err == nil { + t.Fatal("a config whose tokens did not open was saved back — it would blank them") + } + if err := st.SetOffsiteSecretKey([]byte("felhom-hub-test-only-seal-key-32")); err != nil { + t.Fatal(err) + } + cfg, _ = st.GetCustomerConfig("c1") + if tun, _, _ := r975Leaves(t, cfg.ConfigJSON); tun != r975Tunnel || cfg.SecretsUnreadable { + t.Fatal("the stored tunnel token was damaged while the key was wrong") + } +} + +// The roll-back: `felhom-hub -unseal-box-secrets` opens the config tokens too, so a hub older than +// 0.168.0 (which serves config_json as it is stored) hands the box the real tunnel token again. +func TestR975_UnsealRestoresPlaintextForAnOlderHub(t *testing.T) { + st := sealTestStore(t) + r879Seed(t, st) + if err := st.SaveCustomerConfig(&CustomerConfig{CustomerID: "c9", RetrievalPassword: "p9", APIKey: "k9", ConfigJSON: r975ConfigJSON()}); err != nil { + t.Fatal(err) + } + before := r975RawConfig(t, st, "c9") + if err := st.SetOffsiteSecretKey([]byte("another-key-of-exactly-32-bytes!")); err != nil { + t.Fatal(err) + } + if n, err := st.UnsealBoxSecrets(); err == nil || n != 0 { + t.Fatalf("UnsealBoxSecrets with a wrong key = %d, %v — want a refusal", n, err) + } + if r975RawConfig(t, st, "c9") != before { + t.Fatal("a refused unseal changed config_json") + } + if err := st.SetOffsiteSecretKey([]byte("felhom-hub-test-only-seal-key-32")); err != nil { + t.Fatal(err) + } + // r879Seed's four values, c9's two sealed columns, and its three tokens. + if n, err := st.UnsealBoxSecrets(); err != nil || n != 9 { + t.Fatalf("UnsealBoxSecrets = %d, %v — want 4 + 2 + 3", n, err) + } + tun, api, git := r975Leaves(t, r975RawConfig(t, st, "c9")) + if tun != r975Tunnel || api != r975CFAPI || git != r975Git { + t.Fatal("after the unseal the raw config_json does not hold the plaintext tokens an older hub serves") + } + if n, err := st.UnsealBoxSecrets(); err != nil || n != 0 { + t.Fatalf("second UnsealBoxSecrets = %d, %v — want a no-op", n, err) + } + if n, err := st.SealLegacyConfigTokens(); err != nil || n != 3 { + t.Fatalf("re-seal after unseal = %d, %v", n, err) + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 6d385ce9..356f05f9 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -1699,6 +1699,11 @@ func (s *Store) SaveCustomerConfig(cfg *CustomerConfig) error { if err != nil { return err } + // R-975: the tunnel, Cloudflare API and git tokens inside config_json are sealed too. + sealedJSON, _, err := s.sealConfigJSONTokens(cfg.ConfigJSON) + if err != nil { + return err + } _, err = s.db.Exec(` INSERT INTO customer_configs (customer_id, customer_name, domain, email, retrieval_password, api_key, api_key_hash, config_json, min_controller_version, dr_tier, language, config_version, updated_at) @@ -1717,19 +1722,21 @@ func (s *Store) SaveCustomerConfig(cfg *CustomerConfig) error { config_version = customer_configs.config_version + 1, updated_at = datetime('now')`, cfg.CustomerID, cfg.CustomerName, cfg.Domain, cfg.Email, - sealedPass, sealedKey, keyHash, cfg.ConfigJSON, cfg.MinControllerVersion, cfg.DRTier, + sealedPass, sealedKey, keyHash, sealedJSON, cfg.MinControllerVersion, cfg.DRTier, createdLanguage(cfg.Language), ) return err } -// openCustomerSecrets opens the two sealed columns of a scanned config in place (R-879). A value that -// does not open leaves the field "" and sets SecretsUnreadable; legacy plaintext passes through. +// openCustomerSecrets opens the two sealed columns of a scanned config in place (R-879), and the sealed +// token leaves of config_json (R-975). A value that does not open leaves the field (or the leaf) "" and +// sets SecretsUnreadable; legacy plaintext passes through. func (s *Store) openCustomerSecrets(cfg *CustomerConfig) { - var ok1, ok2 bool + var ok1, ok2, ok3 bool cfg.RetrievalPassword, ok1 = s.openAtRest("customer_configs.retrieval_password", cfg.CustomerID, cfg.RetrievalPassword) cfg.APIKey, ok2 = s.openAtRest("customer_configs.api_key", cfg.CustomerID, cfg.APIKey) - cfg.SecretsUnreadable = !ok1 || !ok2 + cfg.ConfigJSON, ok3 = s.openConfigJSONTokens(cfg.CustomerID, cfg.ConfigJSON) + cfg.SecretsUnreadable = !ok1 || !ok2 || !ok3 } // createdLanguage keeps only a language the hub can write mail in, defaulting to Hungarian. diff --git a/hub/internal/web/r975_form_tokens_test.go b/hub/internal/web/r975_form_tokens_test.go new file mode 100644 index 00000000..b54efa34 --- /dev/null +++ b/hub/internal/web/r975_form_tokens_test.go @@ -0,0 +1,70 @@ +package web + +import ( + "encoding/json" + "io" + "log" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-975: every secret the operator's edit form writes into config_json is stored sealed. The form is +// posted with EVERY field buildConfigJSON reads; each leaf of the stored JSON must then be either on the +// short non-secret list below or sealed. A new secret field added to the form without adding its path +// to store.configTokenPaths fails here (its leaf is neither). +func TestR975_EveryFormSecretIsSealed(t *testing.T) { + st, err := store.New(filepath.Join(t.TempDir(), "t.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + form := "cf_tunnel_token=tunnel-canary-r975&cf_api_token=cfapi-canary-r975&git_username=bot&git_token=git-canary-r975&debug_mode=on" + js := buildConfigJSON(postForm("/configs/c1/edit", form)) + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", RetrievalPassword: "p", APIKey: "k", ConfigJSON: js}); err != nil { + t.Fatal(err) + } + raw, err := st.RawConfigJSONForTest("c1") + if err != nil { + t.Fatal(err) + } + for _, c := range []string{"tunnel-canary-r975", "cfapi-canary-r975", "git-canary-r975"} { + if strings.Contains(raw, c) { + t.Errorf("stored config_json holds the form's secret %q in plaintext", c) + } + } + notSecret := map[string]bool{"git.username": true, "logging.level": true} + var m map[string]any + if err := json.Unmarshal([]byte(raw), &m); err != nil { + t.Fatal(err) + } + leaves := 0 + var walk func(p string, v any) + walk = func(p string, v any) { + switch x := v.(type) { + case map[string]any: + for k, vv := range x { + walk(strings.TrimPrefix(p+"."+k, "."), vv) + } + case string: + leaves++ + if !notSecret[p] && !strings.HasPrefix(x, "enc:v1:") { + t.Errorf("config_json leaf %s from the form is neither on the non-secret list nor sealed", p) + } + } + } + walk("", m) + if leaves != 5 { + t.Fatalf("the form wrote %d leaves, want 5 — a field was added or lost; classify it here", leaves) + } + // And the hub reads the plaintext back (storedCFToken feeds the Cloudflare zone check). + cfg, err := st.GetCustomerConfig("c1") + if err != nil || cfg == nil { + t.Fatal(err) + } + if got := storedCFToken(cfg.ConfigJSON); got != "cfapi-canary-r975" { + t.Fatalf("storedCFToken after the round trip = %q", got) + } +}