From 88aecf55b618ea95c97ff3b6857f19193b91a265 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 18 Oct 2026 12:50:49 +0200 Subject: [PATCH] Installer step 4b: vault first, then set, then a PVE login (R-1145); host key off curl's argv (review F4); harness green and a gate (R-1146); hub 0.158.0 deploy evidence; independent review + verdicts Live on demo-hp (--rotate-recovery-only): vault time moved, new value 200, old 401. Harness 52/0; the R1145 cases fail against HEAD. Publishes only with the next installer tag. --- .../day-2026-10-18/P1/access-log-files.txt | 5 ++ .../P1/access-log-live-test.txt | 12 +++ .../day-2026-10-18/P1/day0-page-readonly.txt | 9 ++ .../day-2026-10-18/P1/hub-deploy-readback.txt | 7 ++ .../day-2026-10-18/P2/independent-review.md | 54 ++++++++++++ .../audits/day-2026-10-18/P2/verdicts.md | 13 +++ .../day-2026-10-18/P3/redproof-r1144.txt | 11 +++ .../day-2026-10-18/P5/demo-hp-rotation.txt | 22 +++++ .../day-2026-10-18/P5/redproof-r1145.txt | 12 +++ documentation/backlog/OPEN-ITEMS.md | 2 +- documentation/runbooks/break-glass.md | 3 +- scripts/CHANGELOG.md | 16 ++++ scripts/felhom-host-install.sh | 85 +++++++++++++----- scripts/hostinstall-mode-harness.sh | 86 ++++++++++++++++++- scripts/hostinstall_harness_gate.py | 47 ++++++++++ scripts/repo_gates.py | 3 + scripts/test_gate_decoys.py | 14 +++ 17 files changed, 374 insertions(+), 27 deletions(-) create mode 100644 documentation/audits/day-2026-10-18/P1/access-log-files.txt create mode 100644 documentation/audits/day-2026-10-18/P1/access-log-live-test.txt create mode 100644 documentation/audits/day-2026-10-18/P1/day0-page-readonly.txt create mode 100644 documentation/audits/day-2026-10-18/P1/hub-deploy-readback.txt create mode 100644 documentation/audits/day-2026-10-18/P2/independent-review.md create mode 100644 documentation/audits/day-2026-10-18/P2/verdicts.md create mode 100644 documentation/audits/day-2026-10-18/P3/redproof-r1144.txt create mode 100644 documentation/audits/day-2026-10-18/P5/demo-hp-rotation.txt create mode 100644 documentation/audits/day-2026-10-18/P5/redproof-r1145.txt create mode 100644 scripts/hostinstall_harness_gate.py diff --git a/documentation/audits/day-2026-10-18/P1/access-log-files.txt b/documentation/audits/day-2026-10-18/P1/access-log-files.txt new file mode 100644 index 00000000..2f847fcf --- /dev/null +++ b/documentation/audits/day-2026-10-18/P1/access-log-files.txt @@ -0,0 +1,5 @@ +total 12 +drwx------ 2 root root 4096 Oct 18 12:25 . +drwxr-xr-x 6 root root 4096 Oct 18 12:25 .. +-rw------- 1 root root 1120 Oct 18 12:27 access-2026-10-18.log +10 /data/access-log/access-2026-10-18.log diff --git a/documentation/audits/day-2026-10-18/P1/access-log-live-test.txt b/documentation/audits/day-2026-10-18/P1/access-log-live-test.txt new file mode 100644 index 00000000..609fdec6 --- /dev/null +++ b/documentation/audits/day-2026-10-18/P1/access-log-live-test.txt @@ -0,0 +1,12 @@ +== live redaction test 2026-10-18T10:28:49Z: a fresh fake token (48 hex) sent in a query, a /bind/ path, a session query, a Bearer header, a Cookie, a POST body +lines in today's file: 15 +the fake token in the log: 0 +control (log + the token appended): 1 +secret shapes (Bearer value, 32+ hex/base64 run, password=, token=, session=): 0 +control for the shapes on the appended copy: 1 +== my test lines (path fields only): +GET /healthz 200 cred=- auth=- +GET /bind/[redacted] 200 cred=- auth=- +GET /configuration 302 cred=- auth=- +POST /login 401 cred=- auth=- +== distinct box sources of /api/v1/report: xff=192.168.0.101 xff=192.168.0.149 xff=192.168.0.155 diff --git a/documentation/audits/day-2026-10-18/P1/day0-page-readonly.txt b/documentation/audits/day-2026-10-18/P1/day0-page-readonly.txt new file mode 100644 index 00000000..b24e2322 --- /dev/null +++ b/documentation/audits/day-2026-10-18/P1/day0-page-readonly.txt @@ -0,0 +1,9 @@ +== GET /configuration with CC's token (read-only): 25000 bytes + 2 0.159.1 + 3 ArtifactSha + 1 Day-0 + 1 Day-0 artifacts — agent & golden + 3 artifact + 1 artifacts + 1 golden (v0.329.0 +POST /configuration/artifacts with the token: 403 (the vouch is the operator's, by design) diff --git a/documentation/audits/day-2026-10-18/P1/hub-deploy-readback.txt b/documentation/audits/day-2026-10-18/P1/hub-deploy-readback.txt new file mode 100644 index 00000000..007de170 --- /dev/null +++ b/documentation/audits/day-2026-10-18/P1/hub-deploy-readback.txt @@ -0,0 +1,7 @@ +integrity: ok +('artifact_agent_version', '0.159.1') +('artifact_golden_version', '0.329.0') +('artifact_min_agent', '0.158.1') +('min_controller_version', '0.329.0') +('min_controller_version_declared_min_agent', '0.329.0=0.158.1') +host_reports cols: ['id', 'host_id', 'customer_id', 'received_at', 'report_json', 'agent_version', 'cpu_percent', 'memory_percent', 'disk_percent', 'guest_total', 'guest_running', 'cloudflared_status'] diff --git a/documentation/audits/day-2026-10-18/P2/independent-review.md b/documentation/audits/day-2026-10-18/P2/independent-review.md new file mode 100644 index 00000000..a221da63 --- /dev/null +++ b/documentation/audits/day-2026-10-18/P2/independent-review.md @@ -0,0 +1,54 @@ +# Independent security review — P2, 2026-10-18 + +Reviewer: independent (did not write the code). Read-only. Adversarial threat model per the brief. + +Diffs reviewed: +- felhom-controller `5d67041..03ab121` (v0.328.0 → v0.329.0): `internal/privpolicy`, `internal/stacks/allowlist.go`, `internal/web/storage_handlers.go`, `internal/agentapi/client.go`, `build.sh`. +- app-catalog-felhom.eu `0140fff..fc93846`: `scripts/check-privileges.py`, `templates/homepage`. +- felhom.eu hub `7d64038f`: agent-only vouch (`internal/web/configs.go`), `internal/accesslog`, `internal/offsitekeys/snapshot_watch.go` + `service.go` + `store/offsite_snapshots.go`, the `accesslog.Mark` call sites. +- felhom-agent `6545ccc`: orphan thin-check only after a failed snapshot step. +- felhom.eu installer `7d64038f`: `--rotate-recovery-only`. + +Architecture consulted: `02-controller-module-map.md` "The catalog allow-list"; `05-hub-architecture.md` (Day-0 manifest, R-120 guard, §16.4); `07-backup-architecture.md` (off-site custody, R-822/R-895); `runbooks/break-glass.md`. + +--- + +## Findings + +### F1 (medium) `scripts/felhom-host-install.sh` ~1767–1783 (the `--rotate-recovery-only` healthz pre-check) + `step_break_glass` ~1657 +**The pre-flight that is supposed to stop a rotation that cannot be vaulted uses the wrong credential, so a stale/invalid host key leaves `root@pam` rotated-but-unvaulted.** + +Scenario: operator runs `--rotate-recovery-only` on an appliance box whose `hub.api_key` in `agent.json` is stale (host deleted/re-enrolled at the hub, or a copied/old config). The mode's only gate before the destructive `chpasswd` is `http_code "$HUB_URL/healthz"` — an **unauthenticated** endpoint. healthz returns 200, so the script proceeds; `step_break_glass` generates a new random password, sets it with `chpasswd`, then `PUT /hosts/$HOST_ID/recovery-credential` with `Bearer $HOST_API_KEY` returns 401/403 → `die`. At that point `root@pam` has already been changed to a random string that was never vaulted; the hub still holds the *old* credential, which no longer works. The PVE web console (:8006) — the layer-3 break-glass path this mode exists to maintain — is now unreachable with any known password. + +The code comment claims the healthz check covers "a vault that fails AFTER chpasswd leaves a password nobody holds", but healthz exercises **no** auth, whereas the vault PUT needs the host key — a different channel, so the pre-check cannot catch the failure it is meant to prevent (the same class as the R-286 "control from a different channel" rule). In the full install this is masked because `step_enroll` validates the host key before `step_break_glass`; the new standalone mode has no enroll, so nothing confirms the key is good before the irreversible `chpasswd`. + +Fix direction: before `chpasswd`, make one **host-key-authenticated** call (e.g. `GET /api/v1/hosts/$HOST_ID/desired-state` with the Bearer key) and refuse the rotation if it is not 2xx. Confidence: confirmed by reading. + +### F2 (low–medium) `hub/internal/offsitekeys/snapshot_watch.go:62–73` (`explainedByRun`) + `hub/internal/store/offsite_snapshots.go` (`OffsiteRunTimesSince`) +**The "unexplained snapshot" (fake-injection) alarm trusts the box's own self-reported run times, so an attacker holding the box's key can suppress it.** + +`offsite_snapshots_unexplained` fires only when a newly-appeared snapshot's upload time is **not** near any `offsite.last_run` the box reported. But `OffsiteRunTimesSince` reads those run times from the customer's `report_json` — data produced by the same guest an attacker who holds the append-only key typically already controls. An attacker who uploads a fabricated snapshot through the box's add-only key can also have the box report a `last_run` whose time brackets the injected file's mtime, so `explainedByRun` returns true and no warning is raised. The injection signature the alarm is built to catch is defeatable by the very actor it targets. + +This is partly acknowledged in the design ("Fakes added with the box's key would look like this; a crashed run would too"), and the custody-critical half — `offsite_snapshots_vanished` — is **not** affected: it is judged on the hub's own name listing against hub-side window credits (`WindowCreditsBetween`), a channel the box cannot forge, and correctly uses the full listing `l.Entries` (not the settle-filtered `next`) for the "still present" test, so a too-new file is not mistaken for a deletion. Reported because the brief explicitly asks about faking the off-site alarms; the deletion alarm is the one that matters most and it is sound. Confidence: confirmed by reading. + +### F3 (low) `controller/internal/web/storage_handlers.go:544–572` ("anyway") + `finalizeDecommissionWith` ~608 +**Decommission-anyway stops the drive's apps before the agent is asked; on an agent policy refusal the apps are left stopped with nothing retired (half-state).** + +The handler calls `StopStack` on every app whose `HDD_PATH` is the target drive, then `finalizeDecommission`. With the R-1063 reorder the agent is now asked first and can refuse with 403 (protected role) or 409 (whole-guest backup target); on that refusal the function returns and nothing is marked — but the apps were already stopped and are not restarted, so a household that presses "decommission anyway" on a drive that is also the backup target ends with its apps down and a 409. This is **not newly introduced** (the stop always preceded `finalizeDecommission`), and the reorder is a net improvement for the mark/agent consistency the comment describes (previously the registry was marked decommissioned while the agent kept serving). Noted as a residual half-state the retirement change did not close. Confidence: confirmed by reading. + +### F4 (low) `scripts/felhom-host-install.sh` `step_break_glass` ~1685 (unchanged, moved) +**The host API key is passed in `curl` argv (`-H "Authorization: Bearer $HOST_API_KEY"`) during the vault PUT, visible in process args while the call runs.** The password is handled correctly (stdin→chpasswd, stdin→`curl -d @-`, scrubbed), but the host key rides on the command line. On a PVE host this is root-only and the code is moved verbatim from the original step, so it is pre-existing, not a regression. Worth a line because `--rotate-recovery-only` makes this a standalone, operator-run path. Confidence: confirmed by reading. + +--- + +## Checked and found sound + +- **Per-app appdata root (R-1142).** `AppdataFolder` narrows `${HDD_PATH}` to `appdata/`; a template named like another app's mapped folder (e.g. `paperless` vs `paperless-ngx` → `paperless`) gets **no** root (reverse-lookup over `appdataFolders` values), an app with no usable name gets no root (fail-closed), `oneSegment` rejects `/ \ $ . .. ""`, and a duplicate folder mapping panics at load. Another app's folder/sub-folder/the whole `appdata`/a prefix-collision name/a literal path to another app are all refused; tests pin each case. +- **Catalog CI parity.** `scripts/privilege-allowlist.json` is byte-identical to the compiled `allowlist.json` (both v2, empty entries, `paperless-ngx→paperless`); `check-privileges.py`'s `one_segment`/`appdata_folder`/reverse-lookup/duplicate-folder checks mirror the Go, and `app` = the template's top-level dir name, matching the box's `appName`. +- **USERDATA_PATH can no longer be named by a template.** `policyEnv` deletes any app-env `USERDATA_PATH` and re-derives it; the **runtime** converges on the same value — Go `exec` dedups `cmd.Env` to the last occurrence and `envLookup`/`ensureUserdataMounts`/`renderMediaOverride` all read the last match, which is the `withPathVars`-appended derivation, so an injected `app.yaml USERDATA_PATH` wins neither the check nor the bind. `IMPORT_PATH` is likewise always controller-derived or unset. +- **Docker socket is refused everywhere.** Empty allow-list ⇒ any `docker.sock` bind (short/long syntax, `/run` or `/var/run`, `.sock` suffix, or a symlink whose `EvalSymlinks` target leaves the root) is a violation with no allowed entry; homepage's template no longer mounts it and `TestAllowList_HasNoExceptionAndRefusesHomepagesSocket` red-proofs the removal. Controller-infra stack names are refused in `copyTemplates`. +- **Agent-only vouch.** The R-120 gate is skipped only when the submitted golden equals the vouched one in **both** version and sha256; a changed version *or* a same-version re-push (different sha, freshly resolved from the registry) is still judged and refused if behind the fleet (test-pinned). The R-273 tag+package installability gate, the bundle-sha resolve, and operator-only `RequireAuth`/CSRF are all intact; MinAgent floor logic is untouched. Nothing was loosened that lets a stale/wrong golden be saved. +- **Access log.** Query string dropped entirely; path segments ≥32 chars and the `/bind/` capability redacted; `cred`/`auth` record only the kind, never a value (`Authorization`, session cookie, `X-Retrieval-Password` → names only). Recovery-credential PUT keeps the password to body + hub log prints only user+length. A logging failure never fails/delays the request (non-blocking channel, rate-limited error); prune removes only exact `access-YYYY-MM-DD.log` names. No request shape observed that lands a secret in a line. +- **Agent orphan thin-check reorder.** `thinCheck` now runs only from `explainFailure` (vzdump start/task failure), bounded by a 2-minute `context.WithoutCancel` timeout, read-only; the success path no longer snapshots the live pool. Capability manifest description updated to match. +- **build.sh.** `:latest` moves only for a release tag matching `^[0-9]+\.[0-9]+\.[0-9]+$`; rc/dev tags push only their own tag. +- **agentapi typed refusals.** `ErrNoDurableID` (409 + "has no durable id") and `DecommissionRefusedError` (403→protected, 409→backup-target) carry only the agent's truncated reason text, never bodies/secrets; the agent-side role gate fails safe to protected on an unresolvable mount. diff --git a/documentation/audits/day-2026-10-18/P2/verdicts.md b/documentation/audits/day-2026-10-18/P2/verdicts.md new file mode 100644 index 00000000..a09a179d --- /dev/null +++ b/documentation/audits/day-2026-10-18/P2/verdicts.md @@ -0,0 +1,13 @@ +# Verdicts on the independent review (2026-10-18) — each checked against source by the attending session + +The report: `independent-review.md` (this folder; written by a fresh reviewer with no context from the night; read back +here — 54 lines, 4 findings). + +| # | Finding | Verdict | What was done | +|---|---|---|---| +| F1 (medium) | `--rotate-recovery-only` checks only an unauthenticated `/healthz`; a stale host key → chpasswd runs → the vault PUT 401s → root holds an unvaulted password | **REAL in the night's code** (`felhom-host-install.sh` step 4b set the password first). | **Fixed by R-1145 (today):** the vault comes FIRST, so a stale key stops at the PUT and the password is not touched. Harness `R1145-e` (vault 401 → no chpasswd); red-proof: it FAILS against HEAD (`P5/redproof-r1145.txt`). | +| F2 (low–medium) | `offsite_snapshots_unexplained` is explained by the box's OWN reported `offsite.last_run` — a box (or whoever holds its key) can report a matching run and silence the warning | **REAL, by design today** (decision 68 accepts a box-trusted run time; the custody-critical `offsite_snapshots_vanished` is judged on the hub's own listing and window credits and is not affected — the reviewer confirms). | Recorded in R-895's row (its design §3.1/§3.2 remainder is where an independent upload-time check belongs); not a new row. | +| F3 (low) | „anyway" stops the drive's apps before the agent is asked; a refusal leaves them stopped | **REAL, already filed** (R-1144). | **Fixed today** (controller): the agent's own disk list is asked first; a refusal after the stops starts exactly the apps that were running. `TestR1144_*`, red-proofed (`P3/`). | +| F4 (low) | step 4b passes the host api_key in curl's argv (`-H "Authorization: Bearer …"`), visible in `ps` | **REAL**, pre-existing (moved verbatim). | **Fixed today:** the header rides a 0600 temp file written by the printf builtin (`-H @file`), removed after. The harness asserts the key is in no fake's argv (all R1145 cases). | + +Nothing else was reported. The reviewer's „checked and found sound" list is in its report. diff --git a/documentation/audits/day-2026-10-18/P3/redproof-r1144.txt b/documentation/audits/day-2026-10-18/P3/redproof-r1144.txt new file mode 100644 index 00000000..167be61f --- /dev/null +++ b/documentation/audits/day-2026-10-18/P3/redproof-r1144.txt @@ -0,0 +1,11 @@ +== RED-PROOF R-1144: no restart after a refusal + r1144_decommission_anyway_test.go:62: HTTP 403: starts = [] (all=true), want exactly the apps that were running +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/web 0.011s +FAIL + +== RED-PROOF R-1144: no agent verdict before the stops + r1144_decommission_anyway_test.go:86: protected role: err= stops=[immich jellyfin komga] agent calls=[/mnt/old] — want a refusal before anything is stopped +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/web 0.011s +FAIL diff --git a/documentation/audits/day-2026-10-18/P5/demo-hp-rotation.txt b/documentation/audits/day-2026-10-18/P5/demo-hp-rotation.txt new file mode 100644 index 00000000..b54587f1 --- /dev/null +++ b/documentation/audits/day-2026-10-18/P5/demo-hp-rotation.txt @@ -0,0 +1,22 @@ +== before 2026-10-18T10:39:55Z +demo-hp-bb76ea HTTP 200 user root@pam set_at 2026-07-21T16:24:27Z password_len 32 keys ['host_id', 'set_at', 'username'] +== rotate (the installer from this session's tree, --rotate-recovery-only, on demo-hp) 2026-10-18T10:43:33Z + +[INFO] felhom-host-install v1.33.0 — mode=rotate-recovery-only + +[INFO] agent config: /etc/felhom-agent/agent.json +[INFO] host_id: demo-hp-bb76ea (host key read from the agent config, not logged); hub: https://hub.felhom.eu +[INFO] hub answers (healthz 200) +[STEP] 4b/8 break-glass credential (root@pam console password → hub vault) +[INFO] recovery password vaulted at the hub (stored) — now setting it on root@pam +[OK] root@pam password vaulted, set, and the PVE login takes it (retrieve via the operator /admin path; never logged here) +[WARN] NOTE: the root@pam password just CHANGED — the old one now fails at the PVE web GUI (:8006). +[WARN] Retrieve the new one at hub → host page (vaulted recovery credential). + +[OK] ROTATE-RECOVERY-ONLY complete — only root@pam's password changed (and its hub vault); agent, config, root files and guests untouched. +exit=0 +== after +demo-hp-bb76ea HTTP 200 user root@pam set_at 2026-10-18T10:43:33Z password_len 32 keys ['host_id', 'set_at', 'username'] +PVE ticket with the new value: 200 +PVE ticket with the old value: 401 +values differ: yes diff --git a/documentation/audits/day-2026-10-18/P5/redproof-r1145.txt b/documentation/audits/day-2026-10-18/P5/redproof-r1145.txt new file mode 100644 index 00000000..ea226a5c --- /dev/null +++ b/documentation/audits/day-2026-10-18/P5/redproof-r1145.txt @@ -0,0 +1,12 @@ +== RED-PROOF R-1145: the new harness cases against the installer at HEAD (set first, then vault) +=== hostinstall-mode-harness — script under test: /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/649cb06d-b4ca-49f2-9e2c-16c8bc36531f/scratchpad/old-install.sh === +FAIL R1145-a vault fails → the password is NOT touched +FAIL R1145-b vault, then set, then the PVE login — in that order +FAIL R1145-c set fails twice after the vault → the state is said, root unchanged +FAIL R1145-d the PVE login refuses the new value → die, the state is said +=== 46 passed, 5 failed, 1 skipped === + +== RED-PROOF review F1/F4 against HEAD (set before vault; key in argv) +=== hostinstall-mode-harness — script under test: /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/649cb06d-b4ca-49f2-9e2c-16c8bc36531f/scratchpad/old-install.sh === +FAIL R1145-e a stale host key (vault 401) → the password is NOT touched (review F1) +=== 46 passed, 6 failed, 1 skipped === diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index d262c169..8dff87cb 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -165,7 +165,7 @@ stopping line that lies. | **R-973** | Backup & restore | P3 | **A compromised controller can make the whole-guest backup run again and again, and each run prunes older restore points: the local tier directly, and the off-site tier through the store's own nightly keep-last-2.** So `07` §8 row 10 (ransomware) is weaker than it reads. The second reader confirmed the local half; the off-site half is inferred. Fix: a cadence or daily cap on the local-API backup route, and off-site snapshots that a nightly prune cannot remove. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). | OPEN | — | Add the cap with a test; decide the off-site retention shape | CC | | **R-232** | Backup & restore | P2 | **DooPlex's backup makes every copy inside the same box — and nothing tells anyone when it fails.** Surveyed read-only 2026-08-06 (`audits/RECON-dooplex-backup-2026-08-06.md`). **What works:** five sets, 14/14 successful runs in 14 days; a file was restored from the `data` repo and matched the live original **byte for byte**; every set except two is cross-disk; k3s is integrity-checked on every run. **What the matrix exposes, ranked:** (a) **`notify_failure` is a no-op** — `NOTIFY_ON_FAILURE=true` but `NOTIFY_WEBHOOK_URL` is commented out, so a failed backup notifies **nobody**; the project already has a working Resend path that CI uses. Cheapest item, and it makes every other failure visible. (b) **Nothing leaves the box** — no rclone, no remote repo, no off-site target anywhere; Longhorn's target is `nfs://192.168.0.180:` pointing at DooPlex itself, and the only outbound-looking cron pulls *inbound* from Hetzner for a different project. The machine that runs the hub managing the customers' off-site chain has no off-site copy of its own. (c) **The backup tree is a single writable path** and the restic repos are not append-only — one bad script or ransomware destroys every copy at once. (d) **Two same-disk sets**: `.claude-memory` and the PostgreSQL dumps, whose source directory sits *inside* the backup tree. (e) **Longhorn `retain=1`** — one generation per volume, so a corruption noticed a day late has no earlier copy. (f) **`/opt/backup/docs/BACKUP-RESTORE.md` does not exist** though the systemd unit advertises it. (g) **`secrets/restic-repo` has never held a snapshot** — `backup-secrets.sh` contains no `restic` call; the secrets are GPG files on `sda1` only. (h) **No restore has ever been run** beyond today's single-file probe — the matrix's "ever demonstrated?" column is otherwise entirely empty. **Not a finding:** the restic passphrase. The on-box copy is on `sdb1`, a different disk from the backups, and the **operator holds an offline copy out of band** — so a disk loss is recoverable. The narrow residual is that it is operator-held rather than system-held, unlike the customer case's hub-vaulted escrow, so it should be confirmed current and findable by someone else. **Nothing was changed by the recon.** | **NARROWED 2026-10-08 — owner Viktor.** (a) DONE with the operator's yes in chat: `notify_failure` now mails admin@felhom.eu through Resend; proven by one test mail that reached the inbox (`audits/day-2026-10-08/r232/`; no backup was started). (b) partly: the hub database leaves DooPlex nightly to ep0 (R-173); everything else stays on the box. (c)–(h) unchanged. **2026-10-09 (operator yes in chat): (b) and (h) DONE for Gitea and the secrets.** Nightly 00:20 `felhom-dooplex-offsite` pushes Gitea (614 MB of repositories, the `gitea` database dump taken first, `app.ini`) and the nightly GPG secrets export, encrypted with a NEW key, to ep0 `operator` (`host/dooplex-gitea`) on the hub-DB write-only token — no change on ep0; first push 57 s; restore test weekly (manifest, `git fsck` every repo, `pg_restore --list`), run once: 27 805 files, 10 repos OK; `DooplexGiteaOffsiteStale`/`DooplexGiteaRestoreTestStale` in force (`absent()` included, promtool + 2 red-proofs); every backup unit now mails on failure (`OnFailure=`, dry run reached the inbox). **First real Gitea restore:** into a throwaway on the bench, 10/10 repos, product `main` = live, a file byte for byte, a login (`runbooks/gitea-restore.md`). The hub came back too (R-173, closed). **The registry is left out on purpose:** its images rebuild from the code. **Correction to the recon §7:** the API mirror holds ONE repo (`homelab-manifests`), not all — Gitea had a single copy, on DooPlex. `audits/dooplex-survival-2026-10-09/` **LEFT:** (c) append-only for the on-box tree, (d)–(g) as written; the paper copy of the new key (operator). **READY** for the rest | — | — | operator | | **R-893** | Backup & restore | P3 | **After a failed OFF-SITE replay, the rollback pours the NEWER pre-restore copy over the OLDER volume just put back.** Read in source 2026-10-06 (R-638 option A, not measured): `internal/backup/offbox_reconstitute.go` writes the undo copy from the live (newer) database, replaces the volumes with the snapshot's older tars, then — when the replay fails — `rollbackSafetyDump` loads that newer dump over the older database volume. The loader only drops what the dump knows, so tables the newer migration removed stay; and when the snapshot's older definition was written, the rollback branch does not put the newer definition back, so the older app starts on rolled-back data; non-database volumes stay at the snapshot's state. An order change cannot fix it (the only undo is a logical dump, and its volume was replaced). Known limit in `07` §6.3. **-- 2026-10-08 design:** `audits/day-2026-10-08/design-R-893.md` — re-verified; and the screen's „your data is back as it was" (`err.backup.db_restore_failed_rolled_back`) is false for files, other volumes and the app version. Question D8 on STATUS's decision sheet. **-- 2026-10-08 14:16 operator ruling D8 (`09` §3 decision 192):** yes, both, in that order — first the app stays stopped for support (option C), next „put back exactly as it was" (option A). | **NARROWED — 2026-10-09: the first half DELIVERED in controller 0.304.0. Slice 0 (the live measurement) NOT run: 9202 has no off-site, and a dump cannot be cut by hand inside an encrypted off-site copy on Tester 1; it needs a scratch off-site repo — a plan, not a finding.** **NARROWED 2026-10-08 — the first half (D8 option C, the app held stopped) is built on controller main and ships tomorrow; what stays open is the second half, „put back exactly as it was” (option A: a pre-restore copy of the app's volumes and placed files, a fit check, disk for one copy; the slice-0 measurement on 9202 first).** **OPEN — filed 2026-10-06** **2026-10-06 night: verified in source, no code** — `offbox_reconstitute.go:758` (undo dump from the live DB), `:778-860` (files and volumes from the snapshot), `:804` (the snapshot's definition is written when its version differs), `:898` (the rollback loads the newer dump over the older volume; nothing writes the live definition back). Not a reorder fix: it needs R-638 option B (a rebuilding loader) or a pre-restore volume copy (disk cost; R-685's class). Which state a household gets after a failed off-site replay is the operator's call. Next: the 9202 measurement, then the design. | a design: R-638 option B (a loader that rebuilds instead of overlays) or a pre-restore volume copy | Measure it once on 9202 (a forced replay failure after an off-site restore over a migrated app); then a design for the operator | CC | -| **R-895** | Backup & restore | P2 | **The hub's clean-up-window check trusts the snapshot counts the box sends, so a broken-into box (or past-dated fakes added through the add-only key) can shrink the real off-site history without an alarm.** READ 2026-10-06 night in source (R-822's design): the before/after comparison uses counts the box itself reports (`hub/internal/offsitekeys/service.go:284`, `:343`); new fakes keep the count level. Decision 68 already accepts a box-trusted count. | **NARROWED 2026-10-18 — slice 2 built, not deployed** (hub v0.158.0): the 07:10 audit compares the `snapshots/` listing with yesterday's by name — files gone that no clean-up window explains → `offsite_snapshots_vanished` (error), files no reported box run explains → `offsite_snapshots_unexplained` (warning); operator-only, hub-origin (not in `allowedEventTypes`). First listing = baseline; a move-aside or RESET drops it; vanishing judged on a pinned tier only; files younger than 1 h judged the next day. Red-proofed twice (`snapshot_watch_test.go`). **Not built:** the window's before/after on the hub's own count (design §3.1) and refusing a window while unexplained files exist (§3.2). Unmeasured: the provider shell's `ls -l` time zone (± 3 h slack until then). | the operator's yes to deploy hub v0.158.0 | Deploy; read the first 07:10 lines (newest upload vs the box's last run = the time-zone calibration), shrink the slack; then decide §3.1/§3.2 | CC | +| **R-895** | Backup & restore | P2 | **The hub's clean-up-window check trusts the snapshot counts the box sends, so a broken-into box (or past-dated fakes added through the add-only key) can shrink the real off-site history without an alarm.** READ 2026-10-06 night in source (R-822's design): the before/after comparison uses counts the box itself reports (`hub/internal/offsitekeys/service.go:284`, `:343`); new fakes keep the count level. Decision 68 already accepts a box-trusted count. | **NARROWED 2026-10-18 — slice 2 built, not deployed** (hub v0.158.0): the 07:10 audit compares the `snapshots/` listing with yesterday's by name — files gone that no clean-up window explains → `offsite_snapshots_vanished` (error), files no reported box run explains → `offsite_snapshots_unexplained` (warning); operator-only, hub-origin (not in `allowedEventTypes`). First listing = baseline; a move-aside or RESET drops it; vanishing judged on a pinned tier only; files younger than 1 h judged the next day. Red-proofed twice (`snapshot_watch_test.go`). **Not built:** the window's before/after on the hub's own count (design §3.1) and refusing a window while unexplained files exist (§3.2). Unmeasured: the provider shell's `ls -l` time zone (± 3 h slack until then). **Deployed 2026-10-18 12:23 (hub v0.158.0).** The first listing (tomorrow 07:10) is a baseline; the first verdict is 2026-10-20 07:10. **Known limit (independent review 2026-10-18, F2):** `_unexplained` is explained by the box's OWN reported `offsite.last_run`, so whoever holds a box's key can silence it; `_vanished` is judged on the hub's own listing and is not affected. | the operator's yes to deploy hub v0.158.0 | Deploy; read the first 07:10 lines (newest upload vs the box's last run = the time-zone calibration), shrink the slack; then decide §3.1/§3.2 | CC | | **R-923** | Backup & restore | P2 | **The password manager (Vaultwarden) runs on DooPlex and holds the keys that open DooPlex's own off-site copies — a loss of DooPlex loses the keys with the copies.** Operator finding, 2026-10-09 11:19: "the password manager also runs on DooPlex (Vaultwarden), so I probably need to back up its DB somewhere too." Measured: `vaultwarden-system/vaultwarden` (1.37.3), PVC `vaultwarden-data` 7.4 MB (`db.sqlite3` + WAL, `rsa_key.pem`, no attachments); its only backups were Longhorn on DooPlex. **A key in a password manager that runs on DooPlex is not off DooPlex** — R-173's "both keys are off DooPlex" (2026-10-05) and the hub-DB runbook Step 0 were wrong in that sense. Also seen: Vaultwarden warns its `ADMIN_TOKEN` is plain text (an Argon2 hash is its guidance). | **NARROWED 2026-10-09 (operator yes in chat):** Vaultwarden's own `backup` copy + `rsa_key.pem` ride the nightly encrypted DooPlex→ep0 job (`scripts/dooplex-offsite/`), checked on every push and every Sunday (integrity, users, items — rows only). That copy opens only with the DooPlex off-site key, and the vault only with the master password — so the circle is broken only once both are on paper outside DooPlex: `runbooks/break-glass-sheet.md`, `runbooks/total-loss-of-dooplex.md`. **LEFT:** the operator prints the sheet and stores it away from home; the `ADMIN_TOKEN` hash (a DooPlex change, operator's word). | operator: print the sheet | Operator: print + store the sheet; then close | operator | | **R-49** | Backup & restore | P3 | **[P2] The offsite capture set is ~90% cache and duplication — 1.1 GB of a 1.2 GB immich "photo backup".** Measured 2026-07-19: `immich_ml_cache.tar` **823 660 032 B (~60%)** — re-downloadable ML model weights; `immich_postgres_data.tar` **308 251 136 B (~23%)** — a raw tar of the postgres data dir that DUPLICATES the logical `.sql` dump captured beside it; `upload/backups/` **18 MB** — immich's own nightly dump, a backup inside the backup, growing daily; plus the stranded pre-v3 `dccc13fe…` tree (~36 MB) no DB has ever referenced. Actual irreplaceable content: **72 MB of originals**. **MIGRATED FROM `ROADMAP.md` 2026-08-22 (R-369) — originally filed 2026-07-19, size S–M, roadmap state `idea`.** Moved verbatim; nothing added or reinterpreted. The roadmap keeps its copy as history, marked moved. | **OPEN — migrated from ROADMAP 2026-08-22, rank unchanged** **Re-ranked 2026-10-03: P2→P3: wasted space and transfer, no data risk; needs a capture-set ruling, not a sale blocker.** | — | **Evidence: `audits/DIAG-immich-restore-round2-2026-07-19.md` §4 (full byte breakdown).** This is the customer's offsite quota and transfer cost, and it lands on the Hetzner sub-account they are billed for. **Recorded, deliberately not changed** — a capture-set exclusion is a data-loss-shaped decision and gets its own ruling, not a drive-by edit. Candidates in priority order: (a) `immich_ml_cache` — pure cache, strongest case; (b) the `postgres_data` volume tar where a logical dump of the same DB is already captured (the dump is what the restore path actually replays); (c) `upload/backups/`. Likely generalises past immich into a template-classification rule about cache volumes and self-backup directories, so it should be specified against the catalog, not one app | CC | | **R-231** | Backup & restore | P3 | **`/opt/backup/scripts/` on DooPlex is unversioned host state** — found 2026-08-06 while adding the auto-memory store to the backup set. No repository tracks the scripts that protect the recovery chain, so the edit made that day (`CLAUDE_MEMORY_DIR` in `backup-config.sh`, multi-path restic call in `backup-data.sh`) exists only on the box. This is the same class the part-2 session was closing, found inside the fix for it; the change is transcribed in `felhom.eu/workspace/README.md` so it is at least *recorded*. **Two related facts, both understating current safety:** the backup destination (`/mnt/5_hdd/backup`) is on the **same physical disk** as the workspace it protects, and the DooPlex backup set has **no off-site leg** (`sync-hetzner-backups.sh` is jarrs.eu and pulls *from* Hetzner *to* DooPlex). Bringing a root-owned production backup script under version control, and deciding what installs it, is its own scoped change. | **VERIFY** — **MEASURED 2026-10-13 — the due check is GREEN** (read by the sizing session, two channels): `systemctl show dooplex-backup.service` → `Result=success`, `ExecMainStatus=0`, ran 03:27:39–03:33:29 CEST; its journal ends „All backup components completed successfully!”, and the manifests leg (the header-token change) logged „Successfully backed up homelab-manifests” with the plain URL `https://gitea.dooplex.hu/admin/homelab-manifests` (no token in it). Left open, not closed: the row also carries the same-disk and no-off-site facts, which closing would drop — the owner decides. **VERIFY** — **2026-10-12 evening: the token left the clone URL** (operator yes in chat; homelab-manifests `1943a70`, installed by the role): `backup-manifests.sh` sends it as an `http.extraHeader` through `GIT_CONFIG_*` env; read-only proof: `git ls-remote` of the private repo with the header → HEAD, without → refused. The 2026-10-13 03:10 run is the test of both changes. **VERIFY** — **2026-10-12: in git and installed from it** (operator yes in chat, operator in the session): homelab-manifests `fdd324b`, `homelab-ansible/roles/dooplex_backup/` (11 scripts + the timer units, byte-identical to the live ones; no secret in any file; `install.sh` dry run / `--apply`). The install changed only the units' mode (744 → 644). Baseline: last night's run `Result=success`, exit 0 (`audits/answers-2026-10-12/s20-backup-scripts.txt`). **Seen by the commit's security review, NOT changed (a DooPlex behaviour change needs the operator's word):** `backup-manifests.sh` puts the Gitea token into the clone URL for the mirror run (visible in the process list for those seconds) and strips it from the stored remote afterwards; no token VALUE is in any file, and the repo is private (anonymous API read = 404). A header-based fetch (`git -c http.extraHeader=…` from the token file) would remove it — one line in the decision list. **READY** — operator ruling 2026-10-12 (S20, `09` §3 decision 215): yes — the scripts into a repo + an install script; the operator present for the install. **READY** — owner Viktor | — | Owner: close the row (the install from git is proven) and file the same-disk / no-off-site facts as their own row if they still hold | CC | diff --git a/documentation/runbooks/break-glass.md b/documentation/runbooks/break-glass.md index a2a89f42..27378240 100644 --- a/documentation/runbooks/break-glass.md +++ b/documentation/runbooks/break-glass.md @@ -88,7 +88,8 @@ Fix the cause; the three layers are the safety net, not the cure. before the web-console path was used. A working break-glass path independent of sshd must always exist. - **Rotate the console credential ALONE** with `sudo bash felhom-host-install.sh --rotate-recovery-only` (R-1141; try it first with `--dry-run`). It runs step 4b and nothing else: a new random `root@pam` - password, set with `chpasswd` and re-vaulted to the hub with the host key from the agent's own config + password, VAULTED at the hub FIRST and only then set with `chpasswd`, then checked with a PVE login (R-1145, 2026-10-18 — + a refused vault changes nothing), with the host key from the agent's own config (`/etc/felhom-agent/agent.json`). It does NOT reinstall or change the agent, its config or the root files, needs no passphrase or customer id, checks that the hub answers (`/healthz` 200) before the password changes, and refuses a byo host (also an agent config whose `deployment_mode` is not diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index 3a7f35ff..a64ce3e8 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -1,3 +1,19 @@ +## 2026-10-18 — step 4b vaults FIRST, then sets the password (R-1145); the host key off curl's argv; the harness is a gate (R-1146); not yet tagged + +- **R-1145:** step 4b (the full install and `--rotate-recovery-only`) now runs generate → PUT to the hub vault (200 = + stored) → chpasswd (one retry) → a PVE API login with the new value. A refused vault (a stale key included — the + independent review's F1) changes nothing; a set that fails twice after the vault leaves root on its previous password + and says so (its shadow line is fingerprinted before and after, never read out); a refused PVE login is said too. + The host key cannot read the vault back (by design), so the previous value cannot be re-vaulted — the next run fixes + that state. Live on demo-hp (`--rotate-recovery-only`): vault time moved, the new value 200, the old one 401. +- **Review F4:** the host key rides a 0600 header file (`-H @file`), never curl's argv. +- **R-1146:** shellcheck SC2066 fixed (a one-file loop became an `if`, same effect); GL8-F6's stale check now judges + where a line is (the uninstall's own dnsmasq, R-300, and the gate's hint text are allowed); the harness runs as gate + `hostinstall-harness` (`hostinstall_harness_gate.py`, fast; skips out loud on a runner without bash) with two decoys. + Harness: 52 passed, 0 failed (was 45/2). New cases R1145-a..e run the REAL step with fake commands; red-proofed against + HEAD (all five fail there). +- Pushing to `main` publishes nothing — this rides the next `installer-v…` tag. SCRIPT_VERSION unchanged. + ## 2026-10-18 — `felhom-host-install.sh --rotate-recovery-only` (R-1141); not yet tagged A new flag runs step 4b ALONE: a new random root@pam password, set with chpasswd and re-vaulted to the hub with the diff --git a/scripts/felhom-host-install.sh b/scripts/felhom-host-install.sh index 0f3bce21..1be2b52e 100644 --- a/scripts/felhom-host-install.sh +++ b/scripts/felhom-host-install.sh @@ -1628,6 +1628,11 @@ run_rescope_acl() { # --rotate-recovery (a re-set would strand the operator's saved copy). # Defined HERE, ahead of the argument parse (R-1141): --rotate-recovery-only is dispatched before the # install's mode validation and calls it. Moved unchanged; the install still calls it from Main. +# _root_shadow_fingerprint — sha256 of root's /etc/shadow line (never the line itself): did the password change? +# FELHOM_SHADOW_FILE is a test-harness-only override (hostinstall-mode-harness.sh R1145-*). +_root_shadow_fingerprint() { + grep '^root:' "${FELHOM_SHADOW_FILE:-/etc/shadow}" 2>/dev/null | sha256sum | cut -c1-64 +} step_break_glass() { log_step "4b/8 break-glass credential (root@pam console password → hub vault)" if [[ -z "${HOST_ID:-}" || -z "${HOST_API_KEY:-}" ]]; then @@ -1635,38 +1640,74 @@ step_break_glass() { return 0 fi if $DRY_RUN; then - log_dry "openssl rand → strong root@pam password (never logged) ; chpasswd ; PUT $HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential (Bearer host key)" + log_dry "openssl rand → strong root@pam password (never logged) ; PUT $HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential (Bearer host key) FIRST ; then chpasswd ; then a PVE login check (R-1145)" _state_mark break_glass; return 0 fi if _state_has break_glass && ! $ROTATE_RECOVERY; then log_skip " recovery credential already vaulted (use --rotate-recovery to regenerate)" return 0 fi + # R-1145 (2026-10-18): VAULT FIRST, THEN SET. Until 1.33.0 the password was set before the vault; a vault that + # then failed left root@pam with a password nobody holds — and the console is the last door into a broken box. + # Now: generate → PUT to the hub (200 = stored: the hub answers only after its DB write) → chpasswd (one retry) + # → a PVE API login with the new value. The possible end states, each logged in words, never the value: + # vault refused → nothing changed (root keeps its password, the hub its old copy) — die + # vault ok, set ok → root and the hub agree — then the PVE login is checked + # vault ok, set failed ×2 → the hub holds a password root does NOT have; root keeps its previous one (checked: + # its shadow line did not change). A host key cannot read the vault back (by design), + # so the previous value cannot be re-vaulted — die, and the next run fixes it. + # set ok, PVE login fails → root and the hub agree on the shell; the web console refuses — die, say so. + # Pinned by hostinstall-mode-harness.sh R1145-a..d (the step run with fake commands, call order recorded). # Strong password: 24 url-safe bytes (~144 bits). Kept ONLY in a local shell var, never on disk. - local newpw + local newpw before after code attempt set_ok=false newpw=$(openssl rand -base64 24 2>/dev/null | tr -d '\n' | tr '+/' '-_') [[ ${#newpw} -ge 24 ]] || die "failed to generate a strong recovery password" - # Set root@pam (= the Linux root user on PVE) via chpasswd on STDIN — no argv, no log. - if ! printf 'root:%s\n' "$newpw" | chpasswd 2>/dev/null; then - newpw="" # scrub - die "chpasswd failed to set the root@pam recovery password" - fi - # Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-). - local code + before=$(_root_shadow_fingerprint) + # 1. Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-). The host key rides + # a 0600 header FILE (`-H @file`, written by the printf builtin), never curl's argv — `ps` showed it there + # (independent review 2026-10-18, F4). + local hdr + hdr=$(mktemp) || die "cannot create a temp file for the vault request; the root@pam password was NOT changed" + chmod 600 "$hdr" + printf 'Authorization: Bearer %s\n' "$HOST_API_KEY" > "$hdr" code=$(printf '{"username":"root@pam","password":"%s"}' "$newpw" \ | curl -sS -o /dev/null -w '%{http_code}' -X PUT \ "$HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential" \ - -H "Authorization: Bearer $HOST_API_KEY" -H 'Content-Type: application/json' -d @- 2>/dev/null) - newpw="" # scrub the plaintext from the shell var the moment it is vaulted + -H @"$hdr" -H 'Content-Type: application/json' -d @- 2>/dev/null) + rm -f "$hdr" case "$code" in - 200) log_success " root@pam password set + vaulted to the hub (retrieve via the operator /admin path; never logged here)" - # Drill F-8: say this OUT LOUD — the operator was live-surprised by a 401 at the PVE - # GUI. Pointer only, never the credential. - log_warn " NOTE: the root@pam password just CHANGED — the old one now fails at the PVE web GUI (:8006)." - log_warn " Retrieve the new one at hub → host page (vaulted recovery credential)." ;; - 401|403) die "recovery-credential vault rejected ($code) — host key/authorization problem" ;; - *) die "recovery-credential vault failed (HTTP $code)" ;; + 200) log_info " recovery password vaulted at the hub (stored) — now setting it on root@pam" ;; + 401|403) newpw=""; die "recovery-credential vault rejected ($code) — host key/authorization problem; the root@pam password was NOT changed" ;; + *) newpw=""; die "recovery-credential vault failed (HTTP ${code:-000}); the root@pam password was NOT changed" ;; esac + # 2. Set root@pam (= the Linux root user on PVE) via chpasswd on STDIN — no argv, no log. One retry. + for attempt in 1 2; do + if printf 'root:%s\n' "$newpw" | chpasswd 2>/dev/null; then set_ok=true; break; fi + [[ $attempt -eq 1 ]] && { log_warn " chpasswd failed — retrying once"; sleep 2; } + done + after=$(_root_shadow_fingerprint) + if ! $set_ok && [[ -n "$after" && "$after" != "$before" ]]; then + set_ok=true # chpasswd reported failure but root's shadow line changed: the password IS the new one + fi + if ! $set_ok; then + newpw="" + log_error " STATE: the hub now holds a recovery password that root@pam does NOT have; root@pam keeps its previous password (its shadow line is unchanged)." + die "chpasswd failed twice after the vault — re-run (--rotate-recovery-only on an installed box): it vaults a new password first again" + fi + # 3. The proof a person needs: the PVE web login takes the new value (password on stdin, never argv). + code=$(printf '%s' "$newpw" | curl -sk -o /dev/null -w '%{http_code}' --max-time 20 \ + --data-urlencode 'username=root@pam' --data-urlencode 'password@-' \ + "https://127.0.0.1:8006/api2/json/access/ticket" 2>/dev/null) + newpw="" # scrub the plaintext from the shell var + if [[ "$code" != "200" ]]; then + log_error " STATE: root@pam and the hub hold the same NEW password, but the PVE web login answered HTTP ${code:-000}." + die "the PVE login with the new recovery password failed (HTTP ${code:-000}) — check pveproxy (:8006); the shell login uses the new password" + fi + log_success " root@pam password vaulted, set, and the PVE login takes it (retrieve via the operator /admin path; never logged here)" + # Drill F-8: say this OUT LOUD — the operator was live-surprised by a 401 at the PVE + # GUI. Pointer only, never the credential. + log_warn " NOTE: the root@pam password just CHANGED — the old one now fails at the PVE web GUI (:8006)." + log_warn " Retrieve the new one at hub → host page (vaulted recovery credential)." _state_mark break_glass } @@ -2556,12 +2597,12 @@ align_apt_repos() { log_info " disabled $(basename "$f") (Enabled: no)" done # 1b. legacy .list enterprise line → commented - for f in "$d/pve-enterprise.list"; do - [[ -f "$f" ]] || continue - grep -qE '^[[:space:]]*deb[[:space:]].*enterprise\.proxmox\.com' "$f" || continue + # (R-1146: one file, so no loop — the old one-item `for` was shellcheck SC2066; same file, same effect) + f="$d/pve-enterprise.list" + if [[ -f "$f" ]] && grep -qE '^[[:space:]]*deb[[:space:]].*enterprise\.proxmox\.com' "$f"; then sed -i 's|^[[:space:]]*\(deb[[:space:]].*enterprise\.proxmox\.com.*\)$|# \1 # felhom: no-subscription|' "$f"; changed=1 log_info " commented enterprise line in $(basename "$f")" - done + fi # 2. add pve-no-subscription if it is not already present in any list/sources if ! grep -rqsE 'download\.proxmox\.com/debian/pve([[:space:]/]|$)' "$d"/*.sources "$d"/*.list 2>/dev/null; then cat > "$d/pve-no-subscription.sources" </dev/null; then + && [[ "$gl8_f6_byo_before" == "yes" ]] && [[ -z "$gl8_f6_offenders" ]]; then verdict PASS "GL8-F6 byo :53 gate refuses+instructs, never mutates the owner's resolver" else - verdict FAIL "GL8-F6 byo :53 gate refuses+instructs, never mutates the owner's resolver" + verdict FAIL "GL8-F6 byo :53 gate refuses+instructs, never mutates the owner's resolver" "byo-gate=${gl8_f6_byo_before:-none} offenders=${gl8_f6_offenders:-none}" fi # GL8-F1 / R-275: the agent config and EVERY copy of it go — the agent's own config dir is purged as a @@ -427,6 +442,71 @@ else verdict FAIL "R1141-k usage documents --rotate-recovery-only" "rc=$rc" fi +# ── R-1145 (2026-10-18): step 4b is VAULT FIRST, THEN SET. The REAL function (cut out of the script) runs with fake +# openssl/curl/chpasswd/sleep on PATH and a fake shadow file; each fake appends one word to a calls file, so the +# ORDER is what is judged. The fake password must never appear in the step's output or any fake's argv. +r1145_dir="$WORK/r1145"; mkdir -p "$r1145_dir/bin" +R1145_PW="FAKEPW-r1145-AAAAAAAAAAAAAAAAAAAAAAAAAA" +cat > "$r1145_dir/bin/openssl" < "$r1145_dir/bin/curl" <<'EOF' +#!/bin/bash +cat >/dev/null # the body (password) arrives on stdin and goes nowhere +echo "argv: $*" >> "$R1145_DIR/argv" +case "$*" in + *recovery-credential*) echo vault >> "$R1145_DIR/calls"; printf '%s' "$FAKE_VAULT" ;; + *access/ticket*) echo login >> "$R1145_DIR/calls"; printf '%s' "$FAKE_LOGIN" ;; +esac +EOF +cat > "$r1145_dir/bin/chpasswd" <<'EOF' +#!/bin/bash +cat >/dev/null +echo chpasswd >> "$R1145_DIR/calls" +[[ "$FAKE_CHPASSWD" == ok ]] || exit 1 +echo "root:\$y\$changed:20000::::::" > "$R1145_SHADOW" +EOF +printf '#!/bin/bash\nexit 0\n' > "$r1145_dir/bin/sleep" +chmod +x "$r1145_dir/bin/"* +sed -n '/^_root_shadow_fingerprint() {/,/^}/p; /^step_break_glass() {/,/^}/p' "$SCRIPT" > "$r1145_dir/step.sh" +# r1145_run — sets r_rc, r_out, r_calls, r_shadow_changed +r1145_run() { + : > "$r1145_dir/calls"; : > "$r1145_dir/argv"; echo 'root:$y$old:20000::::::' > "$r1145_dir/shadow" + set +e + r_out=$(PATH="$r1145_dir/bin:$PATH" R1145_DIR="$r1145_dir" R1145_SHADOW="$r1145_dir/shadow" \ + FELHOM_SHADOW_FILE="$r1145_dir/shadow" FAKE_VAULT="$1" FAKE_CHPASSWD="$2" FAKE_LOGIN="$3" bash -c ' + log_step() { echo "STEP $*"; }; log_info() { echo "INFO $*"; }; log_warn() { echo "WARN $*"; } + log_error() { echo "ERROR $*"; }; log_success() { echo "OK $*"; }; log_skip() { echo "SKIP $*"; }; log_dry() { echo "DRY $*"; } + die() { echo "DIE $*"; exit 1; }; _state_mark() { echo "MARK $*"; }; _state_has() { return 1; } + DRY_RUN=false; ROTATE_RECOVERY=true; HOST_ID=h-test; HOST_API_KEY=k-test; HUB_URL=http://127.0.0.1:9 + source "'"$r1145_dir/step.sh"'"; step_break_glass' 2>&1) + r_rc=$? + set -e + r_calls=$(tr '\n' ' ' < "$r1145_dir/calls") + grep -q 'old' "$r1145_dir/shadow" && r_shadow_changed=no || r_shadow_changed=yes +} +r1145_check() { # + local ok=true why="" + [[ "$r_rc" == "$2" || ( "$2" == 1 && "$r_rc" != 0 ) ]] || { ok=false; why+="rc=$r_rc "; } + [[ "$r_calls" == "$3" ]] || { ok=false; why+="calls='$r_calls' want '$3' "; } + [[ "$r_shadow_changed" == "$4" ]] || { ok=false; why+="shadow changed=$r_shadow_changed "; } + [[ "$r_out" == *"$5"* ]] || { ok=false; why+="missing '$5' "; } + if [[ "$r_out" == *"$R1145_PW"* ]] || grep -qF "$R1145_PW" "$r1145_dir/argv"; then ok=false; why+="the password leaked "; fi + if grep -qF "k-test" "$r1145_dir/argv" || [[ "$r_out" == *"k-test"* ]]; then ok=false; why+="the host key is in curl's argv or the output "; fi + if $ok; then verdict PASS "$1"; else verdict FAIL "$1" "$why"; fi +} +r1145_run 500 ok 200 +r1145_check "R1145-a vault fails → the password is NOT touched" 1 "vault " no "the root@pam password was NOT changed" +r1145_run 401 ok 200 +r1145_check "R1145-e a stale host key (vault 401) → the password is NOT touched (review F1)" 1 "vault " no "vault rejected (401)" +r1145_run 200 ok 200 +r1145_check "R1145-b vault, then set, then the PVE login — in that order" 0 "vault chpasswd login " yes "the PVE login takes it" +r1145_run 200 fail 200 +r1145_check "R1145-c set fails twice after the vault → the state is said, root unchanged" 1 "vault chpasswd chpasswd " no "STATE: the hub now holds a recovery password that root@pam does NOT have" +r1145_run 200 ok 401 +r1145_check "R1145-d the PVE login refuses the new value → die, the state is said" 1 "vault chpasswd login " yes "the PVE web login answered HTTP 401" + echo "" echo "--- F5 appliance guest auto-sizing (VALIDATION-n100) ---" # The auto-size log line is emitted right after mode validation (BEFORE any PVE/hub contact), so it diff --git a/scripts/hostinstall_harness_gate.py b/scripts/hostinstall_harness_gate.py new file mode 100644 index 00000000..74894e74 --- /dev/null +++ b/scripts/hostinstall_harness_gate.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""hostinstall_harness_gate.py — runs scripts/hostinstall-mode-harness.sh as a gate (R-1146, 2026-10-18). + +WHY. The harness (bash -n, shellcheck, the argument-refusal matrix, the mode invariants, the --rotate-recovery-only cases +R1141-*, and the vault-first order R1145-*) was RED on `main` for weeks — SC2066 and a stale GL8-F6 check — because no +gate ran it: `repo_gates.py` ran only `hostinstall_gates.py`. The R-29 shape. It is fast (no network: its hub is a closed +port; no container; its PVE tier skips itself off a PVE host), so it runs in `--fast` and in the pre-push hook. + +THE CI RUNNER HAS NO bash (python3 and git only, `.gitea/workflows/gates.yml`). There the gate says so and exits 0 — the +same out-loud skip as check-onboarding.py's absent sibling — and the pre-push hook on DooPlex is where it bites. +Exit 0 harness green (or no bash, said) · 1 a harness case failed. +""" +import os +import shutil +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HARNESS = os.path.join(ROOT, "scripts", "hostinstall-mode-harness.sh") + + +def main(): + if not os.path.isfile(HARNESS): + print("hostinstall-harness: FAIL — %s is missing" % HARNESS) + return 1 + bash = shutil.which("bash") + if not bash: + print("hostinstall-harness: NOT CHECKED here — no bash on this runner; the pre-push hook on DooPlex runs it") + return 0 + p = subprocess.run([bash, HARNESS, os.path.join(ROOT, "scripts", "felhom-host-install.sh")], cwd=ROOT, + capture_output=True, text=True, errors="replace") + out = p.stdout + p.stderr + fails = [l for l in out.splitlines() if l.startswith("FAIL ")] + tail = [l for l in out.splitlines() if l.startswith("=== ") and "passed" in l] + for l in fails: + print(" " + l) + print("hostinstall-harness: %s" % (tail[-1] if tail else "(no summary line)")) + if p.returncode != 0 or fails or not tail: + print("hostinstall-harness: CONVICTED (rc=%d, %d failing case(s))" % (p.returncode, len(fails))) + return 1 + print("hostinstall-harness OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/repo_gates.py b/scripts/repo_gates.py index 0b69eb0b..e1f57783 100644 --- a/scripts/repo_gates.py +++ b/scripts/repo_gates.py @@ -10,6 +10,7 @@ Gates, in order (all must pass; **non-zero exit on any failure**): 1. site website HTML: BOM, emoji, nav/footer, analytics, CDN, tokens, cache-busting 2. hostinstall felhom-host-install.sh's five drill-swept invariants (+ R-94's absent-version) + 2b. hostinstall-harness scripts/hostinstall-mode-harness.sh as a gate (R-1146; skips out loud without bash) 3. hub-confirm no native confirm()/prompt() in hub templates 4. manifest-bearer no bearer-shaped literal anywhere in manifests/ 5. reuse-refs every path cited by this repo's REUSE.md still resolves @@ -137,6 +138,8 @@ SCRIPTS = os.path.join(ROOT, "scripts") GATES = [ ("site", os.path.join(SCRIPTS, "site_gates.py"), [], True, False), ("hostinstall", os.path.join(SCRIPTS, "hostinstall_gates.py"), [], True, False), + # R-1146 — the installer's own harness (incl. R-1141's and R-1145's cases) was red for weeks and run by nothing. + ("hostinstall-harness", os.path.join(SCRIPTS, "hostinstall_harness_gate.py"), [], True, False), ("hub-confirm", os.path.join(SCRIPTS, "hub_confirm_gate.py"), [], True, False), ("manifest-bearer", os.path.join(SCRIPTS, "manifest_bearer_gate.py"), [], True, False), ("reuse-refs", os.path.join(SCRIPTS, "reuse_refs_check.py"), [ROOT], True, False), diff --git a/scripts/test_gate_decoys.py b/scripts/test_gate_decoys.py index ebded43a..8c7af5ab 100644 --- a/scripts/test_gate_decoys.py +++ b/scripts/test_gate_decoys.py @@ -103,6 +103,8 @@ COVERS = { "controller clone must be INCONCLUSIVE, never a pass"), "hostinstall": ("R-426: R-185's regression with only its dry-run echo left, the age install commented " "out, and a version const in a new hub sub-package; a comment naming the identifier passes"), + "hostinstall-harness": ("R-1146: the vault-after-set order of step 4b put back (R-1145's hole) must convict " + "with R1145-b; a chpasswd outside step_break_glass convicts INV-2"), "due-checks": ("R-426: a dated check whose id survives only in prose (orphan), the due-TODAY boundary " "(and due-tomorrow, which must pass), and the block's marker named in prose (exit 2)"), "one-register": ("R-426: an open roadmap row whose state NAMES another row's CLOSED (a live hole until " @@ -718,6 +720,18 @@ decoy("hostinstall/comment-names-identifier", "hostinstall_gates.py", u'package zzr426decoy\n\n// hostInstallVersion was deleted on purpose (R-94).\n'), expect="accept") +# ── hostinstall-harness (R-1146, 2026-10-18) ──────────────────────────────────────────────────────── +# The label without the fact: step 4b keeps every message and every call, but sets the password BEFORE it vaults it +# (the shape until installer 1.33.0, R-1145) — the harness must name R1145-b. And a chpasswd outside the step's body. +decoy("hostinstall-harness/set-before-vault", "hostinstall_harness_gate.py", + replace_in(_HI, " # 1. Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-). The host key rides\n", + " printf 'root:%s\\n' \"$newpw\" | chpasswd 2>/dev/null || die \"chpasswd failed\"\n" + " # 1. Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-). The host key rides\n"), + must="R1145-b") +decoy("hostinstall-harness/chpasswd-outside-the-step", "hostinstall_harness_gate.py", + replace_in(_HI, "step_break_glass() {\n", "_decoy_r1146() { printf 'root:x\\n' | chpasswd; }\nstep_break_glass() {\n"), + must="INV-2") + # ── due-checks (R-426, 2026-10-06) ────────────────────────────────────────────────────────────── # # The 2026-09-01 sweep's attempt was a no-op and was withdrawn. These are the shapes a session really