From 0fa79cd90d15e643e0e7f14786811e7a544e3d58 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Fri, 3 Jul 2026 15:14:47 +0200 Subject: [PATCH] Go-live package: Day-0 install runbook + clean-room drill + host-install v1.9.1 - documentation/runbooks/day0-install.md (NEW): operator Day-0 guide, every command drill-executed verbatim (Parts A-F incl. the mandatory one-time controller update D.1b and the OQ-3 prereq checklist). - documentation/audits/DRILL-day0-cleanroom-2026-07-03.md (NEW): nested-PVE clean-room drill evidence D0-D6, residue tables, OQ-1/2/3 answers, findings ledger (B1-B5), go/no-go for Peti (GO). - scripts/felhom-host-install.sh v1.9.1: residue-free uninstall (agent config, shared-parent unit/script/mount, mkfs wrapper, hook snippet, dnsmasq snippets; zero-residue diff proven) + post-provision guest reboot (R6) + header/version sync. - CONTEXT.md, REPORT.md, scripts/CHANGELOG.md updated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- CONTEXT.md | 12 + REPORT.md | 90 +++-- .../audits/DRILL-day0-cleanroom-2026-07-03.md | 197 +++++++++++ documentation/runbooks/day0-install.md | 328 ++++++++++++++++++ scripts/CHANGELOG.md | 23 ++ scripts/felhom-host-install.sh | 74 +++- 6 files changed, 690 insertions(+), 34 deletions(-) create mode 100644 documentation/audits/DRILL-day0-cleanroom-2026-07-03.md create mode 100644 documentation/runbooks/day0-install.md diff --git a/CONTEXT.md b/CONTEXT.md index 2fef0b1..fc4396c 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -3,6 +3,18 @@ > Created with the REUSE.md rollout (2026-07-03). Authoritative history: `hub/CHANGELOG.md` (hub), > `website/CHANGELOG.md`, `scripts/CHANGELOG.md`; end-of-task detail in `REPORT.md`. +- **2026-07-03 — GO-LIVE PACKAGE SHIPPED (host-install v1.9.1 + day0-install runbook + clean-room drill)** — + `documentation/runbooks/day0-install.md` is the operator Day-0 guide, **every command executed + verbatim** on a throwaway nested PVE 9.2.2 (build server): install → verify → app deploy → + uninstall → **zero-residue diff** → reinstall. v1.9.1 = residue-free uninstall (agent config w/ + hub api_key, shared-parent unit/script/mount, mkfs wrapper, hook snippet, dnsmasq snippets) + + post-provision guest reboot (without it the controller never deploys on slower hardware) + header + sync. Evidence + go/no-go for Peti (**GO**, 3 install-day items): + `documentation/audits/DRILL-day0-cleanroom-2026-07-03.md`. Key OQ-2 finding: golden bakes a + pre-floor controller 0.85.1 → **fresh installs need the one-time D.1b update** (agent restart → + update button → 0.98.3); follow-ups recorded: rebuild golden ≥0.86.0 + re-vouch (operator), agent + `mkdir -p` for the guesthook snippet dir, fresh-install local-API 401 (pre-existing), hub has no + host-delete path. Drill VM parked virgin-snapshotted on 180 (`~/drill/`). - **2026-07-03 — A1 CLOSED (host-install v1.9.0 + agent v0.62.0)** — `Pool.Audit` added to `FelhomAgentGuest`; the agent's stale-lock reaper now intersects its scan with `GET /pools/felhom` membership (fail-safe skip on read failure). Spike: diff --git a/REPORT.md b/REPORT.md index 9bb400a..0815e4f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -4,31 +4,75 @@ --- -## felhom-host-install.sh v1.9.0 — Pool.Audit for the stale-lock reaper (A1) (2026-07-03) +# Go-live package — Day-0 install guide, clean-room drill, uninstall proof (2026-07-03) -**Class:** implementation (this repo's share of the A1 fix; agent side = felhom-agent **v0.62.0**, -see that repo's REPORT for the full evidence). Spec basis: -`documentation/audits/SPIKE-a1-pool-membership-read-2026-07-03.md`. Commit **`639a57e`** -(baseline `5e3dfcb`). +## Baselines -**Change:** `PVE_PRIVS_GUEST` gains **`Pool.Audit`** (12 → 13 privs; comment cites the spike and -why `Pool.Allocate` does not satisfy the read), `SCRIPT_VERSION` 1.8.0 → **1.9.0**. No structural -change — `_ensure_role` already `role modify`s to the exact set, so `--rescope-acl` is the -idempotent upgrade path; `remove_scoped_acl` deletes by role name and needed nothing. Gates: -`bash -n` OK, `reuse_refs_check.py` OK (31 paths). +| Repo | `main` before | After | +|---|---|---| +| felhom.eu | `996526273a` (script v1.9.0) | this commit — script **v1.9.1** + `documentation/runbooks/day0-install.md` (NEW) + `documentation/audits/DRILL-day0-cleanroom-2026-07-03.md` (NEW) | +| felhom-agent | `84f3f7ddb1` v0.62.0 | untouched (read-only; 3 findings recorded, not patched) | -**Live run (felhom-pve, the PUSHED script fetched from Gitea raw):** -`--rescope-acl --vmid 9201` → roles re-ensured; after: `FelhomAgentGuest` = -`Pool.Allocate,Pool.Audit,VM.*…` — Base/Store unchanged. Production-token `GET /pools/felhom` -went 403 → **200 (members [9201])**. Running v0.61.0 agent unaffected; agent v0.62.0 deployed -AFTER the rescope (order per spec) — startup shows -`stale-lock: scanning pool guests pool=felhom listed=1 scanned=1` and 47/47 capabilities incl. -`pve:pool-read` ok (also verified in the hub's stored host report). +## What shipped -**Also in this commit:** REUSE.md §2 errata — the secret-handling row claimed `gitea-creds` is -out-of-band; it is in fact COMMITTED in `manifests/felhom.secret.yaml` AND live-consumed by -hub.yaml. §3 trap row now lists it too. Rotation + de-git of the 3 committed secrets remains a -pending operator task (spike appendix). +1. **`documentation/runbooks/day0-install.md`** — the complete first-time deployment guide + (Parts A–F: hub onboarding / box-prereq checklist (OQ-3) / the canonical shared-box install + command / post-install verification incl. the mandatory one-time controller update D.1b / + uninstall + expected-remnants / troubleshooting from real drill hiccups). Header states the + drill date/environment/versions. Every command was executed verbatim during the drill. +2. **Clean-room drill (Phase 0 + D0–D6)** — throwaway nested PVE 9.2.2 under QEMU/KVM on the build + server (unattended install via `proxmox-auto-install-assistant`, virgin qcow2 snapshot, slirp + NAT; the primary Phase-0 mechanism worked — no fallback needed). Full cycle proven: baseline + manifest → hub onboarding of throwaway customer `drill-1` → install (real + `--force-gitea-golden` + `--cores 2 --memory 4096` + `--acl-storages "local local-lvm"`, + sha256-verified agent v0.62.0 + golden v0.85.1) → verify (selftest incl. `pve:pool-read`, + hub reporting, dashboard 200 behind traefik) → app deploy via the exact UI endpoint + (`bentopdf`, 200 at its own hostname) → uninstall → **residue diff** → reinstall + (host-enroll REUSED; fresh leaf on an empty box). Evidence doc: + `documentation/audits/DRILL-day0-cleanroom-2026-07-03.md`. +3. **`scripts/felhom-host-install.sh` v1.9.1** — drill-justified fixes, each re-verified live: + - header/version sync (v1.8.0 header vs 1.9.0 var) + keep-in-sync note; + - uninstall removes the five drill-found residue items (agent **config with the live hub + api_key**, shared-parent unit+script+`/mnt/felhom-drives`, guarded-mkfs wrapper, guest-hook + snippet, dnsmasq snippets) — v1.9.1 re-drill diff vs the virgin baseline shows **zero + Felhom-named leftovers**; + - **post-provision guest reboot + bounded verify wait (R6)** — without it the golden's + controller-bootstrap unit (boot-time `ConditionPathExists`) loses the race with the agent's + hot-plugged bootstrap mount on slower hardware and the controller never deploys; v1.9.1 + reinstall brought the controller up with no manual intervention. + Gates: `bash -n`, `--dry-run` previews, live re-drill of both paths. -**Operator follow-ups:** hub Day-0 artifact manifest → vouch agent **0.62.0** -(sha256 `ed1192c9ae8f355dee19128cf4b01984ba7398e860b4a16a8009655b84b32d81`); secrets rotation above. +## OQ answers (detail in the drill doc §8) + +- **OQ-1:** go live **local-backups-only** (`--acl-storages "local local-lvm"`) — PBS is LAN-only + until Headscale; retrofit later via `--rescope-acl`. Documented as the guide's standard. +- **OQ-2 (evidence-based):** a fresh install lands controller **0.85.1** (golden-baked, pre-floor) + and **never self-updates** — raising the floor does not help a fresh box. The guide's D.1b + (restart `felhom-agent` → trigger the settings-page update → verify) is the mandatory + install-day step; drill-proven 0.85.1 → 0.98.3. Structural fix = rebuild+re-vouch the golden + (operator follow-up). +- **OQ-3:** Part B of the guide is the customer-confirmed pre-install checklist. + +## Go/no-go for Peti: **GO** + +With three install-day items: serve v1.9.1 (this push), execute D.1b, create the real CF tunnel + +git credentials in Part A. Full list + blockers ledger: drill doc §8–§9. + +## Recorded findings NOT fixed here (per the gap policy) + +- **B1** (agent/golden, LOW): product-side fix for the bootstrap-race (golden path unit or + agent-side restart) — the installer reboot is a correct external fix. +- **B2** (agent, LOW): `guesthook.InstallSnippet` fails on fresh boxes (`/var/lib/vz/snippets` + missing; `install` can't create parents) → no pre-start self-heal hook, warn-only. +- **B3** (agent, MED, pre-existing): fresh-install local-API 401 until agent restart — new + consequence found: blocks the controller self-update swap. +- **B4** (hub, LOW): no host-delete path (UI or API); drill rows cleaned via direct SQL. +- **B5** (fleet, MED): golden bakes a pre-floor controller → D.1b needed on every fresh install + until the golden is rebuilt ≥ 0.86.0 and re-vouched. + +## Cleanup + +Hub `drill-1` rows fully deleted (all tables → 0; `demo-felhom` intact). Drill VM +**virgin-snapshotted and stopped** (kept at `~/drill/` on 192.168.0.180, ~6.7 GiB, for future +drills); evidence logs archived there. Nothing touched felhom-pve, guest 9201, or any real +customer/host record. No secrets in any committed file. diff --git a/documentation/audits/DRILL-day0-cleanroom-2026-07-03.md b/documentation/audits/DRILL-day0-cleanroom-2026-07-03.md new file mode 100644 index 0000000..84f50d0 --- /dev/null +++ b/documentation/audits/DRILL-day0-cleanroom-2026-07-03.md @@ -0,0 +1,197 @@ +# DRILL — Day-0 clean-room install / uninstall / reinstall (2026-07-03) + +**Class:** Clean-room drill (empirical validation of `documentation/runbooks/day0-install.md` + +residue-proof of `--uninstall`). Companion deliverables: the runbook itself and +`scripts/felhom-host-install.sh` **v1.9.1**. + +**Verdict (short):** the Day-0 guide is **executable verbatim end-to-end** on a virgin PVE 9.2 box: +install → verify → app deploy → uninstall → **zero-Felhom-residue diff** → reinstall all pass. +The drill found **one installer gap that mattered** (the controller never deployed without a guest +reboot — fixed in v1.9.1), **five uninstall residue items** (all fixed in v1.9.1, re-proven by a +second residue-diff), and produced a decisive **OQ-2 answer** (a fresh box does NOT self-update its +controller — a one-time manual update is a mandatory install-day step). **Go for Peti** with the +go/no-go list in §8. + +No secrets appear in this document. Drill credentials lived only on the build server (0600) and in +the drill VM; the drill VM was reverted to its virgin snapshot afterwards. + +--- + +## 1. Environment (Phase 0 — primary mechanism, validated) + +| Item | Value | +|---|---| +| Host | build server 192.168.0.180 (Debian 13, /dev/kvm, QEMU 10.0.8 installed for the drill) | +| VM | nested **PVE 9.2.2** (unattended install: `proxmox-auto-install-assistant` 9.2.7 answer file, ext4/LVM-thin layout), 4 vCPU / 8 GiB RAM / 150 GiB sparse qcow2 | +| Resulting storage | `local` 46 GiB dir + `local-lvm` **76 GiB** thin — deliberately BELOW the 120 GiB floor, so the warn path is exercised | +| Network | QEMU user-mode NAT (slirp); outbound to hub.felhom.eu / gitea.dooplex.hu / felhom.eu confirmed from host AND guest; SSH via hostfwd | +| Restore point | qcow2 internal snapshot `virgin` taken post-install, pre-first-boot | +| Hub side | live hub v0.31.0; throwaway customer **drill-1** (domain `drill.demo-felhom.eu`, fake CF tunnel token, real Gitea read credential copied from the demo customer's row, hub-generated-format passphrase) inserted directly into `customer_configs` (the web create form is operator-password-gated — CC cannot drive it; the row is byte-identical to `SaveCustomerConfig`, spike-day0 precedent) | +| Versions under test | script v1.9.0 → v1.9.1, agent v0.62.0, golden v0.85.1 (hub artifact manifest), controller floor 0.87.0, latest 0.98.3 | + +**Environment quirk (drill-only, not product):** slirp's DHCP pool starts at 10.0.2.15 — the same +address the PVE host holds statically — so the freshly-provisioned guest's DHCP grabbed the host's +IP, intermittently breaking the drill SSH (and truncating the D2 transcript). Worked around by +pinning the guest to 10.0.2.40. A real LAN with a real DHCP server does not have this failure mode. + +## 2. D0 — baseline + +Captured off-box before any Felhom artifact touched the box: sorted file list of +`/etc /usr/local/bin /usr/local/sbin /var/lib` (5 076 entries), `systemctl list-unit-files` +(379 units), `pveum user/role/acl/pool list`, `pvesm status`, `pct list; qm list` (empty). +Stored at `~/drill/D0/` on the build server. + +## 3. D1 — hub onboarding (guide Part A) + +- Customer-create field set verified against `hub/internal/web/configs.go` + (`handleConfigCreate` + `buildConfigJSON`): customer_id/name/domain/email + + `infrastructure.cf_tunnel_token`, `infrastructure.cf_api_token`, `git.username`, `git.token`; + passphrase (5 words) + api-key hub-generated. **`git.token` is load-bearing for Day-0** — the + installer dies at step 5/8 without it (guide Part A marks it required). +- Box-facing contracts probed for drill-1: `GET /api/v1/config/drill-1` → **200**; + `GET /api/v1/artifacts/drill-1` → agent 0.62.0 + golden 0.85.1; wrong passphrase → **401**. +- **UI caveat:** the operator-password-gated create form itself was not click-driven (CC has no + operator password); the row was inserted store-identically. Part A's field list is code-verified + and its outputs are live-verified. + +## 4. D2 — install (guide Parts B/C, script v1.9.0) + +Part B checklist ran verbatim (two guide corrections folded in: hub root answers **302**, and a +cleaner node-count one-liner). Script fetched from `https://felhom.eu/scripts/felhom-host-install.sh` +(text/plain, the git-synced repo file). `--dry-run` previewed all 8 steps correctly. Real run: + +``` +./felhom-host-install.sh --customer-id drill-1 --vmid 120 --cores 2 --memory 4096 \ + --force-gitea-golden --acl-storages "local local-lvm" --passphrase-file /root/.drill-pass +``` + +- pre-flight: PVE 9.2.2, node auto (single), **`local-lvm ~75 GiB < 120 GiB` warn fired as designed**, + passphrase validated, grows auto-computed for the small box (docker +34G, sys +12G). +- token + pool + scoped 3-role ACL applied (Base@/, Guest@/pool/felhom, Store@[local local-lvm]). +- host-enroll: **201 MINTED** (`drill-1-c867f3`). +- agent 0.62.0 fetched + **sha256-verified** against the hub manifest; sudo installed; non-root + user/sudoers/unit installed; selftest passed; service active. +- golden 0.85.1 fetched + sha256-verified from Gitea (`--force-gitea-golden` — the real fresh-box path). +- provision: guest 120 up in 1m26s, cores=2/memory=4096 applied, 32/50/20G volumes, mp9 bootstrap + mount, onboot=1, pool member. State file recorded `provisioned_vmid=120` + (**confirms** the v0.62.0-era open question: a fresh install DOES record it). + +## 5. D3/D4 — verify + app deploy + +- Agent: `--selftest` all-OK **including `pool read pool "felhom", 1 member`** (the v0.62.0 + `pve:pool-read` capability); service active as `felhom-agent`. +- Hub: host row reporting (agent 0.62.0), guest 120 `running`; events pipeline live from the fresh + box (`controller_started`, `app_deployed`, `controller_updated`, 2× `host_leaf_changed` warnings + from the reinstalls — correct behavior). +- **Finding R6 (the big one):** after provision the controller was NOT running — + `felhom-controller-bootstrap.service` was skipped + (`ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json` unmet at boot): the agent back-half + **hot-plugs** the bootstrap mount into the already-running guest and its own output says + *"next: reboot the guest"*, but the v1.9.0 script never rebooted. Fast hardware (demo) usually + won this race; the slow clean-room box lost it deterministically. **Fixed in v1.9.1** + (post-provision `pct reboot` + bounded verify wait); re-proven in D6b (§7). +- Dashboard: the controller sits behind traefik — correct probe is + `curl -k -H "Host: felhom." https://127.0.0.1/` → **200** (Hungarian dashboard). The + guide's draft `:8080` probe was wrong and was corrected. +- **D4 app deploy:** `bentopdf` deployed through the exact UI pipeline (page fetch → the same + `POST /api/stacks/bentopdf/deploy` + empty-CSRF headers the dashboard JS sends — endpoint-proxy + method per the live-validation rule; no browser bridge was available in this session). Container + healthy; app answers **200** at `pdf.drill.demo-felhom.eu` via traefik. Public Cloudflare leg not + drill-testable (fake tunnel token) — that leg is live-proven daily on the demo box. +- **OQ-2 evidence:** fresh box landed controller **0.85.1** (golden-baked) and did NOT + self-update (floor 0.87.0 ignored — 0.85.1 predates the v0.86.0 floor code; its one report ACK + carried no consequence). Manual update (`/api/selfupdate/check` + `/update`, the settings-page + button flow) first **failed with the known fresh-install local-API 401** (`POST /controller/swap: + HTTP 401`, stale token map); after `systemctl restart felhom-agent` the same trigger pulled + 0.98.3 and the agent swap landed it: **0.85.1 → 0.98.3, healthy**. Both the failure and the + recovery are now guide steps (D.1b). + +## 6. D5 — uninstall + residue diff (v1.9.0 → the gap list) + +`--uninstall --vmid 120 --remove-golden` ran with the typed-vmid confirmation; guest/agent/pveum/ +pool/state/golden all removed. Full re-capture + diff vs D0 classified every delta: + +| Delta (v1.9.0 uninstall) | Class | +|---|---| +| `/etc/felhom-agent/agent.json` (+dir) — **holds the live hub api_key** | **RESIDUE R1** → fixed v1.9.1 | +| `felhom-shared-parent.service` (+wants links) + `/usr/local/sbin/felhom-shared-parent.sh` + `/mnt/felhom-drives` self-bind still mounted | **RESIDUE R2** → fixed v1.9.1 | +| `/usr/local/sbin/felhom-mkfs-guarded` | **RESIDUE R3** (suspected by the spec — confirmed) → fixed v1.9.1 | +| `/var/lib/vz/snippets/felhom-guest-hook.sh` | **RESIDUE R4** (would remain where the snippet install succeeded; on this box it never installed — see B2) → fixed v1.9.1 | +| `/etc/dnsmasq.d/felhom-resolver-base.conf` | **RESIDUE R5** → fixed v1.9.1 | +| `sudo` + `dnsmasq` packages (+ their files/units) | expected remnant (documented in the guide; dnsmasq newly noted) | +| hub host record + report/guest/event history | expected remnant (no hub delete path — B4) | +| LVM archive/backup metadata, dpkg lists, PVE rrd + `/var/lib/lxc/120` scratch, `.pve-staged-mounts` | system noise (PVE/dpkg-owned) | + +## 7. D6 / D6b — reinstall proofs + +- **D6 (v1.9.0, post-uninstall box):** full reinstall succeeded — host-enroll **200 REUSED** + (mint-once-reuse proven), fresh leaf minted (box had no guests → the populated-host guard + correctly stayed silent), golden re-fetched, guest provisioned. Uninstall leaves a reinstallable + box. (Controller again required the reboot — the R6 evidence repeated deterministically, incl. + the agent's own "next: reboot the guest" line and the guesthook warn, see B2.) +- **v1.9.1 re-verification (all fixes):** + - `bash -n` + `-h` (header now v1.9.1) + full `--dry-run` previews of the new removal steps. + - **Uninstall re-drill:** v1.9.1 `--uninstall --remove-golden` → re-diff vs D0: + **zero `felhom`-named entries** in the file diff, zero pveum objects, no mounts, both dirs gone. + - **D6b install re-drill:** full v1.9.1 install on the clean box → the script rebooted the guest + itself → verify reported `controller: Up … (after ~0s)`, cloudflared up — **no manual + intervention**. R6 closed at the installer. + +## 8. OQ answers + go/no-go for Peti (sajatfelhom.hu) + +**OQ-1 (offsite backup): recommendation stands — go live with local backups only** +(`--acl-storages "local local-lvm"`, exactly the drilled canonical command). The PBS box is +LAN-only until Headscale/WireGuard lands; the ACL/`--rescope-acl` path retrofits the offsite grant +later without reinstalling. The guide documents this as the current standard. (Operator holds the +pen — if offsite-first is wanted instead, install day waits on the VPN spike.) + +**OQ-2 (fresh-install controller version): answered with evidence** — lands 0.85.1, never +self-updates (pre-floor code). Raising the floor does NOT fix a fresh box. Install-day step D.1b +(restart agent → trigger update → verify 0.98.3) is mandatory and drill-proven. Structural fix = +rebuild + re-vouch the golden with a ≥ 0.86.0 controller (operator follow-up, not a Peti blocker). + +**OQ-3 (pre-install facts):** the guide's Part B checklist is the collection instrument (PVE 9.x, +single-node vs cluster → `--node`, ≥120 GiB local-lvm, free vmid from `pct list`+`qm list`, host +cores/RAM for the cap, root SSH, outbound reach incl. the three exact URLs+expected codes, python3). + +**Go/no-go list:** + +| # | Item | State | +|---|---|---| +| 1 | Install path proven verbatim on a virgin box (twice + reinstall) | ✅ GO | +| 2 | Uninstall residue-clean (v1.9.1) — safe on a shared box | ✅ GO (deploy/serve v1.9.1 BEFORE install day; drilled from the served URL at v1.9.0 + locally at v1.9.1) | +| 3 | D.1b one-time controller update executed on install day | ⚠️ REQUIRED STEP (guide covers it; 2 minutes) | +| 4 | Day-0 manifest vouches agent **0.62.0** + golden **0.85.1** | ✅ current; **but hub Day-0 manifest bump to agent 0.62.0 was already flagged as operator follow-up — CONFIRMED DONE (drill fetched 0.62.0)** | +| 5 | Real CF tunnel token created + entered in Part A (drill used a fake) | ⚠️ operator does A.1 for sajatfelhom.hu; tunnel leg is demo-proven, not drill-proven | +| 6 | `git.username`/`git.token` set in the customer config | ⚠️ REQUIRED (installer dies without it) | +| 7 | Multi-node? If Peti's box is a cluster, add `--node` | per Part B answer | +| 8 | Known non-blockers on the box: B2 (no pre-start hook on fresh boxes), B3 (401-until-agent-restart) | documented + workarounds in the guide | + +**Verdict: GO** — with items 3/5/6 executed on/before install day. + +## 9. Findings ledger (blockers + follow-ups, per the §9 gap policy) + +| ID | Where | Severity | What | Status | +|---|---|---|---|---| +| R1–R5 | installer | — | uninstall residue (config w/ secrets, shared-parent unit+script+mount, mkfs wrapper, hook snippet, dnsmasq snippets) | **FIXED v1.9.1**, residue-diff-proven | +| R6 | installer | — | controller-bootstrap unit skipped: no post-provision reboot (agent hot-plugs the mount; unit condition is boot-time) | **FIXED v1.9.1** (reboot + bounded verify wait), D6b-proven | +| B1 | agent/golden | LOW | structural fix for R6 belongs in the product too (path unit in the golden, or agent-side restart after back-half) — installer reboot is a correct but external crutch | RECORDED (agent/golden follow-up) | +| B2 | agent | LOW | `guesthook.InstallSnippet` fails on a fresh box: `/var/lib/vz/snippets` doesn't exist and `install` won't create it → no pre-start self-heal hook, warn-only (`install: cannot create regular file … No such file or directory`, agent journal 14:58:40) | RECORDED (fix: mkdir -p SnippetDir; demo box unaffected — dir pre-exists) | +| B3 | agent | MED | fresh-install local-API **401 until `systemctl restart felhom-agent`** (stale token map, pre-existing open issue) — NEW consequence found: it blocks the controller self-update agent-swap | RECORDED (workaround = guide D.1b step 1) | +| B4 | hub | LOW | **no host-delete path** (UI or API) — drill host rows removed via direct SQL (spike precedent); customer-delete exists but leaves hosts/reports/guests/events orphaned | RECORDED (candidate small hub follow-up) | +| B5 | golden/fleet | MED | golden bakes a pre-floor controller (0.85.1) → every fresh install needs D.1b manually | RECORDED (operator follow-up: rebuild golden ≥ 0.86.0 + re-vouch in the Day-0 manifest UI) | + +## 10. Cleanup confirmation + +- **Hub:** all `drill-1` rows deleted (customer_configs / hosts / host_reports / reports / guests / + events / notification_log / customer_notifications → remaining **0**); `demo-felhom` confirmed + intact. (The web customer-delete would have left the host/report rows — B4 — hence SQL, spike + precedent.) +- **Drill VM:** powered off; disk reverted to the `virgin` snapshot and kept + (**virgin-snapshotted-and-stopped**, not removed — reusable for future drills). Footprint + ~6.7 GiB at `~/drill/` on the build server (ISO + qcow2 + evidence logs + keys, keys 0600); + the baked auto-install ISO variant was deleted. +- Evidence archived off-box at `~/drill/{D0,D5,D5b}/` + `D2-install.log`, `D5-uninstall.log`, + `D5b-uninstall.log`, `D6-install.log`, `D6b-install.log` on the build server. +- QEMU/`proxmox-auto-install-assistant` packages remain installed on the build server (useful for + re-drills; noted). diff --git a/documentation/runbooks/day0-install.md b/documentation/runbooks/day0-install.md new file mode 100644 index 0000000..3a64ead --- /dev/null +++ b/documentation/runbooks/day0-install.md @@ -0,0 +1,328 @@ +# Runbook — Day-0 install: first-time deployment of a Felhom host + +> **The complete, step-by-step first-time deployment guide** for putting Felhom on a customer's +> Proxmox box — hub-side onboarding, box prerequisites, the install run, post-install verification, +> first app deploy, and rollback/uninstall. Every command in this guide was executed verbatim during +> the clean-room drill (see the header line below); placeholders are marked ``. +> +> **Drill-validated: 2026-07-03** on a clean-room nested PVE 9.2.2 VM (qemu/KVM on the build +> server) — install → verify → app deploy → uninstall → residue-diff → reinstall, all executed +> verbatim. Versions: script **v1.9.1**, agent **v0.62.0**, golden **v0.85.1** (controller updated +> to 0.98.3 via D.1b). Evidence: `documentation/audits/DRILL-day0-cleanroom-2026-07-03.md`. +> +> Scope: a **shared** Proxmox box (the colleague's-box model) — a box that already runs, or will run, +> non-Felhom guests. A dedicated box is the same procedure minus the sizing caution. + +## Who does what + +| Actor | Where | Does | +|---|---|---| +| **Operator** | HQ (hub UI + Cloudflare) | Part A (customer onboarding), go/no-go, on call during install | +| **Operator (or trusted colleague with root SSH)** | the customer's Proxmox box | Parts B–E | +| **Customer** | — | confirms the Part B prerequisites, receives the dashboard URL | + +One secret crosses to the box: the customer's **retrieval passphrase** (5 Hungarian words, generated +by the hub in Part A). Everything else the box needs, it fetches and verifies itself. + +--- + +## Part A — hub-side customer onboarding (operator, at HQ) + +### A.1 Create the Cloudflare tunnel (per customer domain) + +The customer's apps are published through a Cloudflare Tunnel; the in-guest controller runs +`cloudflared` with a per-customer token. In the Cloudflare dashboard (Zero Trust → Networks → +Tunnels): + +1. Create a tunnel named after the customer (e.g. ``). Connector type: Cloudflared. +2. Copy the **tunnel token** (the long base64 string from the `cloudflared service install ` + command) — this goes into the hub customer form in A.2. +3. In the customer domain's DNS zone, the controller manages per-app hostnames itself via the + tunnel; make sure the domain is on Cloudflare (nameservers active). + +Also have ready (optional but recommended): a **Cloudflare API token** with Zone edit rights for the +customer's zone — the hub uses it for geo-restriction management. + +### A.2 Create the customer in the hub + +Hub UI (`https://hub.felhom.eu`, operator password) → **Customers → New**: + +| Field | Value | Notes | +|---|---|---| +| Customer ID | e.g. `sajatfelhom` | letters, numbers, dots, hyphens only; **immutable** — it keys everything | +| Customer name | display name | | +| Domain | e.g. `sajatfelhom.hu` | the customer's app domain (on Cloudflare) | +| Email | customer's email | used for customer-tier notifications (Hungarian) | +| CF tunnel token | from A.1 | → `infrastructure.cf_tunnel_token` | +| CF API token | from A.1 (optional) | → `infrastructure.cf_api_token` (geo rules) | +| Git username | Gitea read account | → `git.username` — **required for Day-0** | +| Git token | Gitea read token | → `git.token` — **required for Day-0**: the install script and the controller fetch artifacts from Gitea with this credential; without it the install dies at step 5/8 | + +On save the hub generates two credentials: + +- **Retrieval passphrase** (5 Hungarian words) — the ONE secret the installer carries to the box. + Dictate or hand it to whoever runs Part C. Treat it like a password. +- **Customer API key** — internal (baked into the generated controller.yaml); never handled manually. + +### A.3 Verify the Day-0 artifact manifest + +Hub UI → **Configuration → Day-0 artifacts**. The manifest must vouch an **agent version** and a +**golden version** (with their sha256s — auto-resolved from Gitea). The install script refuses to +install anything the manifest doesn't vouch. If the vouched versions lag the fleet, bump them here +first (dropdowns; the sha is fetched authoritatively). + +### A.4 Check the global controller-version floor + +Hub UI → **Configuration**. Know the current `min_controller_version` floor, but understand its +limit (drill-proven, OQ-2): a fresh box lands on the golden's baked-in controller (**v0.85.1**, +which predates the floor-honoring code), so **the floor does NOT pull a fresh install up** — the +one-time update in D.1b is the required step. Raising the floor still matters for keeping the box +current afterwards. The structural fix — rebuilding + re-vouching the golden with a ≥ 0.86.0 +controller — is a recorded operator follow-up. + +--- + +## Part B — box prerequisites (confirm BEFORE install day) + +Collect these from the customer / the box ahead of time. Every line is checkable in two minutes; a +"no" on any of them turns install day into debugging day. + +```text +[ ] Proxmox VE 9.x installed and reachable over SSH as root + pveversion # want pve-manager/9.x +[ ] Single node or cluster? + pvesh get /nodes --output-format json | python3 -c 'import json,sys; ns=json.load(sys.stdin); print(len(ns), [n["node"] for n in ns])' + → more than one node: the install command MUST carry --node +[ ] local-lvm free space ≥ 120 GiB + lvs /dev/pve/data -o lv_size,data_percent +[ ] Free vmid chosen from the FULL guest list (never from the hub view) + pct list; qm list # pick a vmid not in either list; avoid 9999 and 990000-990009 +[ ] Host cores + RAM known (to size the appliance cap) + nproc; free -m + → shared box: cap the guest so the other guests keep breathing (Part C uses --cores/--memory) +[ ] Outbound HTTPS reachability from the box: + curl -fsSo /dev/null -w '%{http_code}\n' https://hub.felhom.eu # want 302 (login redirect) + curl -fsSo /dev/null -w '%{http_code}\n' https://gitea.dooplex.hu # want 200 + curl -fsSo /dev/null -w '%{http_code}\n' https://felhom.eu # want 200 +[ ] python3 present (PVE 9 default: yes) + command -v python3 +[ ] The operator has the retrieval passphrase from Part A at hand +``` + +Notes: +- The box needs **no inbound** reachability — hub, Gitea and the Cloudflare tunnel are all + outbound-only. +- vmid picking: the agent's token is pool-scoped, so the hub's guest inventory only shows + Felhom-pool guests — other guests on a shared box are invisible there. Always pick from + `pct list` + `qm list` on the box (`documentation/runbooks/provisioning.md`). +- Offsite backup (`felhom-pbs`) is NOT part of a go-live today: the PBS host is LAN-only until the + Headscale/WireGuard work lands. Go-live standard = **local backups only** (that is what the + Part C command configures); offsite DR is retrofitted later via `--rescope-acl` / + `--acl-storages` without reinstalling. + +--- + +## Part C — the install run (on the box, as root) + +### C.1 Fetch the installer + +```bash +curl -fsSO https://felhom.eu/scripts/felhom-host-install.sh +chmod +x felhom-host-install.sh +./felhom-host-install.sh -h | head -3 # sanity: must print v1.9.1 (or newer) — the version this guide was drilled against +``` + +### C.2 Preview (recommended) + +`--dry-run` prints every mutating command without executing anything (it still contacts the hub +read-only and asks for the passphrase): + +```bash +./felhom-host-install.sh --customer-id --vmid \ + --cores 2 --memory 4096 \ + --force-gitea-golden --acl-storages "local local-lvm" \ + --dry-run +``` + +### C.3 The canonical shared-box install command + +```bash +./felhom-host-install.sh --customer-id --vmid \ + --cores 2 --memory 4096 \ + --force-gitea-golden --acl-storages "local local-lvm" +``` + +Placeholders and flags — what and why: + +| Flag | Value | Why | +|---|---|---| +| `--customer-id` | the Part A customer ID | keys every hub call | +| `--vmid` | the free vmid from Part B | ALWAYS pass it explicitly — deterministic, and recorded in the install state for a later `--uninstall` | +| `--cores 2 --memory 4096` | the appliance cap | protects the other guests on a shared box; size to roughly half the host if the box is small (cap ≤ host resources) | +| `--force-gitea-golden` | — | a fresh box has no local golden archive; fetch it from Gitea and verify its sha256 against the hub manifest (this is the normal customer path) | +| `--acl-storages "local local-lvm"` | — | grants the agent's token write access on exactly these storages. The default set includes `felhom-pbs` (offsite), which does not exist on a customer box — granting it is harmless but misleading; scope to what is real. | +| `--node ` | only on a **cluster** | the script refuses to auto-pick a node on a multi-node cluster | + +When prompted, type the **retrieval passphrase** (no echo; never on the command line). Alternative +for remote/scripted runs: put it in a 0600 file and pass `--passphrase-file `. + +### C.4 What the script does (so you can follow the output) + +Eight steps, all idempotent/resumable (`--resume` skips completed steps after a mid-run failure): + +1. **pre-flight** — root/PVE/hub checks, cluster refusal without `--node`, LVM/RAM floors, + passphrase validated read-only against the hub, vmid guard. +2. **Proxmox API token** — `felhom-agent@pve!agent` minted (privsep), the `felhom` pool created, and + the pool-scoped 3-role ACL applied (blast-radius containment: the token can only touch + Felhom-pool guests + the granted storages). +3. **volume grows** — computed from free space (Docker-data/user-data sizing). +4. **host enrollment** — `POST /host-enroll` (passphrase-authed, idempotent) mints/returns the + per-host hub credential. +5. **agent install** — binary fetched from Gitea, **sha256-verified against the hub manifest**, + installed with the non-root service user, sudoers (visudo-validated), guarded-mkfs wrapper, + systemd unit. +6. **agent config + service** — config written 0600, read-only selftest must pass, daemon started. +7. **golden archive** — fetched from Gitea + sha256-verified (because `--force-gitea-golden`). +8. **provision** — the agent restores the golden into vmid `` (into the `felhom` pool), + grows volumes, renders `bootstrap.json`, sets onboot, starts the guest. Then a verify pass. + +Expected duration: minutes; dominated by the two Gitea downloads (agent ~20 MB, golden ~1-2 GB) on +the customer's uplink. + +If a step fails: read the error (they are specific), fix the cause, re-run the SAME command with +`--resume` appended. + +--- + +## Part D — post-install verification (on the box + hub) + +Run through ALL of these before calling it done. + +### D.1 On the box + +```bash +# 1. The script's own final verdict said: "Day-0 provision SUCCESS — vmid=… host_id=…" + +# 2. Agent service up, running non-root +systemctl is-active felhom-agent # active +systemctl show felhom-agent -p User --value # felhom-agent + +# 3. Agent selftest — all capabilities, including the pool read (stale-lock reaper) +felhom-agent --config /etc/felhom-agent/agent.json --selftest 2>&1 | tail -25 +# want: no FAILs; the "pve:pool-read" line ok (agent >= v0.62.0) + +# 4. Guest up + onboot +pct status # status: running +pct config | grep -E '^(onboot|rootfs|mp0|mp1)' + +# 5. Controller container healthy in-guest +pct exec -- docker ps --filter name=felhom-controller --format '{{.Status}} {{.Image}}' + +# 6. Dashboard answers in-guest (the controller sits BEHIND traefik — probe with the Host header; +# the controller's own :8080 is internal to the Docker network and not directly reachable) +pct exec -- curl -ksSo /dev/null -w '%{http_code}\n' \ + -H "Host: felhom." https://127.0.0.1/ # want 200 +``` + +### D.1b One-time controller update (REQUIRED — do not skip) + +The golden bakes controller **v0.85.1**, which predates the v0.86.0 floor-honoring code — **a fresh +install does NOT auto-update, regardless of the hub floor**. Bring it current once; from then on the +floor mechanism manages it: + +```bash +# 1. Restart the agent first — a fresh install's local API 401s the swap until the agent restarts +# (known open issue; see troubleshooting) +systemctl restart felhom-agent + +# 2. In the dashboard: Beállítások → "Frissítés" (update to latest), or equivalently: +pct exec -- curl -ksS -X POST -H "Host: felhom." https://127.0.0.1/api/selfupdate/check +pct exec -- curl -ksS -X POST -H "Host: felhom." https://127.0.0.1/api/selfupdate/update + +# 3. Wait ~1 min, confirm the new version is running (image pull happens first, then an agent swap): +pct exec -- docker ps --filter name=felhom-controller --format '{{.Image}} {{.Status}}' +``` + +### D.2 In the hub UI + +- **Hosts**: the new host reports (status OK, fresh last-report timestamp). +- The host's guest list shows `` running. +- **Capabilities**: 0 degraded. On a reduced-storage box (no `felhom-pbs`) the offsite-backup + related capability is expected degraded — that is the documented go-live standard, not a fault. +- The controller version shown for the guest matches what D.1b brought it to (the golden's baked + 0.85.1 does NOT self-converge — D.1b is the required manual step until the golden is rebuilt with + a ≥ 0.86.0 controller). + +### D.3 Customer-visible + +- `https://felhom.` loads the dashboard through the tunnel (cloudflared up + in-guest: `pct exec -- docker ps --filter name=cloudflared`). + +### D.4 First app deploy (smoke) + +In the dashboard, deploy one small catalog app end-to-end (pick something dependency-free). The app +reaches "running" and its URL answers. This proves the full customer-visible chain: catalog → +controller → Docker → tunnel → browser. + +--- + +## Part E — rollback / uninstall + +`--uninstall` cleanly reverts an install — no hub contact, no passphrase needed: + +```bash +./felhom-host-install.sh --uninstall --vmid +``` + +Guardrails (all verified in the drill): + +- **Ours-check**: refuses to destroy a guest without the `/etc/felhom-bootstrap` mount (i.e. not + Felhom-provisioned) unless `--force`. +- **Typed confirmation**: you must type the vmid back before anything is destroyed. +- **Shared-box safety**: if OTHER Felhom guests remain on the host, only the named guest is + removed; the agent/token/ACL stay (guest-only mode). `--force` overrides. +- The `felhom` pool is deleted only if empty. +- `--vmid` is optional when the install state file recorded the provisioned vmid (a fresh + v1.9.x install records it); passing it explicitly is still the recommended, deterministic form. + +What it removes (v1.9.1, residue-diff-proven — a full-filesystem diff against the pre-install +baseline showed **zero Felhom-named leftovers**): the guest, the agent (unit/sudoers/binary/ +state dir/config/service user), the shared-parent unit + wrapper + `/mnt/felhom-drives`, the +guarded-mkfs wrapper, the guest-hook snippet, the lan-resolver dnsmasq snippets, the pveum +roles/ACL/token/user, the pool (if empty), the install state file. + +**Expected remnants** (documented, not residue): + +- The **hub host record** (+ its report/guest history) — the hub currently has **no host-delete**; + removing it is a direct-DB operator action (recorded follow-up). +- The `sudo` and `dnsmasq` packages (system packages installed as dependencies; inert). +- The **golden vzdump** on the archive storage — pass `--remove-golden` to delete it too. +- System noise: LVM archive metadata, dpkg lists, PVE's own rrd/`/var/lib/lxc` scratch entries. + +--- + +## Part F — troubleshooting (from the drill + live history) + +| Symptom | Cause | Fix | +|---|---|---| +| step 1 dies: "this is a N-node cluster" | multi-node cluster | re-run with `--node ` | +| step 5 dies: "hub artifact manifest has no agent version" | Day-0 manifest unset/incomplete | Part A.3 — set it in the operator UI | +| step 5 dies: "no git token in controller.yaml" | customer created without git credentials | Part A.2 — add `git.username`/`git.token`, regenerate config | +| step 1: passphrase REJECTED (401) | typo / wrong customer | re-check with the hub UI's printed curl command | +| step 8 fails: "CT already exists" | vmid collision with a hub-invisible guest | pick from `pct list` + `qm list` (Part B); the agent destroys nothing on collision — re-run with a free vmid and `--resume` | +| controller container missing in-guest after provision (docker ps empty) | pre-v1.9.1 script: the controller-bootstrap unit's boot-time condition lost the race with the bootstrap-mount attach | `pct reboot ` — the unit runs on the next boot (v1.9.1 does this reboot itself) | +| self-update fails: "Agent swap request failed: HTTP 401" (or dashboard agent calls 401) | fresh-install local-API stale token map — known open issue | `systemctl restart felhom-agent`, then retry (this is why D.1b restarts the agent first) | +| `local-lvm free < 120 GiB` warning | small/shared box | proceed only if you sized the grows deliberately; otherwise free space first | +| cloudflared restarting in-guest | bad/missing tunnel token | re-check A.1/A.2 token, regenerate config, `docker restart cloudflared` in-guest | + +(Extended during the drill with every real hiccup encountered.) + +--- + +## Related + +- vmid picking + reserved bands: `documentation/runbooks/provisioning.md` +- The Day-0 trust chain (why the box can verify what it fetches): + `documentation/audits/SPIKE-day0-firstboot-handshake-2026-06-26.md` +- Pool-scoped ACL model: `documentation/audits/SPIKE-pool-scoped-acl-2026-07-01.md` +- Secrets handling: `documentation/runbooks/secrets.md` diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index d734837..854b2c1 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -1,5 +1,28 @@ # Felhom scripts — Changelog +## felhom-host-install.sh v1.9.1 — clean-room drill fixes: residue-free uninstall + post-provision reboot (2026-07-03) + +Companion to the Day-0 go-live package (`documentation/runbooks/day0-install.md` + +`documentation/audits/DRILL-day0-cleanroom-2026-07-03.md`). Every fix was found by the clean-room +drill (virgin nested PVE 9.2.2) and re-verified there (v1.9.1 uninstall → **zero-Felhom-residue +diff vs the pre-install baseline**; v1.9.1 install → controller up with no manual intervention). + +- **Header/version sync** (the header said v1.8.0 while `SCRIPT_VERSION` said 1.9.0); keep-in-sync + note on `SCRIPT_VERSION`; usage sed range follows the header (2,95). +- **Uninstall now removes the drill-found residue (R1–R5):** the agent **config** + (resolved from the unit's `-config` BEFORE the unit is removed — it holds the per-host hub + api_key), the `felhom-shared-parent` unit + wants links + `/usr/local/sbin/felhom-shared-parent.sh` + + the `/mnt/felhom-drives` self-bind/dir, `/usr/local/sbin/felhom-mkfs-guarded`, + `/var/lib/vz/snippets/felhom-guest-hook.sh`, and `/etc/dnsmasq.d/felhom-*.conf` + (+ dnsmasq restart when touched). All tolerate-absent; summary lines updated (`sudo` AND + `dnsmasq` packages are the documented package remnants). +- **Post-provision guest reboot (R6):** the golden's `felhom-controller-bootstrap.service` + evaluates `ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json` at BOOT, but the agent + back-half hot-plugs the mount into the running guest — on slower hardware the first boot loses + that race deterministically and the controller never deploys. `step_provision` now reboots the + guest once (the agent's own output says "next: reboot the guest"); `step_verify` waits bounded + (180 s) for the controller container instead of a momentary look. + ## felhom-host-install.sh v1.9.0 — Pool.Audit for the stale-lock reaper (A1) (2026-07-03) Companion to felhom-agent v0.62.0 (audit A1: pool-membership ownership check). `PVE_PRIVS_GUEST` diff --git a/scripts/felhom-host-install.sh b/scripts/felhom-host-install.sh index f96cbbf..3105c41 100644 --- a/scripts/felhom-host-install.sh +++ b/scripts/felhom-host-install.sh @@ -1,6 +1,6 @@ #!/bin/bash #=============================================================================== -# felhom-host-install.sh v1.8.0 +# felhom-host-install.sh v1.9.1 # Day-0 host-bootstrap for a Felhom Proxmox host (operator-deploy model). # # Run by the operator on a FRESHLY-PVE-INSTALLED box (after a manual PVE install @@ -71,8 +71,9 @@ # # Uninstall (local host teardown — no hub contact, no passphrase): # --uninstall cleanly revert an install: destroy the Felhom guest, remove the agent -# (unit/sudoers/binary/state/user), the pveum role/user/token/ACL, and the -# install state file. Guarded: refuses a non-Felhom guest (no +# (unit/sudoers/binary/state/config/user + runtime artifacts: shared-parent +# unit, mkfs wrapper, hook snippet, dnsmasq snippets), the pveum +# role/user/token/ACL, and the install state file. Refuses a non-Felhom guest (no # /etc/felhom-bootstrap mount) and skips host-level removal if OTHER Felhom # guests remain (both overridable with --force). Typed vmid confirmation # required. Reuses --vmid (else the recorded provisioned_vmid), --force, @@ -104,7 +105,7 @@ set -euo pipefail -SCRIPT_VERSION="1.9.0" +SCRIPT_VERSION="1.9.1" # keep in sync with the header line at the top of this file #------------------------------------------------------------------------------- # Logging (mirrors felhom-controller/scripts/docker-setup.sh) @@ -211,7 +212,7 @@ ART_GOLDEN_SHA="" #------------------------------------------------------------------------------- # Helpers #------------------------------------------------------------------------------- -usage() { sed -n '2,94p' "$0" | sed 's/^# \{0,1\}//'; exit 0; } +usage() { sed -n '2,95p' "$0" | sed 's/^# \{0,1\}//'; exit 0; } run() { # simple (no pipes/redirects) mutating command if $DRY_RUN; then log_dry "$*"; else "$@"; fi @@ -552,7 +553,15 @@ run_uninstall() { # ── host-level removal (reverse of install) ────────────────────────────────────────────────── log_step "host-level removal" - # 4. Agent removal — service, unit(+.bak), sudoers, binary(+.bak), state dir, user. NEVER `sudo`. + # 4. Agent removal — service, unit(+.bak), sudoers, binary(+.bak), state dir, config, user. + # NEVER `sudo`. Resolve the agent config path BEFORE the unit is removed (mirrors preflight — + # the unit's -config arg is the truth, else the default); the config holds the per-host hub + # api_key and must not survive an uninstall (drill finding R1). + local agent_cfg="" + if systemctl cat felhom-agent >/dev/null 2>&1; then + agent_cfg=$(systemctl cat felhom-agent 2>/dev/null | grep -oP '(?<=-config )\S+' | head -1) + fi + [[ -n "$agent_cfg" ]] || agent_cfg="/etc/felhom-agent/agent.json" if systemctl list-unit-files felhom-agent.service >/dev/null 2>&1; then systemctl is-active --quiet felhom-agent 2>/dev/null && run systemctl stop felhom-agent systemctl is-enabled --quiet felhom-agent 2>/dev/null && run systemctl disable felhom-agent @@ -569,6 +578,36 @@ run_uninstall() { if [[ -d "$AGENT_STATE_DIR" ]]; then run rm -rf "$AGENT_STATE_DIR"; else log_skip " $AGENT_STATE_DIR already absent"; fi if id "$AGENT_USER" >/dev/null 2>&1; then run userdel "$AGENT_USER"; else log_skip " service user $AGENT_USER already absent"; fi + # 4b. Agent config (pve token + per-host hub api_key — secrets must not survive; drill R1). + if [[ -f "$agent_cfg" ]]; then run rm -f "$agent_cfg"; else log_skip " $agent_cfg already absent"; fi + run rmdir "$(dirname "$agent_cfg")" 2>/dev/null || true + + # 4c. Shared-parent unit + wrapper + /mnt/felhom-drives (agent-installed at runtime; drill R2). + # Stop/disable, remove unit + script, unbind + remove the (empty) parent dir. Tolerate-absent. + if systemctl list-unit-files felhom-shared-parent.service 2>/dev/null | grep -q felhom-shared-parent; then + systemctl is-active --quiet felhom-shared-parent 2>/dev/null && run systemctl stop felhom-shared-parent + systemctl is-enabled --quiet felhom-shared-parent 2>/dev/null && run systemctl disable felhom-shared-parent + else + log_skip " felhom-shared-parent unit not loaded — skip stop/disable" + fi + if [[ -f /etc/systemd/system/felhom-shared-parent.service ]]; then run rm -f /etc/systemd/system/felhom-shared-parent.service; else log_skip " felhom-shared-parent.service already absent"; fi + if [[ -f /usr/local/sbin/felhom-shared-parent.sh ]]; then run rm -f /usr/local/sbin/felhom-shared-parent.sh; fi + run systemctl daemon-reload + if mountpoint -q /mnt/felhom-drives 2>/dev/null; then run umount /mnt/felhom-drives; fi + if [[ -d /mnt/felhom-drives ]]; then run rmdir /mnt/felhom-drives 2>/dev/null || true; fi + + # 4d. Guarded-mkfs wrapper, guest-hook snippet, lan-resolver dnsmasq snippets (drill R3-R5). + if [[ -f /usr/local/sbin/felhom-mkfs-guarded ]]; then run rm -f /usr/local/sbin/felhom-mkfs-guarded; else log_skip " felhom-mkfs-guarded already absent"; fi + if [[ -f /var/lib/vz/snippets/felhom-guest-hook.sh ]]; then run rm -f /var/lib/vz/snippets/felhom-guest-hook.sh; fi + local dconf _dnsmasq_touched=false + for dconf in /etc/dnsmasq.d/felhom-*.conf; do + [[ -e "$dconf" ]] || continue + run rm -f "$dconf"; _dnsmasq_touched=true + done + if $_dnsmasq_touched && systemctl is-active --quiet dnsmasq 2>/dev/null; then + run systemctl restart dnsmasq || true + fi + # 5. pveum removal (presence-checked; tolerate-absent; roles deleted only after their grants). # Remove the 3-role scoped grants+roles (3b) AND the pre-3b single-role broad grant if present — # both tolerate-absent so --uninstall works on a box of either shape. @@ -624,9 +663,9 @@ run_uninstall() { # 8. Summary. echo "" - log_success "UNINSTALL complete — removed: guest $vmid, the felhom-agent (unit/sudoers/binary/state/user), the pveum role/user/token/ACL,$( $pool_removed && printf ' the %s pool,' "$PVE_POOL") and $STATE_FILE." + log_success "UNINSTALL complete — removed: guest $vmid, the felhom-agent (unit/sudoers/binary/state/config/user + shared-parent/mkfs-wrapper/hook-snippet/dnsmasq-snippets), the pveum role/user/token/ACL,$( $pool_removed && printf ' the %s pool,' "$PVE_POOL") and $STATE_FILE." if $REMOVE_GOLDEN; then log_info " golden vzdump: removed."; else log_info " golden vzdump: left in place (--remove-golden to remove)."; fi - log_info " NOTE: the 'sudo' package was left installed (system package); the host record still exists in the hub — remove it there if desired." + log_info " NOTE: the 'sudo' and 'dnsmasq' packages were left installed (system packages); the host record still exists in the hub — remove it there if desired." $DRY_RUN && log_warn " DRY-RUN: nothing above was actually executed." return 0 } @@ -1439,6 +1478,14 @@ step_provision() { # Record the provisioned vmid so a later --uninstall resolves the target automatically + safely. _state_put provisioned_vmid "$VMID" # (No pool_add_guest here — the agent's restore --pool already made the guest a member.) + # Reboot the guest ONCE: the golden's controller-bootstrap unit evaluates its + # ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json at BOOT, and the back-half attaches + # the bootstrap mount to the ALREADY-RUNNING guest — without a reboot the unit stays skipped and + # the controller never deploys (drill finding R6; the agent's own provision output says + # "next: reboot the guest"). On fast hosts the first boot sometimes wins the race — the reboot + # is idempotent either way (the unit no-ops when the controller already runs). + log_info " rebooting guest $VMID so the baked controller-bootstrap unit picks up the mount" + run pct reboot "$VMID" } #------------------------------------------------------------------------------- @@ -1452,9 +1499,14 @@ step_verify() { [[ "$st" == "running" ]] && log_success " pct status: running" || { log_error " pct status: $st"; ok=false; } if pct config "$VMID" 2>/dev/null | grep -q '^onboot: 1'; then log_success " onboot: 1"; else log_error " onboot NOT 1"; ok=false; fi pct config "$VMID" 2>/dev/null | grep -E '^(rootfs|mp0|mp1|mp8):' | sed 's/^/ /' - # controller container healthy in-guest - local cstat; cstat=$(pct exec "$VMID" -- docker ps --filter name=felhom-controller --format '{{.Status}}' 2>/dev/null | head -1) - if [[ -n "$cstat" ]]; then log_success " controller: $cstat"; else log_warn " controller container not visible yet (may still be starting)"; fi + # controller container healthy in-guest — bounded wait (the post-provision reboot + docker start + # take a while, especially on modest hardware; drill R6 re-verify) + local cstat="" _waited=0 + while [[ -z "$cstat" && $_waited -lt 180 ]]; do + cstat=$(pct exec "$VMID" -- docker ps --filter name=felhom-controller --format '{{.Status}}' 2>/dev/null | head -1) + [[ -n "$cstat" ]] || { sleep 5; _waited=$((_waited+5)); } + done + if [[ -n "$cstat" ]]; then log_success " controller: $cstat (after ~${_waited}s)"; else log_warn " controller container not visible after ${_waited}s — check 'pct exec $VMID -- journalctl -u felhom-controller-bootstrap'"; fi local cver; cver=$(pct exec "$VMID" -- docker ps --filter name=felhom-controller --format '{{.Image}}' 2>/dev/null | head -1) [[ -n "$cver" ]] && log_info " controller image: $cver" # tunnel