From ff1db11db477ab3569dc9f2787c5d7edcaa6c625 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 4 Oct 2026 20:18:37 +0200 Subject: [PATCH] =?UTF-8?q?hub=20v0.133.0=20(R-859=20test=20approvals=20en?= =?UTF-8?q?d=20with=20the=20test;=20R-840=20bundle=20on=20the=20System=20p?= =?UTF-8?q?age,=20manifest,=20alarm);=20installer=201.31.0=20(root=20files?= =?UTF-8?q?=20from=20the=20config=20bundle);=20bundle=20bootstrap=20script?= =?UTF-8?q?;=20golden=200.293.0=20evidence;=20rulings=2096=E2=80=9399;=20d?= =?UTF-8?q?rill-r50=20removed=20(evidence)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CONTEXT.md | 6 + .../architecture/09-update-architecture.md | 22 ++ .../partB/b1-demo-boxes-before.txt | 64 ++++ .../partB/b2-bootstrap-demo-hp.txt | 16 + .../partB/b3-bootstrap-demo-felhom.txt | 9 + .../b4-demo-hp-jobs-A-wrongsha-B-right.txt | 29 ++ .../partB/hub-bundle-redproof.txt | 8 + .../partB/redproof.txt | 24 ++ .../partD/d-redproof.txt | 7 + .../partE/e1-spike-restart.txt | 41 +++ .../partE/e2-spike-crash-and-socket.txt | 24 ++ .../partE/e3-blind-controller.txt | 30 ++ .../partE/e4-exit-restarts-on-new-socket.txt | 5 + .../partE/e5-traefik-stale-and-repaired.txt | 5 + .../partE/e6-redproof-controller.txt | 10 + .../partE/e7-live-heal-9202.txt | 41 +++ .../partF/f1-bake-step-smoke-9202.txt | 6 + .../partG/g1-before.txt | 39 ++ .../partG/g2-delete-body.txt | 0 .../partG/g2-delete-headers.txt | 5 + .../partG/g3-hub-log.txt | 7 + .../partG/g4-after.txt | 6 + .../02-round-trip.txt | 5 + .../tests/golden-0.293.0-2026-10-04/README.md | 60 +++ .../tests/golden-0.293.0-2026-10-04/bake.log | 342 ++++++++++++++++++ hub/CHANGELOG.md | 22 ++ hub/cmd/hub/main.go | 16 + hub/internal/api/handler.go | 5 + hub/internal/gitea/exact_test.go | 34 ++ hub/internal/gitea/gitea.go | 20 + hub/internal/notify/dispatcher.go | 12 +- hub/internal/osupdates/bundle_alarm_test.go | 90 +++++ hub/internal/osupdates/service.go | 122 ++++++- hub/internal/osupdates/testapproval_test.go | 173 +++++++++ hub/internal/store/os_updates.go | 112 +++++- hub/internal/store/store.go | 33 +- hub/internal/sysfacts/sysfacts.go | 29 +- hub/internal/web/configs.go | 15 +- hub/internal/web/server.go | 1 + hub/internal/web/system.go | 46 ++- hub/internal/web/system_bundle_test.go | 32 ++ hub/internal/web/templates/system.html | 19 +- scripts/CHANGELOG.md | 17 + scripts/felhom-bundle-bootstrap.sh | 54 +++ scripts/felhom-host-install.sh | 86 ++++- 45 files changed, 1707 insertions(+), 42 deletions(-) create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/b1-demo-boxes-before.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/b2-bootstrap-demo-hp.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/b3-bootstrap-demo-felhom.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/b4-demo-hp-jobs-A-wrongsha-B-right.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/hub-bundle-redproof.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partB/redproof.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partD/d-redproof.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e1-spike-restart.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e2-spike-crash-and-socket.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e3-blind-controller.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e4-exit-restarts-on-new-socket.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e5-traefik-stale-and-repaired.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e6-redproof-controller.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partE/e7-live-heal-9202.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partF/f1-bake-step-smoke-9202.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partG/g1-before.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-body.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-headers.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partG/g3-hub-log.txt create mode 100644 documentation/audits/r840-config-bundle-2026-10-04/partG/g4-after.txt create mode 100644 documentation/tests/golden-0.293.0-2026-10-04/02-round-trip.txt create mode 100644 documentation/tests/golden-0.293.0-2026-10-04/README.md create mode 100644 documentation/tests/golden-0.293.0-2026-10-04/bake.log create mode 100644 hub/internal/gitea/exact_test.go create mode 100644 hub/internal/osupdates/bundle_alarm_test.go create mode 100644 hub/internal/osupdates/testapproval_test.go create mode 100644 hub/internal/web/system_bundle_test.go create mode 100644 scripts/felhom-bundle-bootstrap.sh diff --git a/CONTEXT.md b/CONTEXT.md index a953a2fe..80ad9d2b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -16,6 +16,12 @@ > and holds nothing of its own; this file does hold its own content, namely the standing rulings below. +> **2026-10-04 (~18:49) — rulings 96–99 (the R-840 / Tester 2 brief), recorded before the work.** `09` §3: **96** build +> R-840 now (option A, a signed config bundle; replaces 82); **97** Tester 2 may receive only the signed agent update, +> the bundle by the route and the one-time live-restore reload (~19:05: the operator will install the bootstrap file by +> hand through his tunnel — the route cannot reach a root side that predates it); **98** drill-r50 removed from the hub, +> ep0 touch allowed (~19:05); tester-1 stays; **99** the operator's cause with the reviewer's correction. + > **2026-10-04 (~18:00) — R-858 incident + ruling 95.** The operator saw demo-felhom DOWN: the 14:13 UTC Docker step > left `felhom-controller` and `traefik` on the OLD socket (live-restore kept them running; their bind-mounted socket > file was recreated). Repaired by restarting the two (15:57 UTC). `09` §3 decision **95**: the wrapper restarts the diff --git a/documentation/architecture/09-update-architecture.md b/documentation/architecture/09-update-architecture.md index 1ba8112c..5f568cde 100644 --- a/documentation/architecture/09-update-architecture.md +++ b/documentation/architecture/09-update-architecture.md @@ -787,6 +787,28 @@ its length, and both fixes cost something the household would notice — operato controller templates and the golden on every box). Until the fix is released, the demo boxes' root-owned ring-0 Docker mark is OFF (no unsigned step). *Operator ruling 2026-10-04 ~18:00.* +### 2026-10-04 (~18:49) — four operator rulings (recorded before the work; the R-840 / Tester 2 brief) + +96. **R-840: build the route now, option A** (a signed `agent_config_update` op pinning the agent tag and the sha256 of + a config bundle; the root side installs it after its own checks), and use it on Tester 2. Every later box needs it. + **This replaces decision 82.** *Operator ruling 2026-10-04 ~18:49.* +97. **Tester 2 may receive, in this session only:** the signed agent update to the vouched agent; the config bundle + through the new route; the one-time `live-restore` reload through the wrapper (a reload, never a restart). Nothing + else: no Docker engine step, no reboot, no crash test, no app change. *Operator ruling 2026-10-04 ~18:49.* + **Added ~19:05, the operator's answer:** the first bundle cannot reach Tester 2 by the route (its root side predates + the route — see `11` §5.4.2); the operator will try to reach Tester 2 through his own WireGuard tunnel and install the + one bootstrap file by hand, with CC's written steps. +98. **drill-r50 is removed from the hub. tester-1 stays** (CC's disposable test box, a VM on the HP box, down when not + used). **Added ~19:05, the operator's answer:** the product delete may touch ep0 — it destroys drill-r50's PBS + namespace and token there and removes its WireGuard peer. *Operator ruling 2026-10-04 ~18:49 / ~19:05.* +99. **The operator's cause, with the reviewer's correction:** *"we didn't bake a new golden before he joined."* Partly: + the golden carries `live-restore` and the Docker version; the crash guard and the operator-signers file come from the + INSTALLER; the wrapper comes from the agent tag the installer pins. A newer golden alone would not have fixed it, and + with perfect timing the next wrapper change would still leave Tester 2 behind — that is R-840. **CC's measurement + (same evening) corrects the premise further:** Tester 2 was bound at 16:06 **UTC** (18:06 local), after installer + 1.30.0, agent 0.142.0 and the re-made golden; it lacks only agent 0.142.1's wrapper fix (R-858). *Operator ruling + 2026-10-04 ~18:49.* + ### 2026-10-04 (evening) — decided by CC unattended — operator may reverse (System page / Docker / crash-restart brief) 90. **How does a box tell a crash boot from a clean one?** Options: (a) `pstore` — measured on demo-hp: `efi_pstore` is on, diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/b1-demo-boxes-before.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/b1-demo-boxes-before.txt new file mode 100644 index 00000000..16ba01c5 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/b1-demo-boxes-before.txt @@ -0,0 +1,64 @@ +== demo-hp + SAME /usr/local/sbin/felhom-mkfs-guarded 755:root + SAME /usr/local/sbin/felhom-selfupdate-guarded 755:root + SAME /usr/local/sbin/felhom-pbs-apply 755:root + SAME /usr/local/sbin/felhom-backup-target-apply 755:root + DIFF /usr/local/sbin/felhom-os-apply 755:root + SAME /usr/local/sbin/felhom-crash-guard 755:root + SAME /etc/systemd/system/felhom-crash-guard.service 644:root + SAME /etc/systemd/system/felhom-crash-guard-check.service 644:root + SAME /etc/systemd/system/felhom-crash-guard-check.timer 644:root + SAME /etc/felhom/crash-guard.conf 644:root + SAME /etc/systemd/system/felhom-agent.service 644:root + SAME /etc/systemd/system/felhom-agent-rollback.service 644:root + SAME /etc/systemd/system/felhom-agent.service.d/felhom-agent-limits.conf 644:root + SAME /usr/local/sbin/felhom-mgmt-watchdog 755:root + SAME /etc/tmpfiles.d/felhom-privsep.conf 644:root + SAME /etc/systemd/system/felhom-mgmt-watchdog.service 644:root + SAME /etc/systemd/system/felhom-mgmt-watchdog.timer 644:root + SAME /etc/systemd/system/felhom-sshd.service 644:root + SAME /etc/felhom-oob.nft 644:root + SAME /etc/systemd/system/felhom-oob-nft.service 644:root + SAME /etc/sudoers.d/felhom-op 440:root + SAME /etc/sudoers.d/felhom-agent 440:root + total 24 + drwxr-xr-x 2 root root 4096 Oct 4 16:08 . + drwxr-xr-x 102 root root 4096 Oct 4 09:41 .. + -rw-r--r-- 1 root root 0 Aug 21 18:01 .bootstrap-done + -rw-r--r-- 1 root root 7 Aug 21 17:44 appliance-pairing-code + -rw-r--r-- 1 root root 456 Oct 4 16:08 crash-guard.conf + -rw-r--r-- 1 root root 131 Oct 4 16:08 operator-signers + -rw-r--r-- 1 root root 61 Oct 4 18:34 os-trust.json + felhom-agent 0.142.1 +== felhom-pve + SAME /usr/local/sbin/felhom-mkfs-guarded 755:root + SAME /usr/local/sbin/felhom-selfupdate-guarded 755:root + SAME /usr/local/sbin/felhom-pbs-apply 755:root + SAME /usr/local/sbin/felhom-backup-target-apply 755:root + DIFF /usr/local/sbin/felhom-os-apply 755:root + SAME /usr/local/sbin/felhom-crash-guard 755:root + SAME /etc/systemd/system/felhom-crash-guard.service 644:root + SAME /etc/systemd/system/felhom-crash-guard-check.service 644:root + SAME /etc/systemd/system/felhom-crash-guard-check.timer 644:root + SAME /etc/felhom/crash-guard.conf 644:root + SAME /etc/systemd/system/felhom-agent.service 644:root + SAME /etc/systemd/system/felhom-agent-rollback.service 644:root + SAME /etc/systemd/system/felhom-agent.service.d/felhom-agent-limits.conf 644:root + SAME /usr/local/sbin/felhom-mgmt-watchdog 755:root + SAME /etc/tmpfiles.d/felhom-privsep.conf 644:root + SAME /etc/systemd/system/felhom-mgmt-watchdog.service 644:root + SAME /etc/systemd/system/felhom-mgmt-watchdog.timer 644:root + SAME /etc/systemd/system/felhom-sshd.service 644:root + SAME /etc/felhom-oob.nft 644:root + SAME /etc/systemd/system/felhom-oob-nft.service 644:root + SAME /etc/sudoers.d/felhom-op 440:root + SAME /etc/sudoers.d/felhom-agent 440:root + total 24 + drwxr-xr-x 2 root root 4096 Oct 4 16:12 . + drwxr-xr-x 102 root root 4096 Oct 4 13:35 .. + -rw-r--r-- 1 root root 0 Jul 18 18:32 .bootstrap-done + -rw-r--r-- 1 root root 7 Jul 18 18:17 appliance-pairing-code + -rw-r--r-- 1 root root 456 Oct 4 16:12 crash-guard.conf + -rw-r--r-- 1 root root 131 Oct 4 16:12 operator-signers + -rw-r--r-- 1 root root 65 Oct 4 18:34 os-trust.json + felhom-agent 0.142.1 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/b2-bootstrap-demo-hp.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/b2-bootstrap-demo-hp.txt new file mode 100644 index 00000000..7d28ce42 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/b2-bootstrap-demo-hp.txt @@ -0,0 +1,16 @@ +--- wrong sha (red): +1/4 download https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.143.0/felhom-config-bundle.json +STOP: the bundle's sha256 is 8d7273cf5313ef62b867cb6f831c631923a436452d6f90b8ff7f0771170396ba, not 0000000000000000000000000000000000000000000000000000000000000000 — nothing changed +rc=1 +bdf60f5c79a84db7ebcfe1620fe832436caab831259308906eda08627aac7260 /usr/local/sbin/felhom-os-apply +--- right sha: +1/4 download https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.143.0/felhom-config-bundle.json + sha256 OK (8d7273cf5313ef62b867cb6f831c631923a436452d6f90b8ff7f0771170396ba) +2/4 take felhom-os-apply out of the bundle and check it + felhom-os-apply sha256 4c0d5f073b00c96891e2a9d4a50f2374dec38bc8c8ef37578027b1b399611dba +3/4 install it (the previous copy is kept as /usr/local/sbin/felhom-os-apply.pre-bundle) +4/4 self-check + felhom-os-apply ok bundle-format=1 files=22 +DONE. This box can now take signed config bundles. Nothing else was changed. +rc=0 +4c0d5f073b00c96891e2a9d4a50f2374dec38bc8c8ef37578027b1b399611dba /usr/local/sbin/felhom-os-apply diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/b3-bootstrap-demo-felhom.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/b3-bootstrap-demo-felhom.txt new file mode 100644 index 00000000..44f8b494 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/b3-bootstrap-demo-felhom.txt @@ -0,0 +1,9 @@ +1/4 download https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/0.143.0/felhom-config-bundle.json + sha256 OK (8d7273cf5313ef62b867cb6f831c631923a436452d6f90b8ff7f0771170396ba) +2/4 take felhom-os-apply out of the bundle and check it + felhom-os-apply sha256 4c0d5f073b00c96891e2a9d4a50f2374dec38bc8c8ef37578027b1b399611dba +3/4 install it (the previous copy is kept as /usr/local/sbin/felhom-os-apply.pre-bundle) +4/4 self-check + felhom-os-apply ok bundle-format=1 files=22 +DONE. This box can now take signed config bundles. Nothing else was changed. +rc=0 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/b4-demo-hp-jobs-A-wrongsha-B-right.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/b4-demo-hp-jobs-A-wrongsha-B-right.txt new file mode 100644 index 00000000..02a5387e --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/b4-demo-hp-jobs-A-wrongsha-B-right.txt @@ -0,0 +1,29 @@ + felhom-os-apply ok bundle-format=1 files=22 +DONE. This box can now take signed config bundles. Nothing else was changed. +rc=0 +Oct 04 20:04:34 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:34.923+02:00 level=INFO msg="audit: gate decision" class=agent_config_update host=demo-hp-bb76ea guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=10242c3b… durable_id="" +Oct 04 20:04:34 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:34.923+02:00 level=INFO msg="gate decision" class=agent_config_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed +Oct 04 20:04:34 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:34.923+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=1e60311ec9cad857 op=agent_config_update key_id=felhom-op-1 nonce=10242c3bac90859807c1ab3dfd497a9a +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.002+02:00 level=ERROR msg="signedjobs: signed op execution FAILED (nonce spent — clearing)" job=1e60311ec9cad857 op=agent_config_update err="agent_config_update: the downloaded bundle's sha256 is 8d7273cf5313ef62b867cb6f831c631923a436452d6f90b8ff7f0771170396ba, the signed job pins 111111111111111111111111111111111111111111111 +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.016+02:00 level=INFO msg="audit: gate decision" class=agent_config_update host=demo-hp-bb76ea guest="" source=one_shot_job disposition=destructive allowed=true reason=signed key_id=felhom-op-1 nonce=eb8a8219… durable_id="" +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.016+02:00 level=INFO msg="gate decision" class=agent_config_update guest="" source=one_shot_job disposition=destructive allowed=true reason=signed +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.016+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=fa3df5a93740111e op=agent_config_update key_id=felhom-op-1 nonce=eb8a8219c7ab393e6702dc8a34cc578f +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.032+02:00 level=INFO msg="osupdate: config bundle downloaded; handing it to the root wrapper" op=agent_config_update agent_version=0.143.0 sha256=8d7273cf5313ef62 duration_ms=15 +Oct 04 20:04:35 demo-hp felhom-os-apply[496014]: os-apply: BUNDLE START agent=0.143.0 sha=8d7273cf5313ef62 authority=signed files=22 write=0 same=21 kept=1 skipped=0 +Oct 04 20:04:35 demo-hp felhom-os-apply[496158]: os-apply: BUNDLE DONE agent=0.143.0 written=0 same=21 self-check=ok signers-created=False +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.985+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: BUNDLE START agent=0.143.0 sha=8d7273cf5313ef62 authority=signed files=22 write=0 same=21 kept=1 skipped=0" +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.985+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: BUNDLE DONE agent=0.143.0 written=0 same=21 self-check=ok signers-created=False" +Oct 04 20:04:35 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:35.985+02:00 level=WARN msg="osupdate: config bundle INSTALLED" op=agent_config_update agent_version=0.143.0 bundle="{\"agent_version\": \"0.143.0\", \"authority\": \"signed\", \"kept\": [\"/etc/felhom/crash-guard.conf\"], \"prev_dir\": \"/var/lib/felhom-os-apply/bundle-prev/20261004T180435Z-before-0.143.0\", \"same\": 21, \"self_ +Oct 04 20:04:36 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:36.907+02:00 level=WARN msg="osupdate: capability probe after the config bundle" ok=71 total=71 degraded="" +Oct 04 20:04:36 demo-hp felhom-agent[447632]: time=2026-10-04T20:04:36.907+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=fa3df5a93740111e op=agent_config_update + +[exited with code 0] +{ + "agent_version": "0.143.0", + "authority": "signed", + "bundle_sha256": "8d7273cf5313ef62b867cb6f831c631923a436452d6f90b8ff7f0771170396ba", + "files": { + "/etc/felhom-oob.nft": "2c2b9cca89a439ac44efd353f4ea1936b3609ca43491b4c1853483e8321f238f", + "/etc/felhom/crash-guard.conf": "9b9d305b421f81d223f2ccc4594e67d41618239779d983a2f72c191a8b3b65f7", + "/etc/sudoers.d/felhom-agent": "02df92d751f1780aecbf600b2632b2366fcedeb3601852ebfa98c700e95f4dfc", +-rw-r--r-- 1 root root 2754 Oct 4 20:04 /etc/felhom/config-bundle.json diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/hub-bundle-redproof.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/hub-bundle-redproof.txt new file mode 100644 index 00000000..5887b2b6 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/hub-bundle-redproof.txt @@ -0,0 +1,8 @@ +RED no 7-day wait +RED unknown counted as behind +RED clock never clears +GREEN! alarm without a vouched bundle +RED cell never red +RED exact lookup falls back to the first file +unmutated: ok +RED alarm without a vouched bundle (test strengthened: a box WITH a bundle, nothing vouched) diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partB/redproof.txt b/documentation/audits/r840-config-bundle-2026-10-04/partB/redproof.txt new file mode 100644 index 00000000..8240570d --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partB/redproof.txt @@ -0,0 +1,24 @@ +RED R17 trust-path check removed -> test_a_bundle_that_changes_a_signer_is_refused +RED R16 table check removed -> test_a_path_outside_the_table_is_refused +RED bundle sha check removed -> test_wrong_sha_is_refused +RED per-file sha check removed -> test_content_not_matching_its_sha_is_refused +RED version pin removed -> test_version_mismatch_is_refused +RED visudo/sh/nft content check removed -> test_sudoers_failing_visudo_is_refused +RED python compile check removed -> test_python_syntax_error_is_refused +RED RuntimeDirectory guard removed -> test_unit_with_runtime_directory_is_refused +RED agent-unit User= check removed -> test_agent_unit_not_as_the_agent_user_is_refused +RED route-line pre-check removed -> test_sudoers_dropping_the_route_is_refused +RED wrapper bundle-mode pre-check removed -> test_wrapper_without_bundle_mode_is_refused +RED table re-ordering removed -> test_sudoers_is_written_after_every_wrapper +RED self-check sudo -l removed -> test_route_missing_after_install_puts_everything_back +RED undo removed -> test_route_missing_after_install_puts_everything_back +RED crash-guard kernel.panic self-check removed -> test_crash_guard_disagreeing_with_kernel_panic_puts_everything_back +RED nonce burn removed -> test_fresh_box_gets_every_file_and_a_record +RED nonce burned before the sha check -> test_wrong_sha_is_refused +RED pinned-key bootstrap removed (always the file) -> test_missing_signers_verifies_against_the_pinned_key_and_creates_it +RED signers written even when present -> test_present_signers_are_never_touched +RED SUDO_UID refusal removed -> test_installer_entry_is_refused_through_sudo +RED if-absent policy ignored -> test_tuned_crash_guard_conf_is_kept +RED oob policy ignored -> test_fresh_box_gets_every_file_and_a_record +unmutated: OK +ALL RED diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partD/d-redproof.txt b/documentation/audits/r840-config-bundle-2026-10-04/partD/d-redproof.txt new file mode 100644 index 00000000..11be20eb --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partD/d-redproof.txt @@ -0,0 +1,7 @@ +RED approval never marked +RED a cancelled release is still served +RED cancels while the override is still on +RED superseded ones cancelled too +RED no backfill +RED ring-1 boxes not bumped +unmutated: ok diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e1-spike-restart.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e1-spike-restart.txt new file mode 100644 index 00000000..d16911bc --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e1-spike-restart.txt @@ -0,0 +1,41 @@ ++ date -u +%FT%TZ +2026-10-04T17:19:25Z ++ docker ps -q --no-trunc ++ sort ++ wc -l +6 ++ pidof dockerd ++ echo dockerd_pid=225 ++ stat -c host_sock_inode=%i /var/run/docker.sock +dockerd_pid=225 +host_sock_inode=144 ++ docker exec felhom-controller sh -c stat -c %i /var/run/docker.sock +felhom-controller sees: 144 ++ echo felhom-controller sees: 144 ++ docker exec traefik sh -c stat -c %i /var/run/docker.sock +traefik sees: 144 ++ echo traefik sees: 144 ++ systemctl restart docker ++ date -u +%FT%TZ +2026-10-04T17:19:31Z ++ sleep 5 ++ pidof dockerd +dockerd_pid=66020 ++ echo dockerd_pid=66020 ++ stat -c host_sock_inode=%i /var/run/docker.sock +host_sock_inode=144 ++ docker ps -q --no-trunc ++ sort ++ diff /tmp/ids.before /tmp/ids.after +IDS-SAME ++ echo IDS-SAME ++ docker exec felhom-controller sh -c stat -c %i /var/run/docker.sock +felhom-controller sees: 144 ++ echo felhom-controller sees: 144 ++ docker exec traefik sh -c stat -c %i /var/run/docker.sock +traefik sees: 144 ++ echo traefik sees: 144 ++ docker exec felhom-controller docker version --format {{.Server.Version}} ++ tail -1 +controller->docker: 29.8.2 ++ echo controller->docker: 29.8.2 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e2-spike-crash-and-socket.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e2-spike-crash-and-socket.txt new file mode 100644 index 00000000..dabf4219 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e2-spike-crash-and-socket.txt @@ -0,0 +1,24 @@ +After=network-online.target nss-lookup.target docker.socket firewalld.service containerd.service time-set.target +Wants=network-online.target containerd.service +Requires=docker.socket +ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock +Restart=always +# specify ListenStream=/var/run/docker.sock instead. +ListenStream=/run/docker.sock +SocketMode=0660 +== kill -9 dockerd +2026-10-04T17:19:52Z +dockerd_pid=66638 +host_sock_inode=144 +IDS-SAME +controller->docker: 29.8.2 +== systemctl restart docker.socket +2026-10-04T17:20:00Z +active +active +dockerd_pid=67121 +host_sock_inode=7202 +IDS-SAME +felhom-controller sees: 144 +traefik sees: 144 +controller->docker: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e3-blind-controller.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e3-blind-controller.txt new file mode 100644 index 00000000..65bc9b05 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e3-blind-controller.txt @@ -0,0 +1,30 @@ +2026-10-04T17:22:27Z +felhom-controller Up 2 hours (healthy) +traefik Up 2 hours + (took 32ms) +2026/10/04 17:22:09 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: failed after 32ms: docker ps: exec docker ps -a --format {{.Names}} {{.Image}} {{.State}} {{.Status}} {{.Label "com.docker.compose.project"}} --no-trunc: exit status 1 +stderr: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? +2026/10/04 17:22:09 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (0 skipped not due, 0 skipped no container) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] GetInfo starting (hddPath="/mnt/felhom-drives/scratch_hdd", hasCPUCollector=true) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readMemInfo: totalKB=30714356 availKB=25620992 → total=29994MB avail=25020MB used=4974MB (16.6%) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readDiskUsage: path="/" bsize=4096 total=68.4GB used=5.6GB avail=59.2GB (8.3%) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readDiskUsage: path="/mnt/felhom-drives/scratch_hdd" bsize=4096 total=937.8GB used=15.1GB avail=875.0GB (1.6%) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readLoadAvg: raw="0.50 0.90 0.74 5/1182 23856" → 1m=0.50 5m=0.90 15m=0.74 +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readThermalZones: /sys — found 1 zones +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] readTemperature: found via hwmon at /sys — 52.9°C (hwmon1) +2026/10/04 17:22:19 info.go:13: [DEBUG] [system] GetInfo done in 47ms — mem=4974MB/29994MB (16.6%), rootDisk=5.6GB/68.4GB (8.3%), load=0.50/0.90/0.74, temp=52.9°C (hwmon1), cpu=2.8% +2026/10/04 17:22:19 collector.go:107: [WARN] [metrics] docker stats failed: exit status 1 +2026/10/04 17:22:19 scheduler.go:67: [DEBUG] [scheduler] job ring-spill: execution starting +2026/10/04 17:22:19 scheduler.go:67: [DEBUG] [scheduler] job deadapp-check: execution starting +2026/10/04 17:22:19 scheduler.go:67: [DEBUG] [scheduler] job health-probes: execution starting +2026/10/04 17:22:19 healthprobe.go:110: [DEBUG] [stacks] RunHealthProbes: collected 0 targets (0 skipped not due, 0 skipped no container) +2026/10/04 17:22:19 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: execution starting +2026/10/04 17:22:19 manager.go:1539: [ERROR] [stacks] execCommand failed: exit status 1 +2026/10/04 17:22:19 scheduler.go:360: [ERROR] [scheduler] Job status-refresh failed: docker ps: exec docker ps -a --format {{.Names}} {{.Image}} {{.State}} {{.Status}} {{.Label "com.docker.compose.project"}} --no-trunc: exit status 1 +stderr: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? + (took 24ms) +2026/10/04 17:22:19 scheduler.go:67: [DEBUG] [scheduler] job status-refresh: failed after 24ms: docker ps: exec docker ps -a --format {{.Names}} {{.Image}} {{.State}} {{.Status}} {{.Label "com.docker.compose.project"}} --no-trunc: exit status 1 +stderr: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? +2026/10/04 17:22:19 manager.go:1539: [ERROR] [stacks] execCommand failed: exit status 1 + +dashboard=302 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e4-exit-restarts-on-new-socket.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e4-exit-restarts-on-new-socket.txt new file mode 100644 index 00000000..16103c2b --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e4-exit-restarts-on-new-socket.txt @@ -0,0 +1,5 @@ +1502 /usr/local/bin/felhom-controller +2026-10-04T17:22:40Z +17:22:45 running 2026-10-04T17:22:41.057453139Z restarts=1 +controller sees: 7202 host: 7202 +controller->docker: 29.8.2 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e5-traefik-stale-and-repaired.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e5-traefik-stale-and-repaired.txt new file mode 100644 index 00000000..64ef6f2d --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e5-traefik-stale-and-repaired.txt @@ -0,0 +1,5 @@ +traefik before: 144 +2026-10-04T17:22:38Z ERR Provider error, retrying in 8.088174175s error="Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?" providerName=docker +2026-10-04T17:22:46Z ERR Failed to retrieve information of the docker client and server host error="Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?" providerName=docker +2026-10-04T17:22:46Z ERR Provider error, retrying in 11.956516688s error="Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?" providerName=docker +traefik after: 7202 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e6-redproof-controller.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e6-redproof-controller.txt new file mode 100644 index 00000000..f4c79bdb --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e6-redproof-controller.txt @@ -0,0 +1,10 @@ +RED never-worked guard removed +RED timeouts counted as refusals +RED exit removed +RED window ignored (exit at first refusal) +RED a success does not reset +RED CheckUsers acts while blind +RED same-inode skip removed +RED self not skipped +RED ENOENT not a refusal +unmutated: ok diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partE/e7-live-heal-9202.txt b/documentation/audits/r840-config-bundle-2026-10-04/partE/e7-live-heal-9202.txt new file mode 100644 index 00000000..5c9c756e --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partE/e7-live-heal-9202.txt @@ -0,0 +1,41 @@ +T0 17:50:17 systemctl restart docker.socket +host sock inode: 9108 ++10s controller->docker= +BLIND traefik_inode=7202 host=9108 ++16s controller->docker= +BLIND traefik_inode=7202 host=9108 ++21s controller->docker= +BLIND traefik_inode=7202 host=9108 ++26s controller->docker= +BLIND traefik_inode=7202 host=9108 ++31s controller->docker= +BLIND traefik_inode=7202 host=9108 ++36s controller->docker= +BLIND traefik_inode=7202 host=9108 ++41s controller->docker= +BLIND traefik_inode=7202 host=9108 ++47s controller->docker= +BLIND traefik_inode=7202 host=9108 ++52s controller->docker= +BLIND traefik_inode=7202 host=9108 ++57s controller->docker= +BLIND traefik_inode=7202 host=9108 ++62s controller->docker= +BLIND traefik_inode=7202 host=9108 ++67s controller->docker= +BLIND traefik_inode=7202 host=9108 ++73s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++78s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++83s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++88s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++94s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++99s controller->docker=29.8.2 traefik_inode=7202 host=9108 ++104s controller->docker=29.8.2 traefik_inode=9108 host=9108 +HEALED +== containers before/after (name id) +== controller log +2026/10/04 17:50:29 sockheal.go:118: [WARN] [sockheal] Docker refuses the socket (dial unix /var/run/docker.sock: connect: connection refused) — exiting after 1m0s of refusals so Docker restarts this controller on the current socket (R-860) +2026/10/04 17:51:29 sockheal.go:123: [ERROR] [sockheal] Docker has refused the socket for 1m0s (dial unix /var/run/docker.sock: connect: connection refused) — the socket file was re-created and this container holds the old one; EXITING (code 75) so Docker's restart policy brings it back on the current socket (R-860) +2026/10/04 17:52:00 sockheal.go:155: [WARN] [sockheal] traefik holds an old docker socket (inode 7202, current 9108) — restarting it (R-860) +2026/10/04 17:52:01 sockheal.go:160: [INFO] [sockheal] traefik restarted onto the current docker socket +controller restarts=1 started=2026-10-04T17:51:30.005016139Z diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partF/f1-bake-step-smoke-9202.txt b/documentation/audits/r840-config-bundle-2026-10-04/partF/f1-bake-step-smoke-9202.txt new file mode 100644 index 00000000..898cf6cb --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partF/f1-bake-step-smoke-9202.txt @@ -0,0 +1,6 @@ +[golden] approved guest release: the template already runs every approved version +[golden] first-night count vs the approved guest release: 0 (target 0) +rc=0 +[golden] no approved guest release given ��� the template versions stay; first-night count vs an approved release: n/a +[golden] pending Debian upgrades in the baked guest (what a FUTURE approval may bring): 1 +rc=0 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partG/g1-before.txt b/documentation/audits/r840-config-bundle-2026-10-04/partG/g1-before.txt new file mode 100644 index 00000000..6cfef702 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partG/g1-before.txt @@ -0,0 +1,39 @@ +Sun Oct 4 17:08:29 UTC 2026 +# GET /configs/drill-r50/delete (preview) +{ + "claim_present": true, + "cloudflare_manual": null, + "customer_id": "drill-r50", + "customer_name": "drill-r50", + "dr_recipe_present": true, + "has_config": true, + "host_count": 1, + "hosts": [ + { + "host_id": "drill-r50-0a4f9a", + "online": false, + "status": "down" + } + ], + "offsite_enabled": false, + "offsite_identifier": "", + "offsite_type": "", + "one_time_secret": false, + "online_host_present": false, + "pbs_tenancy_configured": true, + "pending_journal": null, + "residue": { + "app_log_tails": 0, + "app_telemetry": 185, + "appliance_registrations": 1, + "log_tail_requests": 0, + "notification_prefs": 0, + "reports": 185, + "selfbind_tokens": 0 + }, + "residue_total": 371, + "superseded_blobs": 0 +} +# GET /hosts/drill-r50-0a4f9a/delete-impact +{"deletable":true,"escrow_present":false,"guests":1,"log_bundles":0,"pbs_secret_present":false,"recovery_present":true,"reports":222,"status":"down","wg_peer_bound":true} +# hub DB (read-only copy): wg peer 10.77.0.4 bound to drill-r50-0a4f9a; customer email empty (no mail can go out); dr_tier 0; host last report 2026-08-13 06:11:41Z, agent 0.129.0 diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-body.txt b/documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-body.txt new file mode 100644 index 00000000..e69de29b diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-headers.txt b/documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-headers.txt new file mode 100644 index 00000000..bd9bb859 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partG/g2-delete-headers.txt @@ -0,0 +1,5 @@ +HTTP/1.1 303 See Other +Location: /configs?flash=deleted +Date: Sun, 04 Oct 2026 17:08:35 GMT +Content-Length: 0 + diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partG/g3-hub-log.txt b/documentation/audits/r840-config-bundle-2026-10-04/partG/g3-hub-log.txt new file mode 100644 index 00000000..18023d8a --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partG/g3-hub-log.txt @@ -0,0 +1,7 @@ +2026/10/04 19:08:30 [INFO] customer DELETE cascade started for drill-r50 (journal #22, 1 host(s)) +2026/10/04 19:08:32 [INFO] delete drill-r50: host drill-r50-0a4f9a deleted (escrow DEMOTED to retained custody) +2026/10/04 19:08:33 [INFO] tenantsync: deprovision ok for drill-r50 (ns=drill-r50, existed=false) +2026/10/04 19:08:33 [INFO] reset drill-r50: PBS tenancy deprovisioned +2026/10/04 19:08:33 [INFO] [claim] reset to unclaimed for drill-r50 (customer RESET) — next onboarding mints a fresh code +2026/10/04 19:08:35 [INFO] delete drill-r50: residue purged (reports=185 app_telemetry=185 app_log_tails=0 log_tail_requests=0 notif_prefs=0 selfbind_tokens=0 appliance_registrations=1) +2026/10/04 19:08:35 [INFO] customer DELETE cascade COMPLETE for drill-r50 (journal #22) — full teardown diff --git a/documentation/audits/r840-config-bundle-2026-10-04/partG/g4-after.txt b/documentation/audits/r840-config-bundle-2026-10-04/partG/g4-after.txt new file mode 100644 index 00000000..ae5fc9d2 --- /dev/null +++ b/documentation/audits/r840-config-bundle-2026-10-04/partG/g4-after.txt @@ -0,0 +1,6 @@ +Sun Oct 4 17:08:46 UTC 2026 +# /hosts after +0 +# preview after +404 page not found + http=404 \ No newline at end of file diff --git a/documentation/tests/golden-0.293.0-2026-10-04/02-round-trip.txt b/documentation/tests/golden-0.293.0-2026-10-04/02-round-trip.txt new file mode 100644 index 00000000..4d6ddc2f --- /dev/null +++ b/documentation/tests/golden-0.293.0-2026-10-04/02-round-trip.txt @@ -0,0 +1,5 @@ +Sun Oct 4 18:15:55 UTC 2026 +anonymous GET https://gitea.dooplex.hu/api/packages/admin/generic/felhom-golden/0.293.0/golden.tar.zst +http=200 bytes=648135998 +sha256=e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7 +expected=e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7 diff --git a/documentation/tests/golden-0.293.0-2026-10-04/README.md b/documentation/tests/golden-0.293.0-2026-10-04/README.md new file mode 100644 index 00000000..2c149622 --- /dev/null +++ b/documentation/tests/golden-0.293.0-2026-10-04/README.md @@ -0,0 +1,60 @@ +# Golden 0.293.0 — bake + publish, 2026-10-04 + +Procedure: `documentation/runbooks/RUNBOOK-manual-build.md` §4.0 and §4.1 steps 1–4, in the drill VM on DooPlex. +Step 5 (vouching in the hub, the floor) was **not** done here — it is the main session's act. + +| | Previous (`../golden-0.292.0-2026-10-04-rebake/`) | This bake | +|---|---|---| +| `build-golden.sh` | v3.1.0 | **v3.2.0** (agent `dc9164c`, sha256 `645b3b659cba…`; VM copy matched) | +| Controller | `felhom-controller:0.292.0` | **`felhom-controller:0.293.0`** (MinAgent 0.131.0, unchanged) | +| Docker engine | pinned, approved set | same pinned set: the operator-approved release `os-docker-20261004-142842` (stays in force) | +| Guest packages | template | template — `GOLDEN_GUEST_PKGS` deliberately EMPTY (see below) | + +**Why the guest list is empty.** The only guest release, `os-guest-20261004-123933`, was approved under a TEST wait; +hub v0.133.0 (R-859) cancels it at start. No guest release is in force tonight, so the bake installs no guest fixes, +and the box's first night installs whatever release is approved then. The bake reported **49 pending Debian upgrades** +in the baked guest — what a future approval may bring, NOT what the first night installs (a ring-1 box installs only +an approved release; with none in force, 0). + +## Launch + +- Drill VM reverted to `virgin`, cold-booted per §4.0; `pveversion` = `pve-manager/9.2.2`. +- `pveam update` → `update successful`; template `debian-13-standard_13.6-1_amd64.tar.zst` (the only `_amd64` + debian-13 entry), downloaded, checksum verified. +- `/root/bake-run.sh` in the VM reads the token from the file and exports `GOLDEN_DOCKER_PKGS` (the six approved + versions) and `GOLDEN_GUEST_PKGS=""`; launched as transient unit `golden-bake`. +- Token copied file → file (`scp`). `systemctl show golden-bake -p Environment -p ExecStart | grep -c -F ` = + **0** (control with the token appended = **1**). + +## Pass markers (from `bake.log`) + +``` +[golden] Docker engine set PINNED to the approved release: containerd.io=2.3.6-1~debian.13~trixie … docker-ce=5:29.8.2-1~debian.13~trixie … +[golden] no approved guest release given - the template versions stay; first-night count vs an approved release: n/a +[golden] pending Debian upgrades in the baked guest (what a FUTURE approval may bring): 49 + docker OK (overlay2; data-root /var/lib/docker) + live-restore: on +INFO: including mount point rootfs ('/') in backup +INFO: including mount point mp0 ('/var/lib/felhom') in backup +[golden] upload OK (HTTP 201) +GOLDEN_VERSION=0.293.0 +GOLDEN_SHA256=e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7 +``` + +No `excluding` and no `FATAL` in the log. Pre-delete before upload: HTTP 404 (a new version, nothing replaced). + +## Round trip + +Anonymous GET of `…/generic/felhom-golden/0.293.0/golden.tar.zst`: HTTP 200, **648135998 bytes**, sha256 +`e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7` = the printed sha (`02-round-trip.txt`). + +## Secrets + +Saved-log leak grep for the literal token: **0**; positive control (a throwaway copy with the token appended): **1**, +copy shredded. + +## Teardown + +`pct destroy 9100 --purge`; `shred -u` of the token, the runner script and the log in the VM (log copied off first); +`poweroff`; qemu gone (`ps -eo comm | grep -c qemu-system-x86` = 0); `qemu-img snapshot -a virgin`. Host: nothing +provisioned. Hub: not touched. diff --git a/documentation/tests/golden-0.293.0-2026-10-04/bake.log b/documentation/tests/golden-0.293.0-2026-10-04/bake.log new file mode 100644 index 00000000..db62fdab --- /dev/null +++ b/documentation/tests/golden-0.293.0-2026-10-04/bake.log @@ -0,0 +1,342 @@ +[golden] build-golden.sh v3.2.0 — baking controller gitea.dooplex.hu/admin/felhom-controller:0.293.0 +[golden] creating build LXC 9100 (nesting=1,keyctl=1, unprivileged; rootfs 32G + ONE data volume 24G @ /var/lib/felhom, backup=1) … + Logical volume "vm-9100-disk-0" created. + Logical volume pve/vm-9100-disk-0 changed. +Creating filesystem with 8388608 4k blocks and 2097152 inodes +Filesystem UUID: 2ba3ff4c-bddd-47f5-b0dc-5f1949f4c908 +Superblock backups stored on blocks: + 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, + 4096000, 7962624 + Logical volume "vm-9100-disk-1" created. + Logical volume pve/vm-9100-disk-1 changed. +Creating filesystem with 6291456 4k blocks and 1572864 inodes +Filesystem UUID: 6565d64c-6e4b-45bc-b4ea-59f5bd2f7d91 +Superblock backups stored on blocks: + 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, +extracting archive '/var/lib/vz/template/cache/debian-13-standard_13.6-1_amd64.tar.zst' +Total bytes read: 553512960 (528MiB, 148MiB/s) +Detected container architecture: amd64 +Creating SSH host key 'ssh_host_ed25519_key' - this may take some time ... +done: SHA256:bGe7bRQch/Qk8hF5xTuvA4Mxm+o6J4x3G1W6uVOxLR0 root@felhom-golden +Creating SSH host key 'ssh_host_ecdsa_key' - this may take some time ... +done: SHA256:NCGNrT/PBO54a0wAfNAyyTzRoJbPfgrmNSmZDW+x2y0 root@felhom-golden +Creating SSH host key 'ssh_host_rsa_key' - this may take some time ... +done: SHA256:2yzTbaa+8zgXS4BGskQLtirKN/dy1YKmVddc2s7925o root@felhom-golden +[golden] starting + installing Docker (official repo, trixie channel) … +[golden] Docker engine set PINNED to the approved release: containerd.io=2.3.6-1~debian.13~trixie docker-buildx-plugin=0.37.1-1~debian.13~trixie docker-ce=5:29.8.2-1~debian.13~trixie docker-ce-cli=5:29.8.2-1~debian.13~trixie docker-ce-rootless-extras=5:29.8.2-1~debian.13~trixie docker-compose-plugin=5.6.0-1~debian.13~trixie +apt-listchanges: Can't set locale; make sure $LC_* and $LANG are correct! +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = (unset), + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to the standard locale ("C"). +locale: Cannot set LC_CTYPE to default locale: No such file or directory +locale: Cannot set LC_MESSAGES to default locale: No such file or directory +locale: Cannot set LC_ALL to default locale: No such file or directory +apt-listchanges: Can't set locale; make sure $LC_* and $LANG are correct! +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = (unset), + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to the standard locale ("C"). +locale: Cannot set LC_CTYPE to default locale: No such file or directory +locale: Cannot set LC_MESSAGES to default locale: No such file or directory +locale: Cannot set LC_ALL to default locale: No such file or directory + installed: containerd.io 2.3.6-1~debian.13~trixie + installed: docker-buildx-plugin 0.37.1-1~debian.13~trixie + installed: docker-ce 5:29.8.2-1~debian.13~trixie + installed: docker-ce-cli 5:29.8.2-1~debian.13~trixie + installed: docker-ce-rootless-extras 5:29.8.2-1~debian.13~trixie + installed: docker-compose-plugin 5.6.0-1~debian.13~trixie +[golden] no approved guest release given - the template versions stay; first-night count vs an approved release: n/a +[golden] pending Debian upgrades in the baked guest (what a FUTURE approval may bring): 49 +[golden] baking daemon.json: classic overlay2 driver (containerd-snapshotter OFF) + log rotation … +[golden] wiring the single data volume (R-165 variant V-c): /var/lib/felhom/{docker,sys_drive} -> binds … +[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) … +Unable to find image 'hello-world:latest' locally +latest: Pulling from library/hello-world +4f55086f7dd0: Pulling fs layer +4f55086f7dd0: Verifying Checksum +4f55086f7dd0: Download complete +4f55086f7dd0: Pull complete +Digest: sha256:5e23090353324d887c48ad5e5c56d294eab81588df9605b07d1afe895f9cc8f8 +Status: Downloaded newer image for hello-world:latest + docker OK (overlay2; data-root /var/lib/docker) + live-restore: on + /var/lib/docker is a real mount: /dev/mapper/pve-vm--9100--disk--1[/docker] ext4 + /mnt/sys_drive is a real mount: /dev/mapper/pve-vm--9100--disk--1[/sys_drive] ext4 + both paths are ONE filesystem: /dev/mapper/pve-vm--9100--disk--1 23317576 +[golden] baking the in-guest controller image gitea.dooplex.hu/admin/felhom-controller:0.293.0 (no registry cred at deploy) … + +WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'. +Configure a credential helper to remove this warning. See +https://docs.docker.com/go/credential-store/ + +0.293.0: Pulling from admin/felhom-controller +774043ccc8cc: Pulling fs layer +ab6b448d4be9: Pulling fs layer +23a5bfa58353: Pulling fs layer +862a57157567: Pulling fs layer +c3ec21eb492c: Pulling fs layer +fc53ea013f38: Pulling fs layer +862a57157567: Waiting +c3ec21eb492c: Waiting +fc53ea013f38: Waiting +774043ccc8cc: Verifying Checksum +774043ccc8cc: Download complete +23a5bfa58353: Verifying Checksum +23a5bfa58353: Download complete +862a57157567: Verifying Checksum +862a57157567: Download complete +c3ec21eb492c: Verifying Checksum +c3ec21eb492c: Download complete +fc53ea013f38: Verifying Checksum +fc53ea013f38: Download complete +ab6b448d4be9: Verifying Checksum +ab6b448d4be9: Download complete +774043ccc8cc: Pull complete +ab6b448d4be9: Pull complete +23a5bfa58353: Pull complete +862a57157567: Pull complete +c3ec21eb492c: Pull complete +fc53ea013f38: Pull complete +Digest: sha256:b1407516c4c4f9d8dcd35ea8ab56d177139e8858d37ab1b3a4e6be84ec9be3ae +Status: Downloaded newer image for gitea.dooplex.hu/admin/felhom-controller:0.293.0 +gitea.dooplex.hu/admin/felhom-controller:0.293.0 +[golden] asking the controller which infra images it manages … +[golden] baking infra images (4): traefik:v3.7.13 cloudflare/cloudflared:2026.9.3 gtstef/filebrowser:1.5.6-stable gitea.dooplex.hu/admin/felhom-samba:1.1.0 … +v3.7.13: Pulling from library/traefik +e2de96513ba9: Pulling fs layer +b686a4f73445: Pulling fs layer +78cb21c375ca: Pulling fs layer +acb2f33459b1: Pulling fs layer +acb2f33459b1: Waiting +e2de96513ba9: Verifying Checksum +e2de96513ba9: Download complete +b686a4f73445: Verifying Checksum +b686a4f73445: Download complete +e2de96513ba9: Pull complete +acb2f33459b1: Verifying Checksum +acb2f33459b1: Download complete +78cb21c375ca: Verifying Checksum +78cb21c375ca: Download complete +b686a4f73445: Pull complete +78cb21c375ca: Pull complete +acb2f33459b1: Pull complete +Digest: sha256:24841fe2de7304c149343d877d2923b4c8800a38ba015dea9174c23b20e344a0 +Status: Downloaded newer image for traefik:v3.7.13 +docker.io/library/traefik:v3.7.13 +2026.9.3: Pulling from cloudflare/cloudflared +2cc7ee286bf3: Pulling fs layer +c172f21841df: Pulling fs layer +218cf840d0d9: Pulling fs layer +f6069939f718: Pulling fs layer +d6b1b89eccac: Pulling fs layer +2780920e5dbf: Pulling fs layer +7c12895b777b: Pulling fs layer +3214acf345c0: Pulling fs layer +52630fc75a18: Pulling fs layer +dd64bf2dd177: Pulling fs layer +b839dfae01f6: Pulling fs layer +ebddc55facdc: Pulling fs layer +c4bc6f35ff5e: Pulling fs layer +b96fe2995f90: Pulling fs layer +58c0c263dc73: Pulling fs layer +bd8962e29291: Pulling fs layer +cac2ae0193cb: Pulling fs layer +f0383d5ebc47: Pulling fs layer +f6069939f718: Waiting +d6b1b89eccac: Waiting +2780920e5dbf: Waiting +7c12895b777b: Waiting +3214acf345c0: Waiting +52630fc75a18: Waiting +dd64bf2dd177: Waiting +b839dfae01f6: Waiting +ebddc55facdc: Waiting +c4bc6f35ff5e: Waiting +b96fe2995f90: Waiting +58c0c263dc73: Waiting +bd8962e29291: Waiting +cac2ae0193cb: Waiting +f0383d5ebc47: Waiting +c172f21841df: Verifying Checksum +c172f21841df: Download complete +2cc7ee286bf3: Download complete +f6069939f718: Verifying Checksum +f6069939f718: Download complete +d6b1b89eccac: Verifying Checksum +d6b1b89eccac: Download complete +2cc7ee286bf3: Pull complete +2780920e5dbf: Verifying Checksum +2780920e5dbf: Download complete +c172f21841df: Pull complete +218cf840d0d9: Verifying Checksum +218cf840d0d9: Download complete +218cf840d0d9: Pull complete +7c12895b777b: Download complete +52630fc75a18: Verifying Checksum +52630fc75a18: Download complete +3214acf345c0: Verifying Checksum +3214acf345c0: Download complete +dd64bf2dd177: Verifying Checksum +dd64bf2dd177: Download complete +b839dfae01f6: Verifying Checksum +b839dfae01f6: Download complete +ebddc55facdc: Verifying Checksum +ebddc55facdc: Download complete +c4bc6f35ff5e: Verifying Checksum +c4bc6f35ff5e: Download complete +b96fe2995f90: Verifying Checksum +b96fe2995f90: Download complete +58c0c263dc73: Verifying Checksum +58c0c263dc73: Download complete +f6069939f718: Pull complete +d6b1b89eccac: Pull complete +cac2ae0193cb: Verifying Checksum +cac2ae0193cb: Download complete +f0383d5ebc47: Verifying Checksum +f0383d5ebc47: Download complete +2780920e5dbf: Pull complete +7c12895b777b: Pull complete +bd8962e29291: Verifying Checksum +bd8962e29291: Download complete +3214acf345c0: Pull complete +52630fc75a18: Pull complete +dd64bf2dd177: Pull complete +b839dfae01f6: Pull complete +ebddc55facdc: Pull complete +c4bc6f35ff5e: Pull complete +b96fe2995f90: Pull complete +58c0c263dc73: Pull complete +bd8962e29291: Pull complete +cac2ae0193cb: Pull complete +f0383d5ebc47: Pull complete +Digest: sha256:072c067d25ccbe61d46e18f0d0723255f2bb5304f7317caa95b27031520ff92c +Status: Downloaded newer image for cloudflare/cloudflared:2026.9.3 +docker.io/cloudflare/cloudflared:2026.9.3 +1.5.6-stable: Pulling from gtstef/filebrowser +55afa1ecc21d: Pulling fs layer +8ed8f35f8d4f: Pulling fs layer +989b226a579c: Pulling fs layer +660aeead31d5: Pulling fs layer +4f4fb700ef54: Pulling fs layer +adce24567e4c: Pulling fs layer +f17ea56b313b: Pulling fs layer +6b6f3b3efe88: Pulling fs layer +4ed1ca4f3fce: Pulling fs layer +e6fc9c6a5757: Pulling fs layer +d47782d1182a: Pulling fs layer +6b6f3b3efe88: Waiting +4ed1ca4f3fce: Waiting +e6fc9c6a5757: Waiting +d47782d1182a: Waiting +4f4fb700ef54: Waiting +adce24567e4c: Waiting +f17ea56b313b: Waiting +660aeead31d5: Waiting +55afa1ecc21d: Verifying Checksum +55afa1ecc21d: Download complete +8ed8f35f8d4f: Verifying Checksum +8ed8f35f8d4f: Download complete +55afa1ecc21d: Pull complete +989b226a579c: Verifying Checksum +989b226a579c: Download complete +4f4fb700ef54: Verifying Checksum +4f4fb700ef54: Download complete +660aeead31d5: Verifying Checksum +660aeead31d5: Download complete +6b6f3b3efe88: Verifying Checksum +6b6f3b3efe88: Download complete +adce24567e4c: Verifying Checksum +adce24567e4c: Download complete +f17ea56b313b: Verifying Checksum +f17ea56b313b: Download complete +4ed1ca4f3fce: Verifying Checksum +4ed1ca4f3fce: Download complete +d47782d1182a: Verifying Checksum +d47782d1182a: Download complete +e6fc9c6a5757: Verifying Checksum +e6fc9c6a5757: Download complete +8ed8f35f8d4f: Pull complete +989b226a579c: Pull complete +660aeead31d5: Pull complete +4f4fb700ef54: Pull complete +adce24567e4c: Pull complete +f17ea56b313b: Pull complete +6b6f3b3efe88: Pull complete +4ed1ca4f3fce: Pull complete +e6fc9c6a5757: Pull complete +d47782d1182a: Pull complete +Digest: sha256:7c5d7ac8ffda31294d278063cf9d2e04303b39e6dce1f4c691342240ca7703b8 +Status: Downloaded newer image for gtstef/filebrowser:1.5.6-stable +docker.io/gtstef/filebrowser:1.5.6-stable +1.1.0: Pulling from admin/felhom-samba +897d797d2723: Pulling fs layer +3051591aa250: Pulling fs layer +ce57a3f93416: Pulling fs layer +fb94eeec2fe1: Pulling fs layer +fb94eeec2fe1: Waiting +ce57a3f93416: Verifying Checksum +ce57a3f93416: Download complete +fb94eeec2fe1: Verifying Checksum +fb94eeec2fe1: Download complete +897d797d2723: Verifying Checksum +897d797d2723: Download complete +3051591aa250: Verifying Checksum +3051591aa250: Download complete +897d797d2723: Pull complete +3051591aa250: Pull complete +ce57a3f93416: Pull complete +fb94eeec2fe1: Pull complete +Digest: sha256:1c17c09422bec0366d7cf0e0fcfc1486ba6c90334a0a5d5c851073a9342f8f10 +Status: Downloaded newer image for gitea.dooplex.hu/admin/felhom-samba:1.1.0 +gitea.dooplex.hu/admin/felhom-samba:1.1.0 +[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-controller-bootstrap.service' → '/etc/systemd/system/felhom-controller-bootstrap.service'. +[golden] baking the controller-bootstrap PATH unit (starts the service on bootstrap-mount hot-plug — B1) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-controller-bootstrap.path' → '/etc/systemd/system/felhom-controller-bootstrap.path'. +[golden] baking the first-boot SSH host-key regeneration unit (F3) … +Created symlink '/etc/systemd/system/multi-user.target.wants/felhom-regen-hostkeys.service' → '/etc/systemd/system/felhom-regen-hostkeys.service'. +[golden] identity-clean + minimize … +[golden] stop + archive … +INFO: including mount point rootfs ('/') in backup +INFO: including mount point mp0 ('/var/lib/felhom') in backup +INFO: archive file size: 618MB +INFO: Finished Backup of VM 9100 (00:00:29) +[golden] DONE. golden archive volid: local:backup/vzdump-lxc-9100-2026_10_04-20_14_33.tar.zst (rootfs 32G + ONE data volume 24G @ /var/lib/felhom, all in the archive) +[golden] publishing golden (648135998 bytes, sha256 e7966872abeb38db…) → https://gitea.dooplex.hu/api/packages/admin/generic/felhom-golden/0.293.0/golden.tar.zst +[golden] pre-delete existing: HTTP 404 (404/204 expected) +[golden] upload OK (HTTP 201) +GOLDEN_VERSION=0.293.0 +GOLDEN_SHA256=e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7 +[golden] Record in the hub operator UI (Configs → Day-0 artifacts): golden 0.293.0 / e7966872abeb38db320e7b26bd9baf6527e87270ed2faf3f0e53763bfc4764b7 +[golden] (the build guest 9100 is stopped; destroy it with: pct destroy 9100 --purge) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index f14d50bb..d46f1986 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,25 @@ +## v0.133.0 — test approvals end with the test (R-859); the root-file bundle on the System page, in the install manifest, and its alarm (R-840) (2026-10-04) + +**Agent v0.143.0** reports the bundle; an older agent shows `unknown` in the new column (never a guess). + +- **R-859 — test approvals end with the test (`11` §5.3.1).** Every OS approval made while a TEST override + (`OS_APPROVE_AFTER`, `OS_APPROVE_NIGHTS`, `OS_DOCKER_APPROVE_NIGHTS`) is active is stored with a `test` mark. At every + start WITHOUT an override, each test approval no real approval has superseded is CANCELLED: never served again (no + ring-1 box gets a plan from it), ring-1 boxes are bumped, one operator event `os_release_cancelled` each. What boxes + already installed stays. A one-time backfill marks the AUTOMATIC approvals made earlier than 24 h after their set was first + seen (the 2026-10-04 guest and host sets — approved 1.5 h after first seen). The operator's Docker button approval of + that day stays in force (a person approved it; decided by CC unattended — operator may reverse). The System page shows a test + approval in amber and lists the cancellations of the last 7 days. The same set is approved again by the ruled wait. +- **R-840 — the config bundle.** The Configuration vouch resolves the vouched agent's `felhom-config-bundle.json` sha + from the registry by EXACT name (`gitea.ExactFileSHA256` — never the binary's sha as a fallback); the install manifest + (`/api/v1/artifacts/`) serves it as `bundle`; the installer 1.31.0 installs exactly it. The System page has a + "Root files" column (the box's bundle; amber when behind the vouched one or changed by hand; red from 7 days); the + alarm `os_config_bundle_behind` (operator, warning) fires after `OS_ALARM_BUNDLE_BEHIND_AFTER` (default 7 d; decided by + CC unattended — operator may reverse). A box saying `none` counts as behind; `unknown` never does. +- `GetLatestHostReportJSON` breaks a same-second tie by id (two reports in one second picked an arbitrary one). +- Tests: `testapproval_test.go`, `bundle_alarm_test.go`, `system_bundle_test.go`, `gitea/exact_test.go`. Red-proofs: + `documentation/audits/r840-config-bundle-2026-10-04/partD/d-redproof.txt`, `.../partB/hub-bundle-redproof.txt`. + ## v0.132.0 — the System page, the Docker engine release, the crash events (R-852, `09` decisions 87–89) (2026-10-04) **Needs agent v0.142.0** for the versions, the Docker step and the crash guard; an older agent shows "no versions diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index e3846814..5dbe0981 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -10,6 +10,7 @@ import ( "os/signal" "path/filepath" "strconv" + "strings" "syscall" "time" @@ -423,6 +424,20 @@ func main() { logger.Printf("[ERROR] OS_DOCKER_APPROVE_NIGHTS=%q invalid — keeping 2", v) } } + // `11` §5.3.1 (hub v0.133.0): an approval made under ANY of the three TEST overrides carries the test mark, and a start + // without them cancels every test approval no real one superseded (no ring-1 box installs it from then on). + var overrides []string + for _, k := range []string{"OS_APPROVE_AFTER", "OS_APPROVE_NIGHTS", "OS_DOCKER_APPROVE_NIGHTS"} { + if v := os.Getenv(k); v != "" { + overrides = append(overrides, k+"="+v) + } + } + osSvc.TestOverride = strings.Join(overrides, " ") + if cancelled, cerr := osSvc.CancelTestReleases(); cerr != nil { + logger.Printf("[ERROR] osupdates: cancelling test approvals at start: %v (cancelled so far: %v)", cerr, cancelled) + } else if len(cancelled) > 0 { + logger.Printf("[WARN] osupdates: %d TEST approval(s) cancelled at start: %s", len(cancelled), strings.Join(cancelled, ", ")) + } logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired) logger.Printf("[INFO] osupdates: the Docker engine set is approved only by the operator, after %d healthy ring-0 night(s)", osSvc.DockerNights) apiHandler.SetOSUpdateService(osSvc) @@ -437,6 +452,7 @@ func main() { {"OS_ALARM_REBOOT_AFTER", &osSvc.RebootAfter, 14 * 24 * time.Hour}, {"OS_ALARM_RING0_STALL_AFTER", &osSvc.Ring0StallAfter, 7 * 24 * time.Hour}, {"OS_ALARM_NOT_COVERED_AFTER", &osSvc.NotCoveredAfter, 14 * 24 * time.Hour}, + {"OS_ALARM_BUNDLE_BEHIND_AFTER", &osSvc.BundleBehindAfter, 7 * 24 * time.Hour}, } { *a.dst = a.def if v := os.Getenv(a.env); v != "" { diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index a2765ef6..06867f6f 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -2675,6 +2675,8 @@ func (h *Handler) handleConfigRetrieve(w http.ResponseWriter, r *http.Request, c type artifactManifestResponse struct { Agent artifactEntry `json:"agent"` Golden artifactEntry `json:"golden"` + // Bundle is the vouched agent's config bundle (R-840); absent when the vouched agent carries none. + Bundle *artifactEntry `json:"bundle,omitempty"` } type artifactEntry struct { @@ -2720,6 +2722,9 @@ func (h *Handler) handleArtifactManifest(w http.ResponseWriter, r *http.Request, Agent: artifactEntry{Version: m.AgentVersion, SHA256: m.AgentSHA256}, Golden: artifactEntry{Version: m.GoldenVersion, SHA256: m.GoldenSHA256}, } + if m.BundleSHA256 != "" { + resp.Bundle = &artifactEntry{Version: m.AgentVersion, SHA256: m.BundleSHA256} + } h.logger.Printf("[INFO] Artifact manifest served for customer %s (agent=%s golden=%s)", customerID, m.AgentVersion, m.GoldenVersion) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/hub/internal/gitea/exact_test.go b/hub/internal/gitea/exact_test.go new file mode 100644 index 00000000..0ce3ac65 --- /dev/null +++ b/hub/internal/gitea/exact_test.go @@ -0,0 +1,34 @@ +package gitea + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +// R-840: a version WITHOUT a config bundle must answer "" — never the binary's sha (FileSHA256's fallback would). +func TestExactFileSHA256_NeverFallsBack(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/packages/admin/generic/felhom-agent/0.142.1/files": + w.Write([]byte(`[{"name":"felhom-agent","sha256":"aaaa"}]`)) + case "/api/v1/packages/admin/generic/felhom-agent/0.143.0/files": + w.Write([]byte(`[{"name":"felhom-agent","sha256":"aaaa"},{"name":"felhom-config-bundle.json","sha256":"bbbb"}]`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := New(srv.URL, "admin", "u", "t") + if got, err := c.ExactFileSHA256(context.Background(), "felhom-agent", "0.142.1", "felhom-config-bundle.json"); err != nil || got != "" { + t.Fatalf("no bundle: got %q %v, want \"\"", got, err) + } + if got, err := c.ExactFileSHA256(context.Background(), "felhom-agent", "0.143.0", "felhom-config-bundle.json"); err != nil || got != "bbbb" { + t.Fatalf("bundle: got %q %v", got, err) + } + if _, err := c.ExactFileSHA256(context.Background(), "felhom-agent", "9.9.9", "felhom-config-bundle.json"); err == nil { + t.Fatal("an unreadable listing must be an error, not \"no bundle\"") + } +} diff --git a/hub/internal/gitea/gitea.go b/hub/internal/gitea/gitea.go index 514895c4..9555941c 100644 --- a/hub/internal/gitea/gitea.go +++ b/hub/internal/gitea/gitea.go @@ -107,6 +107,26 @@ func (c *Client) FileSHA256(ctx context.Context, pkgName, version, preferredFile return files[0].SHA256, nil } +// ExactFileSHA256 returns the sha256 of exactly `file` in the package version, or ("", nil) when the version has no +// such file. Unlike FileSHA256 it NEVER falls back to another file: the config bundle (R-840) must not inherit the +// binary's sha on a version that carries no bundle. +func (c *Client) ExactFileSHA256(ctx context.Context, pkgName, version, file string) (string, error) { + u := fmt.Sprintf("%s/api/v1/packages/%s/generic/%s/%s/files", c.baseURL, c.owner, pkgName, version) + var files []pkgFile + if err := c.getJSON(ctx, u, &files); err != nil { + return "", err + } + for _, f := range files { + if f.Name == file { + if f.SHA256 == "" { + return "", fmt.Errorf("no sha256 for %s/%s file %q", pkgName, version, file) + } + return f.SHA256, nil + } + } + return "", nil +} + func (c *Client) getJSON(ctx context.Context, url string, out interface{}) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 68e72f35..cdb49fa9 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -682,10 +682,14 @@ var operatorOnlyEvents = map[string]bool{ "offsite_window_large_grant": true, // OS updates (hub v0.130.0, `11` §8 step 2): run failures, rings, switches and approvals are operator facts. // os_update_applied is deliberately NOT here — it is the household's one line (info: recorded, never mailed). - "os_update_failed": true, - "os_update_health_failed": true, - "os_release_approved": true, - "os_release_approved_now": true, + "os_update_failed": true, + "os_update_health_failed": true, + "os_release_approved": true, + "os_release_approved_now": true, + // hub v0.133.0 (`11` §5.3.1): a TEST approval cancelled at a start without the override. + "os_release_cancelled": true, + // R-840 (hub v0.133.0): a box's root-owned config bundle behind the vouched one for 7 days. + "os_config_bundle_behind": true, "os_update_settings_changed": true, // R-841 (hub v0.131.0): the tunnel alarm — a box fact the household can do nothing about from inside. "tunnel_down": true, diff --git a/hub/internal/osupdates/bundle_alarm_test.go b/hub/internal/osupdates/bundle_alarm_test.go new file mode 100644 index 00000000..e4b24fcd --- /dev/null +++ b/hub/internal/osupdates/bundle_alarm_test.go @@ -0,0 +1,90 @@ +package osupdates + +import ( + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +func bundleReport(t *testing.T, f *fix, host, version, sha string) { + t.Helper() + body := `{"system":{"pve_version":"pve-manager/9.2.2/x","config_bundle":{"version":"` + version + `","bundle_sha256":"` + sha + `"}}}` + h, err := f.s.Store.GetHost(host) + if err != nil || h == nil { + t.Fatalf("no host %s", host) + } + if err := f.s.Store.SaveHostReport(host, h.CustomerID, []byte(body), store.HostReportDenorm{AgentVersion: "0.143.0"}); err != nil { + t.Fatal(err) + } +} + +func vouch(t *testing.T, f *fix, sha string) { + t.Helper() + if err := f.s.Store.SetArtifactManifest(store.ArtifactManifest{AgentVersion: "0.143.0", AgentSHA256: "x", BundleSHA256: sha}); err != nil { + t.Fatal(err) + } +} + +func count(sent []string, typ string) int { + n := 0 + for _, s := range sent { + if s == typ { + n++ + } + } + return n +} + +// R-840: a box behind the vouched bundle is told to the operator after 7 days — not before, once, and the clock +// restarts when the box catches up. "none" (no bundle ever) counts; "unknown" never does. +func TestBundleAlarm_AfterSevenDaysBehind(t *testing.T) { + f := newFix(t) + vouch(t, f, "new") + bundleReport(t, f, "cust1", "none", "") + sent, _ := f.s.Alarms() + if count(sent, EventBundleBehind) != 0 { + t.Fatal("alarm on the first sight") + } + f.now = f.now.Add(6 * 24 * time.Hour) + if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 0 { + t.Fatal("alarm before 7 days") + } + f.now = f.now.Add(25 * time.Hour) + if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 1 { + t.Fatalf("no alarm after 7 days: %v", sent) + } + if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 0 { + t.Fatal("the alarm must not repeat at once") + } + bundleReport(t, f, "cust1", "0.143.0", "new") + f.s.Alarms() + if !f.s.Store.BundleBehindSince("cust1").IsZero() { + t.Fatal("caught up: the clock must clear") + } +} + +func TestBundleAlarm_UnknownAndNoVouchedBundleSayNothing(t *testing.T) { + f := newFix(t) + vouch(t, f, "new") + bundleReport(t, f, "cust1", "unknown", "") + f.s.Alarms() + f.now = f.now.Add(30 * 24 * time.Hour) + if sent, _ := f.s.Alarms(); count(sent, EventBundleBehind) != 0 || !f.s.Store.BundleBehindSince("cust1").IsZero() { + t.Fatalf("unknown is not a fact: %v", sent) + } + // control: the same box saying "none" IS behind (proves the report above was read at all) + bundleReport(t, f, "cust1", "none", "") + f.s.Alarms() + if f.s.Store.BundleBehindSince("cust1").IsZero() { + t.Fatal("control: a box saying none must start the clock") + } + g := newFix(t) + vouch(t, g, "") + bundleReport(t, g, "cust1", "0.143.0", "some-bundle") + g.s.Alarms() + g.now = g.now.Add(30 * 24 * time.Hour) + if sent, _ := g.s.Alarms(); count(sent, EventBundleBehind) != 0 { + t.Fatalf("nothing vouched, nothing behind: %v", sent) + } +} diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go index 069b1205..0fc4c365 100644 --- a/hub/internal/osupdates/service.go +++ b/hub/internal/osupdates/service.go @@ -29,6 +29,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" + "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // Layers. @@ -68,6 +69,10 @@ const ( EventRebootNeeded = "os_reboot_needed" // warning, operator: reboot needed for RebootAfter EventRing0Stalled = "os_ring0_stalled" // error, operator: ring 0 approved nothing for Ring0StallAfter EventNotCovered = "os_not_covered" // warning, operator: fast-lane packages no release covers + // EventCancelled: a TEST approval was cancelled because the hub started without the TEST override (`11` §5.3.1). + EventCancelled = "os_release_cancelled" // warning, operator + // EventBundleBehind: a box's root-owned config bundle has differed from the vouched one for BundleBehindAfter (R-840). + EventBundleBehind = "os_config_bundle_behind" // warning, operator ) // Package is one name=version with its origin ("Debian" | "Debian-Security"). @@ -165,9 +170,15 @@ type Service struct { RebootAfter time.Duration // 14 d Ring0StallAfter time.Duration // 7 d NotCoveredAfter time.Duration // 14 d - Logger *log.Logger - Now func() time.Time - Bump func(hostID string) + // BundleBehindAfter: a box's config bundle differs from the vouched one this long → an operator alarm (R-840; + // decided by CC unattended — operator may reverse). Zero = 7 d. + BundleBehindAfter time.Duration + Logger *log.Logger + Now func() time.Time + Bump func(hostID string) + // TestOverride names the TEST overrides active at start ("" = none, the ruled waits). Every approval made while it + // is set carries the `test` mark; CancelTestReleases cancels them at a start without it (`11` §5.3.1). + TestOverride string } func (s *Service) now() time.Time { @@ -427,6 +438,8 @@ type ReleaseInfo struct { ApprovedAt time.Time ApprovedBy string Packages int + Test bool // a TEST approval still in force: amber on the System page + Cancelled string // set on a cancelled one (the page lists the last 7 days') } // Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent. @@ -439,7 +452,20 @@ func (s *Service) Releases() []ReleaseInfo { } var list []Package _ = json.Unmarshal([]byte(rel.PackagesJSON), &list) - out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)}) + out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list), Test: rel.Test}) + } + return out +} + +// CancelledReleases lists the approvals cancelled in the last 7 days (the System page says what stopped being served). +func (s *Service) CancelledReleases() []ReleaseInfo { + rels, _ := s.Store.CancelledOSReleasesSince(s.now().Add(-7 * 24 * time.Hour)) + var out []ReleaseInfo + for _, r := range rels { + var list []Package + _ = json.Unmarshal([]byte(r.PackagesJSON), &list) + out = append(out, ReleaseInfo{Layer: r.Layer, ID: r.ID, ApprovedAt: r.ApprovedAt, ApprovedBy: r.ApprovedBy, Packages: len(list), + Test: r.Test, Cancelled: r.CancelledAt}) } return out } @@ -455,6 +481,9 @@ func (s *Service) Candidates() []Status { return append(out, d) } +// BundleThreshold is the config-bundle alarm's wait (the System page turns the cell red at it). +func (s *Service) BundleThreshold() time.Duration { return dflt(s.BundleBehindAfter, 7*24*time.Hour) } + // Thresholds are the alarm numbers the System page colours by (the same values the alarms use). func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) { return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour) @@ -577,12 +606,18 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error { at := s.now().UTC().Truncate(time.Second) id := "os-" + layer + "-" + at.Format("20060102-150405") pj, _ := json.Marshal(list) - if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil { + test := s.TestOverride != "" + if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, + PackagesJSON: string(pj), Test: test}); err != nil { return err } - s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp) - s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)), - map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp}) + mark := "" + if test { + mark = " — TEST approval (" + s.TestOverride + "); cancelled when the hub starts without the override" + } + s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)%s", id, layer, by, len(list), fp, mark) + s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages)%s.", id, layer, by, len(list), mark), + map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp, "test": test}) if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state hosts, _ := s.Store.ListHosts() for _, h := range hosts { @@ -594,6 +629,45 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error { return nil } +// CancelTestReleases runs at every hub start. Without a TEST override it cancels every test-marked approval that no +// real (non-test) approval has superseded: no ring-1 box installs it from then on; what boxes already installed stays. +// Each cancellation is an operator event; ring-1 boxes are bumped so their next plan has no cancelled release. With the +// override still active it does nothing (the test is still running). Returns the cancelled ids. +func (s *Service) CancelTestReleases() ([]string, error) { + if s.TestOverride != "" { + s.logf("[INFO] osupdates: TEST override active (%s) — test approvals stay in force", s.TestOverride) + return nil, nil + } + var ids []string + for _, layer := range AllLayers { + rels, err := s.Store.UnsupersededTestReleases(layer) + if err != nil { + return ids, err + } + for _, r := range rels { + reason := "approved under a TEST override; the hub started without it" + if err := s.Store.CancelOSRelease(r.ID, reason, s.now()); err != nil { + return ids, err + } + ids = append(ids, r.ID) + s.logf("[WARN] osupdates: TEST approval %s (%s, approved %s by %s) CANCELLED — no ring-1 box installs it from now on", + r.ID, layer, r.ApprovedAt.UTC().Format(time.RFC3339), r.ApprovedBy) + s.event("", EventCancelled, "warning", fmt.Sprintf("OS release %s (%s) was a TEST approval and is cancelled: "+ + "no further box installs it. Boxes that already installed it keep it.", r.ID, layer), + map[string]any{"release_id": r.ID, "layer": layer, "approved_at": r.ApprovedAt.UTC().Format(time.RFC3339), "approved_by": r.ApprovedBy}) + } + } + if len(ids) > 0 && s.Bump != nil { + hosts, _ := s.Store.ListHosts() + for _, h := range hosts { + if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { + s.Bump(h.HostID) + } + } + } + return ids, nil +} + func (s *Service) releaseBlock(layer string) *ReleaseBlock { rel, err := s.Store.LatestOSRelease(layer) if err != nil || rel == nil { @@ -860,6 +934,38 @@ func (s *Service) Alarms() ([]string, error) { } } } + // 5. R-840: the box's ROOT-OWNED config bundle (sudoers, wrappers, units) differs from the vouched agent's for + // BundleBehindAfter. "none" (no bundle ever reached the box) counts as behind; "unknown" (the box could not say) + // counts as nothing — never a guess. Only when the vouched agent carries a bundle at all. + man := s.Store.GetArtifactManifest() + for _, h := range hosts { + if man.BundleSHA256 == "" { + break + } + rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID) + sys := sysfacts.Parse(rj) + if !sys.Present || sys.Bundle.Version == sysfacts.Unknown { + continue + } + behind := sys.Bundle.BundleSHA256 != man.BundleSHA256 + since := s.Store.BundleBehindSince(h.HostID) + switch { + case !behind && !since.IsZero(): + _ = s.Store.SetBundleBehindSince(h.HostID, time.Time{}) + since = time.Time{} + case behind && since.IsZero(): + since = now + _ = s.Store.SetBundleBehindSince(h.HostID, since) + } + after := dflt(s.BundleBehindAfter, 7*24*time.Hour) + if s.raise("bundle:"+h.HostID, behind && now.Sub(since) >= after, h.CustomerID, EventBundleBehind, "warning", + fmt.Sprintf("Root files: %s still runs config bundle %s; the vouched agent %s carries a newer one (since %s). "+ + "Send it with a signed agent_config_update (`11` §5.4.2).", h.HostID, sys.Bundle.Version, man.AgentVersion, + since.UTC().Format("2006-01-02")), + map[string]any{"host_id": h.HostID, "box_bundle": sys.Bundle.Version, "vouched_agent": man.AgentVersion, "since": since}) { + sent = append(sent, EventBundleBehind) + } + } // 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped // getting fixes. ring0, _ := s.ring0Hosts() diff --git a/hub/internal/osupdates/testapproval_test.go b/hub/internal/osupdates/testapproval_test.go new file mode 100644 index 00000000..94a8b217 --- /dev/null +++ b/hub/internal/osupdates/testapproval_test.go @@ -0,0 +1,173 @@ +package osupdates + +import ( + "database/sql" + "log" + "os" + "path/filepath" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +// `11` §5.3.1 (hub v0.133.0): test approvals end with the test. + +// approveUnderTest makes ring 0 run a set and approves it at once under a TEST override (OS_APPROVE_AFTER=0 shape). +func approveUnderTest(t *testing.T, f *fix, override string, set ...Package) string { + t.Helper() + f.s.TestOverride = override + f.s.ApproveAfter, f.s.NightsRequired = 0, 0 + f.report(t, "hp", "debug", true, set...) + f.report(t, "n100", "debug", true, set...) + if _, err := f.s.Evaluate(); err != nil { + t.Fatal(err) + } + rel, _ := f.s.Store.LatestOSRelease(LayerGuest) + if rel == nil { + t.Fatal("not approved") + } + return rel.ID +} + +// An approval made under the override carries the mark; one made without it does not. +func TestTestApproval_IsMarked(t *testing.T) { + f := newFix(t) + approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4")) + rel, _ := f.s.Store.LatestOSRelease(LayerGuest) + if !rel.Test { + t.Fatalf("an approval under a TEST override must be marked: %+v", rel) + } + if info := f.s.Releases(); len(info) != 1 || !info[0].Test { + t.Fatalf("the System page must see the mark: %+v", info) + } + f.s.TestOverride = "" + f.now = f.now.Add(time.Hour) + f.report(t, "hp", "debug", true, pk("libc6", "u5")) + f.report(t, "n100", "debug", true, pk("libc6", "u5")) + f.s.Evaluate() + rel, _ = f.s.Store.LatestOSRelease(LayerGuest) + if rel.Test { + t.Fatalf("an approval without the override must not be marked: %+v", rel) + } +} + +// The consequence: after a restart without the override, a ring-1 box gets NO plan from the test approval; the +// cancellation is an operator event and ring-1 boxes are bumped. +func TestTestApproval_CancelledAtAStartWithoutTheOverride(t *testing.T) { + f := newFix(t) + id := approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4")) + if b := f.s.DesiredBlock("cust1"); b.Release == nil || b.Release.ID != id { + t.Fatalf("before the restart the ring-1 box is served the release: %+v", b) + } + f.events, f.bumps = nil, nil + f.s.TestOverride = "" // the hub restarts without the override + ids, err := f.s.CancelTestReleases() + if err != nil || len(ids) != 1 || ids[0] != id { + t.Fatalf("cancelled %v, %v", ids, err) + } + if b := f.s.DesiredBlock("cust1"); b.Release != nil { + t.Fatalf("a ring-1 box must get no plan from a cancelled test approval: %+v", b.Release) + } + if len(f.events) != 1 || f.events[0] != EventCancelled { + t.Fatalf("events = %v", f.events) + } + if len(f.bumps) != 1 || f.bumps[0] != "cust1" { + t.Fatalf("ring-1 boxes must be bumped: %v", f.bumps) + } + if c := f.s.CancelledReleases(); len(c) != 1 || c[0].ID != id { + t.Fatalf("the page must list the cancellation: %+v", c) + } + // once is enough: a second start cancels nothing more + if again, _ := f.s.CancelTestReleases(); len(again) != 0 { + t.Fatalf("second start cancelled %v", again) + } +} + +func TestTestApproval_StaysWhileTheOverrideIsStillOn(t *testing.T) { + f := newFix(t) + approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4")) + if ids, _ := f.s.CancelTestReleases(); len(ids) != 0 { + t.Fatalf("the test is still running; nothing to cancel: %v", ids) + } + if b := f.s.DesiredBlock("cust1"); b.Release == nil { + t.Fatal("still served while the override is on") + } +} + +// A test approval that a REAL approval has superseded is history, not cancelled; the real one stays served. +func TestTestApproval_SupersededIsLeftAlone(t *testing.T) { + f := newFix(t) + old := approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4")) + f.s.TestOverride = "" + f.now = f.now.Add(time.Hour) + f.report(t, "hp", "debug", true, pk("libc6", "u5")) + f.report(t, "n100", "debug", true, pk("libc6", "u5")) + f.s.Evaluate() + real, _ := f.s.Store.LatestOSRelease(LayerGuest) + if real.ID == old || real.Test { + t.Fatalf("setup: %+v", real) + } + if ids, _ := f.s.CancelTestReleases(); len(ids) != 0 { + t.Fatalf("a superseded test approval must not be cancelled: %v", ids) + } + if b := f.s.DesiredBlock("cust1"); b.Release == nil || b.Release.ID != real.ID { + t.Fatalf("the real release stays served: %+v", b.Release) + } +} + +// After a cancellation the SAME set is approved again by the ruled wait (a real release) — the cancel is not a ban. +func TestTestApproval_TheSetIsApprovedAgainByTheRuledWait(t *testing.T) { + f := newFix(t) + set := []Package{pk("libc6", "u4")} + approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", set...) + f.s.TestOverride = "" + f.s.ApproveAfter, f.s.NightsRequired = 24*time.Hour, 1 + f.s.CancelTestReleases() + f.now = f.now.Add(25 * time.Hour) + f.report(t, "hp", "night", true, set...) + f.report(t, "n100", "night", true, set...) + f.s.Evaluate() + rel, _ := f.s.Store.LatestOSRelease(LayerGuest) + if rel == nil || rel.Test { + t.Fatalf("the ruled wait must approve the set again, unmarked: %+v", rel) + } +} + +// The one-time backfill: on a database from before the mark, an approval earlier than 24 h after its set was first +// seen (the 2026-10-04 shape: 1.5 h) is marked test; one after the ruled wait is not. +func TestTestApproval_BackfillMarksTheEarlyApprovals(t *testing.T) { + path := filepath.Join(t.TempDir(), "hub.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `CREATE TABLE os_candidates (fingerprint TEXT PRIMARY KEY, first_seen DATETIME NOT NULL, packages_json TEXT NOT NULL)`, + `CREATE TABLE os_releases (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, approved_at DATETIME NOT NULL, approved_by TEXT NOT NULL, packages_json TEXT NOT NULL, layer TEXT NOT NULL DEFAULT 'guest')`, + `INSERT INTO os_candidates VALUES ('fpA', '2026-10-04 11:07:00', '[]'), ('fpB', '2026-10-02 08:00:00', '[]')`, + `INSERT INTO os_candidates VALUES ('fpD', '2026-10-04 14:28:00', '[]')`, + `INSERT INTO os_releases VALUES ('os-guest-early', 'fpA', '2026-10-04 12:39:33', 'auto', '[]', 'guest'), + ('os-guest-ruled', 'fpB', '2026-10-03 09:00:00', 'auto', '[]', 'guest'), + ('os-docker-button', 'fpD', '2026-10-04 14:28:42', 'operator', '[]', 'docker')`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + db.Close() + st, err := store.New(path, log.New(os.Stderr, "", 0)) + if err != nil { + t.Fatal(err) + } + defer st.Close() + rels, _ := st.UnsupersededTestReleases(LayerGuest) + if len(rels) != 1 || rels[0].ID != "os-guest-early" { + t.Fatalf("backfill: unsuperseded test releases = %+v", rels) + } + // the operator's own button approval is not backfilled (a person approved it) + if d, _ := st.UnsupersededTestReleases(LayerDocker); len(d) != 0 { + t.Fatalf("backfill marked the operator's Docker approval: %+v", d) + } +} diff --git a/hub/internal/store/os_updates.go b/hub/internal/store/os_updates.go index beb7cf7c..e76779e2 100644 --- a/hub/internal/store/os_updates.go +++ b/hub/internal/store/os_updates.go @@ -53,9 +53,42 @@ func (s *Store) migrateOSUpdates() error { // Host fast lane (hub v0.131.0): every report and release belongs to a LAYER; old rows are the guest's. s.db.Exec(`ALTER TABLE os_reports ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`) s.db.Exec(`ALTER TABLE os_releases ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`) + // Test approvals end with the test (hub v0.133.0, `11` §5.3.1): an approval made while a TEST override is active + // carries `test`; a start without the override CANCELS every test approval no real one has superseded. + if !s.hasColumn("os_releases", "test") { + if _, err := s.db.Exec(`ALTER TABLE os_releases ADD COLUMN test INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + // One-time backfill, from the data: an AUTOMATIC approval earlier than 24 h after its set was first seen + // cannot have passed the ruled wait (24 h + one night) — it was a TEST approval (2026-10-04: the guest and host + // sets, approved 1.5 h after first seen under OS_APPROVE_*). The operator's own Docker button approval of that day + // is NOT backfilled: a person approved it; the override only shortened its precondition (decided by CC + // unattended — operator may reverse). From hub v0.133.0 on, every approval under an override is marked. + s.db.Exec(`UPDATE os_releases SET test = 1 WHERE approved_by = 'auto' AND EXISTS (SELECT 1 FROM os_candidates c + WHERE c.fingerprint = os_releases.fingerprint + AND (julianday(os_releases.approved_at) - julianday(c.first_seen)) * 24 < 24)`) + } + s.db.Exec(`ALTER TABLE os_releases ADD COLUMN cancelled_at TEXT NOT NULL DEFAULT ''`) + s.db.Exec(`ALTER TABLE os_releases ADD COLUMN cancel_reason TEXT NOT NULL DEFAULT ''`) return nil } +// hasColumn reports whether table has the column (PRAGMA table_info). +func (s *Store) hasColumn(table, col string) bool { + rows, err := s.db.Query(`SELECT name FROM pragma_table_info(?)`, table) + if err != nil { + return false + } + defer rows.Close() + for rows.Next() { + var n string + if rows.Scan(&n) == nil && n == col { + return true + } + } + return false +} + // OSHostSettings is one box's ring and switch. type OSHostSettings struct { HostID string @@ -191,6 +224,22 @@ type OSRelease struct { ApprovedAt time.Time ApprovedBy string PackagesJSON string + Test bool // approved while a TEST override was active (`11` §5.3.1) + CancelledAt string // "" = in force; a cancelled release is never served (LatestOSRelease skips it) + CancelReason string +} + +const osReleaseCols = `id, layer, fingerprint, approved_at, approved_by, packages_json, test, cancelled_at, cancel_reason` + +func scanOSRelease(sc interface{ Scan(...any) error }) (*OSRelease, error) { + var r OSRelease + var at string + var test int + if err := sc.Scan(&r.ID, &r.Layer, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON, &test, &r.CancelledAt, &r.CancelReason); err != nil { + return nil, err + } + r.ApprovedAt, r.Test = parseSQLiteTime(at), test == 1 + return &r, nil } // SaveOSRelease stores an approved release. @@ -199,25 +248,70 @@ func (s *Store) SaveOSRelease(r OSRelease) error { if layer == "" { layer = "guest" } - _, err := s.db.Exec(`INSERT INTO os_releases (id, layer, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?, ?)`, - r.ID, layer, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON) + test := 0 + if r.Test { + test = 1 + } + _, err := s.db.Exec(`INSERT INTO os_releases (id, layer, fingerprint, approved_at, approved_by, packages_json, test) VALUES (?, ?, ?, ?, ?, ?, ?)`, + r.ID, layer, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON, test) return err } -// LatestOSRelease returns the newest approved release of a layer, or nil. +// LatestOSRelease returns the newest approved release of a layer that is IN FORCE (not cancelled), or nil. func (s *Store) LatestOSRelease(layer string) (*OSRelease, error) { - var r OSRelease - var at string - err := s.db.QueryRow(`SELECT id, layer, fingerprint, approved_at, approved_by, packages_json FROM os_releases WHERE layer = ? ORDER BY approved_at DESC, id DESC LIMIT 1`, layer). - Scan(&r.ID, &r.Layer, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON) + r, err := scanOSRelease(s.db.QueryRow(`SELECT `+osReleaseCols+` FROM os_releases WHERE layer = ? AND cancelled_at = '' + ORDER BY approved_at DESC, id DESC LIMIT 1`, layer)) if err == sql.ErrNoRows { return nil, nil } + return r, err +} + +// UnsupersededTestReleases returns a layer's test releases still in force and newer than its newest REAL (non-test, +// in-force) release — the ones a start without the TEST override must cancel. Newest first. +func (s *Store) UnsupersededTestReleases(layer string) ([]OSRelease, error) { + rows, err := s.db.Query(`SELECT `+osReleaseCols+` FROM os_releases r WHERE r.layer = ? AND r.test = 1 AND r.cancelled_at = '' + AND r.approved_at >= COALESCE((SELECT MAX(approved_at) FROM os_releases WHERE layer = r.layer AND test = 0 AND cancelled_at = ''), '') + ORDER BY r.approved_at DESC, r.id DESC`, layer) if err != nil { return nil, err } - r.ApprovedAt = parseSQLiteTime(at) - return &r, nil + defer rows.Close() + var out []OSRelease + for rows.Next() { + r, err := scanOSRelease(rows) + if err != nil { + return nil, err + } + out = append(out, *r) + } + return out, rows.Err() +} + +// CancelOSRelease marks one release cancelled (it is never served again; its row stays as the record). +func (s *Store) CancelOSRelease(id, reason string, at time.Time) error { + _, err := s.db.Exec(`UPDATE os_releases SET cancelled_at = ?, cancel_reason = ? WHERE id = ? AND cancelled_at = ''`, + at.UTC().Format("2006-01-02 15:04:05"), reason, id) + return err +} + +// CancelledOSReleasesSince lists releases cancelled at or after t (the System page shows them), newest first. +func (s *Store) CancelledOSReleasesSince(t time.Time) ([]OSRelease, error) { + rows, err := s.db.Query(`SELECT `+osReleaseCols+` FROM os_releases WHERE cancelled_at != '' AND cancelled_at >= ? + ORDER BY cancelled_at DESC, id DESC`, t.UTC().Format("2006-01-02 15:04:05")) + if err != nil { + return nil, err + } + defer rows.Close() + var out []OSRelease + for rows.Next() { + r, err := scanOSRelease(rows) + if err != nil { + return nil, err + } + out = append(out, *r) + } + return out, rows.Err() } // BackdateOSCandidateForTest moves a candidate's first_seen into the past. TEST-ONLY. diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index f56bf43b..b32c7b37 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -2118,6 +2118,10 @@ type ArtifactManifest struct { // Recording the hash here does not fix the delivery channel (that is R-50b(b)/(c)) — it makes // DRIFT VISIBLE, which is the cheap honest first step. WrapperSHA256 string `json:"wrapper_sha256"` + // BundleSHA256 is the vouched agent version's CONFIG BUNDLE (R-840): every root-owned file the installer writes, + // published beside the binary as felhom-agent//felhom-config-bundle.json. "" = the vouched agent + // carries none (older than v0.143.0). The installer installs exactly this; boxes behind it raise an alarm. + BundleSHA256 string `json:"bundle_sha256"` } // hub_settings keys for the artifact manifest (BUNDLE slice). Stored as discrete key/value rows in @@ -2130,6 +2134,7 @@ const ( settingArtifactGoldenSHA256 = "artifact_golden_sha256" settingArtifactMinAgent = "artifact_min_agent" settingArtifactWrapperSHA256 = "artifact_wrapper_sha256" // R-50b(a): the vouched felhom-pbs-apply hash + settingArtifactBundleSHA256 = "artifact_bundle_sha256" // R-840: the vouched agent's config bundle ) // settingOperatorPasswordHash is the hub_settings key for the operator login password bcrypt hash, @@ -2178,6 +2183,7 @@ func (s *Store) GetArtifactManifest() ArtifactManifest { GoldenSHA256: s.getSetting(settingArtifactGoldenSHA256), MinAgent: s.getSetting(settingArtifactMinAgent), WrapperSHA256: s.getSetting(settingArtifactWrapperSHA256), + BundleSHA256: s.getSetting(settingArtifactBundleSHA256), } } @@ -2199,7 +2205,30 @@ func (s *Store) SetArtifactManifest(m ArtifactManifest) error { if err := s.setSetting(settingArtifactMinAgent, m.MinAgent); err != nil { return err } - return s.setSetting(settingArtifactWrapperSHA256, m.WrapperSHA256) + if err := s.setSetting(settingArtifactWrapperSHA256, m.WrapperSHA256); err != nil { + return err + } + return s.setSetting(settingArtifactBundleSHA256, m.BundleSHA256) +} + +// BundleBehindSince returns since when a box's config bundle has differed from the vouched one (zero = it matches, or +// was never seen behind). R-840's 7-day alarm counts from here. +func (s *Store) BundleBehindSince(hostID string) time.Time { + v := s.getSetting("bundle_behind_since:" + hostID) + if v == "" { + return time.Time{} + } + t, _ := time.Parse(time.RFC3339, v) + return t +} + +// SetBundleBehindSince records (or, with a zero time, clears) the first moment a box was seen behind. +func (s *Store) SetBundleBehindSince(hostID string, t time.Time) error { + v := "" + if !t.IsZero() { + v = t.UTC().Format(time.RFC3339) + } + return s.setSetting("bundle_behind_since:"+hostID, v) } // EffectiveMinControllerVersion resolves the floor that actually applies to a customer: the @@ -3487,7 +3516,7 @@ func (s *Store) SaveHostReport(hostID, customerID string, reportJSON []byte, d H func (s *Store) GetLatestHostReportJSON(customerID string) (string, error) { var j string err := s.db.QueryRow( - `SELECT report_json FROM host_reports WHERE customer_id = ? ORDER BY received_at DESC LIMIT 1`, + `SELECT report_json FROM host_reports WHERE customer_id = ? ORDER BY received_at DESC, id DESC LIMIT 1`, customerID, ).Scan(&j) if err == sql.ErrNoRows { diff --git a/hub/internal/sysfacts/sysfacts.go b/hub/internal/sysfacts/sysfacts.go index 4b1577d4..e17bd383 100644 --- a/hub/internal/sysfacts/sysfacts.go +++ b/hub/internal/sysfacts/sysfacts.go @@ -50,6 +50,15 @@ type Guest struct { UnknownReason string `json:"unknown_reason"` } +// ConfigBundle is the box's root-owned config bundle (R-840, agent v0.143.0): the agent's own read of the record +// (version "none" = no bundle ever reached the box), with the drift the wrapper's facts add (files changed by hand). +type ConfigBundle struct { + Version string `json:"version"` // agent version of the bundle | none | unknown + BundleSHA256 string `json:"bundle_sha256"` + InstalledAt string `json:"installed_at"` + Drift []string `json:"drift"` +} + // System is the whole stanza. Present is false for a report from an agent older than v0.142.0. type System struct { Present bool @@ -60,6 +69,7 @@ type System struct { ReadAt string Host Host Guest Guest + Bundle ConfigBundle } type wire struct { @@ -70,6 +80,7 @@ type wire struct { Facts json.RawMessage `json:"facts"` FactsError string `json:"facts_error"` ReadAt string `json:"read_at"` + ConfigBundle *ConfigBundle `json:"config_bundle"` } `json:"system"` } @@ -93,11 +104,25 @@ func Parse(reportJSON string) System { out.PVEVersion, out.KernelVersion = orUnknown(w.System.PVEVersion), orUnknown(w.System.KernelVersion) out.VMID, out.FactsError, out.ReadAt = w.System.VMID, w.System.FactsError, w.System.ReadAt var f struct { - Host Host `json:"host"` + Host struct { + Host + ConfigBundle *ConfigBundle `json:"config_bundle"` + } `json:"host"` Guest Guest `json:"guest"` } + out.Bundle = ConfigBundle{Version: Unknown} + if w.System.ConfigBundle != nil && w.System.ConfigBundle.Version != "" { + out.Bundle = *w.System.ConfigBundle + } if len(w.System.Facts) > 0 && json.Unmarshal(w.System.Facts, &f) == nil { - out.Host, out.Guest = f.Host, f.Guest + out.Host, out.Guest = f.Host.Host, f.Guest + // The wrapper's view adds the drift; its record is the same file the agent read. + if fb := f.Host.ConfigBundle; fb != nil && fb.Version != "" && fb.Version != Unknown { + if out.Bundle.Version == Unknown { + out.Bundle = *fb + } + out.Bundle.Drift = fb.Drift + } } out.Host.Debian, out.Host.KernelRunning = orUnknown(out.Host.Debian), orUnknown(out.Host.KernelRunning) out.Host.KernelNextBoot = orUnknown(out.Host.KernelNextBoot) diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index 2ec56c56..a33a1f6e 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -1320,6 +1320,18 @@ func (s *Server) handleSetArtifacts(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/configuration?flash=artifact_sha_invalid", http.StatusSeeOther) return } + // R-840: the vouched agent's CONFIG BUNDLE is resolved from the registry by exact name, like the binary. A version + // without one (older than v0.143.0) vouches none — the installer then falls back to its per-file fetches. + bundleSHA := "" + if agentVer != "" && s.gitea != nil { + b, berr := s.gitea.ExactFileSHA256(r.Context(), pkgAgent, agentVer, fileBundle) + if berr != nil { + s.logger.Printf("[WARN] artifact vouch REFUSED: could not read agent %s's config bundle sha: %v", agentVer, berr) + http.Redirect(w, r, "/configuration?flash=artifact_unverifiable", http.StatusSeeOther) + return + } + bundleSHA = b + } // R-50b(a): the PBS-DR wrapper hash is operator-typed, not resolved from the package registry — // unlike the agent binary and the golden, this artifact is not published there at all. It is // installed from raw/branch/main, which is exactly the drift this field makes visible. @@ -1366,12 +1378,13 @@ func (s *Server) handleSetArtifacts(w http.ResponseWriter, r *http.Request) { GoldenSHA256: goldenSHA, MinAgent: minAgent, WrapperSHA256: wrapperSHA, + BundleSHA256: bundleSHA, }); err != nil { s.logger.Printf("[ERROR] Failed to set artifact manifest: %v", err) http.Error(w, "Internal error", http.StatusInternalServerError) return } - s.logger.Printf("[INFO] Artifact manifest set: agent=%s golden=%s min_agent=%q wrapper_sha=%t", agentVer, goldenVer, minAgent, wrapperSHA != "") + s.logger.Printf("[INFO] Artifact manifest set: agent=%s golden=%s min_agent=%q wrapper_sha=%t bundle_sha=%q", agentVer, goldenVer, minAgent, wrapperSHA != "", bundleSHA) // Agent-plane immediate-sync (Direction-2a, v0.59.0): a MinAgent-floor / vouched-agent change is // a fleet-wide agent-plane intent shift. Fire-and-forget nudge every box so it re-reports at // once (the self-update train's signed op / floor re-evaluation lands in seconds, not ≤15 min). diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index 6288788b..4344c7c0 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -33,6 +33,7 @@ import ( const ( pkgAgent = "felhom-agent" fileAgent = "felhom-agent" + fileBundle = "felhom-config-bundle.json" // R-840: the agent's config bundle, beside the binary pkgGolden = "felhom-golden" fileGolden = "golden.tar.zst" ) diff --git a/hub/internal/web/system.go b/hub/internal/web/system.go index ad96de08..49a8214a 100644 --- a/hub/internal/web/system.go +++ b/hub/internal/web/system.go @@ -31,9 +31,10 @@ type systemRow struct { FactsNote string // host PVE, KernelRunning, KernelNextBoot, HostDebian cell - HostRelease, HostPending, HostNotCovered cell - Held, RebootSince, KernelPanic, Oops cell - CrashRestarts24h, Guard cell + HostRelease, HostPending, HostNotCovered cell + Held, RebootSince, KernelPanic, Oops cell + CrashRestarts24h, Guard cell + Bundle cell // R-840: the root-owned config bundle // guest GuestDebian, GuestRelease, GuestPending, GuestRestart cell // docker @@ -46,13 +47,43 @@ type systemRow struct { type OSSystemView interface { Fleet() ([]osupdates.FleetLine, error) Releases() []osupdates.ReleaseInfo + CancelledReleases() []osupdates.ReleaseInfo Candidates() []osupdates.Status Thresholds() (stale, reboot, notCovered time.Duration) + BundleThreshold() time.Duration ApproveDocker() (string, error) } func plain(s string) cell { return cell{Text: s} } +// bundleCell is the "Root files" cell (R-840): the box's config bundle against the vouched agent's. Amber while behind, +// red from the alarm's wait on, amber when a file was changed by hand (drift); "unknown" is never coloured as a fact. +func bundleCell(f sysfacts.System, vouchedAgent, vouchedSHA string, since time.Time, after time.Duration, now time.Time) cell { + b := f.Bundle + if !f.Present || b.Version == "" || b.Version == sysfacts.Unknown { + return unknownCell("") + } + c := cell{Text: b.Version} + switch { + case vouchedSHA == "": + c.Title = "no vouched bundle to compare with (the vouched agent carries none)" + case b.BundleSHA256 != vouchedSHA: + c.Class, c.Title = "warn", "behind the vouched agent "+vouchedAgent+"'s bundle — send it with a signed agent_config_update" + if !since.IsZero() { + c.Title += " (behind since " + since.UTC().Format("2006-01-02 15:04") + " UTC)" + if now.Sub(since) >= after { + c.Class = "bad" + } + } + } + if len(b.Drift) > 0 { + c.Text += " (changed by hand)" + c.Class = "warn" + c.Title = "files differ from the installed bundle: " + strings.Join(b.Drift, ", ") + } + return c +} + func unknownCell(s string) cell { if s == "" || s == sysfacts.Unknown { return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"} @@ -207,9 +238,16 @@ func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) { } } stale, reboot, notCov := view.Thresholds() + rows := buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()) + man := s.store.GetArtifactManifest() + for i := range rows { + rows[i].Bundle = bundleCell(facts[rows[i].HostID], man.AgentVersion, man.BundleSHA256, + s.store.BundleBehindSince(rows[i].HostID), view.BundleThreshold(), time.Now()) + } data := map[string]interface{}{ - "Rows": buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()), + "Rows": rows, "Releases": view.Releases(), + "Cancelled": view.CancelledReleases(), "Candidates": view.Candidates(), "Flash": r.URL.Query().Get("flash"), "FlashErr": r.URL.Query().Get("err"), diff --git a/hub/internal/web/system_bundle_test.go b/hub/internal/web/system_bundle_test.go new file mode 100644 index 00000000..7c799fc9 --- /dev/null +++ b/hub/internal/web/system_bundle_test.go @@ -0,0 +1,32 @@ +package web + +import ( + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" +) + +// R-840: the "Root files" cell — amber behind, red from the alarm's wait, amber on drift, unknown never coloured as a fact. +func TestBundleCell(t *testing.T) { + now := time.Date(2026, 10, 4, 20, 0, 0, 0, time.UTC) + sys := func(rep string) sysfacts.System { return sysfacts.Parse(rep) } + cur := sys(`{"system":{"config_bundle":{"version":"0.143.0","bundle_sha256":"new"}}}`) + old := sys(`{"system":{"config_bundle":{"version":"none"}}}`) + drift := sys(`{"system":{"config_bundle":{"version":"0.143.0","bundle_sha256":"new"},"facts":{"host":{"config_bundle":{"version":"0.143.0","drift":["/usr/local/sbin/felhom-pbs-apply"]}}}}}`) + if c := bundleCell(cur, "0.143.0", "new", time.Time{}, 7*24*time.Hour, now); c.Class != "" || c.Text != "0.143.0" { + t.Fatalf("current: %+v", c) + } + if c := bundleCell(old, "0.143.0", "new", now.Add(-time.Hour), 7*24*time.Hour, now); c.Class != "warn" || c.Text != "none" { + t.Fatalf("behind 1 h: %+v", c) + } + if c := bundleCell(old, "0.143.0", "new", now.Add(-8*24*time.Hour), 7*24*time.Hour, now); c.Class != "bad" { + t.Fatalf("behind 8 days: %+v", c) + } + if c := bundleCell(drift, "0.143.0", "new", time.Time{}, 7*24*time.Hour, now); c.Class != "warn" || c.Title == "" { + t.Fatalf("drift: %+v", c) + } + if c := bundleCell(sys(`{}`), "0.143.0", "new", time.Time{}, 7*24*time.Hour, now); c.Text != "unknown" { + t.Fatalf("no stanza: %+v", c) + } +} diff --git a/hub/internal/web/templates/system.html b/hub/internal/web/templates/system.html index 9ab5a816..6aa20900 100644 --- a/hub/internal/web/templates/system.html +++ b/hub/internal/web/templates/system.html @@ -41,9 +41,20 @@
{{range .Releases}}
{{.Layer}}: {{.ID}}
- {{.Packages}} packages · {{.ApprovedAt.UTC.Format "2006-01-02 15:04"}} UTC · by {{.ApprovedBy}}
+ {{.Packages}} packages · {{.ApprovedAt.UTC.Format "2006-01-02 15:04"}} UTC · by {{.ApprovedBy}} + {{if .Test}}
TEST approval{{end}}
{{else}}
No release approved yet.
{{end}} + {{if .Cancelled}} +

Cancelled approvals (last 7 days)

+
+ {{range .Cancelled}} +
{{.Layer}}: {{.ID}}
+ cancelled {{.Cancelled}} UTC{{if .Test}} — a TEST approval{{end}}
+ no further box installs it; boxes that installed it keep it
+ {{end}} +
+ {{end}}

What ring 0 runs now

{{range .Candidates}} @@ -72,12 +83,12 @@ BoxRing / updatesTunnel - ProxmoxKernel (running)Kernel (next boot)DebianFelhom releasePendingNot coveredHeldReboot neededkernel.panicOopsCrash restarts 24 hCrash guard + ProxmoxKernel (running)Kernel (next boot)DebianFelhom releasePendingNot coveredHeldReboot neededkernel.panicOopsCrash restarts 24 hCrash guardRoot files Guest DebianFelhom releasePendingRestart needed Dockercontainerdlive-restoreDocker release Last OS leg - hostguestDocker engine + hostguestDocker engine {{range .Rows}} @@ -103,7 +114,7 @@ {{template "sys_cell" .KernelRunning}}{{template "sys_cell" .KernelNextBoot}}{{template "sys_cell" .HostDebian}} {{template "sys_cell" .HostRelease}}{{template "sys_cell" .HostPending}}{{template "sys_cell" .HostNotCovered}} {{template "sys_cell" .Held}}{{template "sys_cell" .RebootSince}}{{template "sys_cell" .KernelPanic}}{{template "sys_cell" .Oops}} - {{template "sys_cell" .CrashRestarts24h}}{{template "sys_cell" .Guard}} + {{template "sys_cell" .CrashRestarts24h}}{{template "sys_cell" .Guard}}{{template "sys_cell" .Bundle}} {{.GuestDebian.Text}} {{template "sys_cell" .GuestRelease}}{{template "sys_cell" .GuestPending}}{{template "sys_cell" .GuestRestart}} {{.Engine.Text}} diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index 247e6c1b..e7280884 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -1,3 +1,20 @@ +## felhom-host-install.sh 1.31.0 — the root-owned files come from the agent's config bundle (R-840) (2026-10-04) + +Needs a vouched agent ≥ 0.143.0 for the bundle (hub ≥ 0.133.0 serves its sha); an older vouched agent: the per-file +path of 1.30.0, unchanged, with a warning. + +- **One source of truth.** Step 5 fetches `felhom-config-bundle.json` of the vouched agent from the package registry, + verifies its sha256 against the hub manifest (`bundle`), takes out the bundle's own `felhom-os-apply` (checked against + its entry, parsed), installs it and runs `felhom-os-apply --install-bundle` — the SAME code that installs a signed + `agent_config_update` on an installed box: every root-owned file (sudoers, the five wrappers, the crash guard and its + units, the agent and rollback units, the start-limit drop-in, the mgmt watchdog, the OOB belt's files), every check + before the first write, a self-check after, the previous copies kept, everything put back on a failure. The OOB + directory and user are created first so the bundle writes the belt's files; the host key, the belt loader and the + unit enables stay here. +- Uninstall also removes `/etc/felhom/config-bundle.json` and `/var/lib/felhom-os-apply`. +- New: `scripts/felhom-bundle-bootstrap.sh` — the ONE by-hand step an installed box from before agent 0.143.0 needs + (installs only the bundle-aware `felhom-os-apply`, checked against the vouched bundle); `11` §5.4.2. + ## felhom-host-install.sh 1.30.0 — the crash guard and the slow-lane trust files (2026-10-04) Needs agent ≥ 0.142.0 at the pinned tag for the crash guard (an older agent: skipped with a warning, the box keeps diff --git a/scripts/felhom-bundle-bootstrap.sh b/scripts/felhom-bundle-bootstrap.sh new file mode 100644 index 00000000..6bcfc424 --- /dev/null +++ b/scripts/felhom-bundle-bootstrap.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# felhom-bundle-bootstrap.sh — the ONE by-hand act that lets an installed box take config bundles (R-840, `11` §5.4.2). +# +# Why it exists: a signed agent_config_update is installed by the box's ROOT-OWNED felhom-os-apply. A box installed +# before agent v0.143.0 has an older felhom-os-apply that has no bundle mode, and no signed job can write a root file +# on such a box (that gap IS R-840). So the first bundle needs this one step, as root, once per box. After it, every +# later change to the box's root files arrives by the signed route. +# +# What it does: downloads the config bundle of AGENT_VERSION, checks its sha256 against the one you pass (the vouched +# one — the hub's Configuration page or the release output), takes out felhom-os-apply, checks it against the bundle's +# own entry and that it parses, installs it (0755 root:root, the old copy kept beside it), and runs its self-check. +# It changes NOTHING else: no sudoers, no unit, no restart, no app, no Docker. +# +# Usage (as root on the Proxmox host): bash felhom-bundle-bootstrap.sh +set -euo pipefail +VER="${1:-}"; SHA="${2:-}" +[[ "$VER" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "usage: $0 " >&2; exit 2; } +[[ "$SHA" =~ ^[0-9a-f]{64}$ ]] || { echo "the bundle sha256 must be 64 lowercase hex characters" >&2; exit 2; } +[[ $EUID -eq 0 ]] || { echo "run as root" >&2; exit 2; } +URL="https://gitea.dooplex.hu/api/packages/admin/generic/felhom-agent/$VER/felhom-config-bundle.json" +DST=/usr/local/sbin/felhom-os-apply +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT + +echo "1/4 download $URL" +curl -fsS -o "$T/bundle.json" "$URL" +got=$(sha256sum "$T/bundle.json" | awk '{print $1}') +[[ "$got" == "$SHA" ]] || { echo "STOP: the bundle's sha256 is $got, not $SHA — nothing changed" >&2; exit 1; } +echo " sha256 OK ($SHA)" + +echo "2/4 take felhom-os-apply out of the bundle and check it" +python3 - "$T/bundle.json" "$T/os-apply" "$VER" <<'PY' +import ast, base64, hashlib, json, sys +b = json.load(open(sys.argv[1])) +assert b.get("agent_version") == sys.argv[3], f"the bundle is for {b.get('agent_version')}, not {sys.argv[3]}" +e = [f for f in b["files"] if f["path"] == "/usr/local/sbin/felhom-os-apply"][0] +data = base64.b64decode(e["content_b64"]) +assert hashlib.sha256(data).hexdigest() == e["sha256"], "felhom-os-apply does not match its own sha in the bundle" +text = data.decode() +ast.parse(text) +assert 'BUNDLE_OP = "agent_config_update"' in text, "this felhom-os-apply has no bundle mode" +open(sys.argv[2], "wb").write(data) +print(" felhom-os-apply sha256", e["sha256"]) +PY + +echo "3/4 install it (the previous copy is kept as $DST.pre-bundle)" +[[ -f "$DST" ]] && cp -p "$DST" "$DST.pre-bundle" +install -m 0755 -o root -g root "$T/os-apply" "$DST.new.$$" +mv "$DST.new.$$" "$DST" + +echo "4/4 self-check" +out=$(python3 "$DST" --self-check) +echo " $out" +[[ "$out" == *"bundle-format=1"* ]] || { echo "STOP: the self-check failed — put the old copy back: mv $DST.pre-bundle $DST" >&2; exit 1; } +echo "DONE. This box can now take signed config bundles. Nothing else was changed." diff --git a/scripts/felhom-host-install.sh b/scripts/felhom-host-install.sh index c6da1b96..8f559b78 100644 --- a/scripts/felhom-host-install.sh +++ b/scripts/felhom-host-install.sh @@ -184,7 +184,7 @@ set -euo pipefail -SCRIPT_VERSION="1.30.0" # the SINGLE version source (F-1): -h and the run banners follow it. +SCRIPT_VERSION="1.31.0" # the SINGLE version source (F-1): -h and the run banners follow it. # The hub used to carry a copy for its Setup tab; R-94 DELETED it # (2026-08-02) because the hub cannot know which version a box runs — # the Setup command fetches this script at run time. scripts/ @@ -444,6 +444,9 @@ resolve_artifacts() { ART_AGENT_SHA=$(python3 -c "import json,sys;print(json.loads(sys.argv[1])['agent']['sha256'])" "$body" 2>/dev/null || echo "") ART_GOLDEN_VER=$(python3 -c "import json,sys;print(json.loads(sys.argv[1])['golden']['version'])" "$body" 2>/dev/null || echo "") ART_GOLDEN_SHA=$(python3 -c "import json,sys;print(json.loads(sys.argv[1])['golden']['sha256'])" "$body" 2>/dev/null || echo "") + # 1.31.0 (R-840): the vouched agent's CONFIG BUNDLE — every root-owned file below, as one checked unit. Absent when + # the vouched agent carries none (older than 0.143.0); step 5 then fetches the files one by one as before. + ART_BUNDLE_SHA=$(python3 -c "import json,sys;print((json.loads(sys.argv[1]).get('bundle') or {}).get('sha256',''))" "$body" 2>/dev/null || echo "") } # Resolve the Gitea fetch credential (git username + token) from the customer's controller.yaml — @@ -1166,10 +1169,13 @@ run_uninstall() { local cgf for cgf in /usr/local/sbin/felhom-crash-guard /etc/systemd/system/felhom-crash-guard.service \ /etc/systemd/system/felhom-crash-guard-check.service /etc/systemd/system/felhom-crash-guard-check.timer \ - /etc/felhom/crash-guard.conf /etc/felhom/os-trust.json /etc/felhom/operator-signers; do + /etc/felhom/crash-guard.conf /etc/felhom/os-trust.json /etc/felhom/operator-signers \ + /etc/felhom/config-bundle.json; do if [[ -e "$cgf" ]]; then run rm -f "$cgf"; fi done if [[ -d /var/lib/felhom-crash-guard ]]; then run rm -rf /var/lib/felhom-crash-guard; fi + # 1.31.0 (R-840): the bundle's previous copies and the wrapper's nonce record. + if [[ -d /var/lib/felhom-os-apply ]]; then run rm -rf /var/lib/felhom-os-apply; fi if [[ -f /var/lib/vz/snippets/felhom-guest-hook.sh ]]; then run rm -f /var/lib/vz/snippets/felhom-guest-hook.sh; fi local dconf _dnsmasq_touched=false for dconf in /etc/dnsmasq.d/felhom-*.conf; do @@ -2328,6 +2334,76 @@ step_agent_install() { fi fi + # 1.31.0 (R-840): the root-owned files come from the vouched agent's CONFIG BUNDLE — the SAME file a signed + # agent_config_update brings to an installed box, installed by the same code (the bundle's own felhom-os-apply), so + # a new box and an updated box cannot drift. A vouched agent older than 0.143.0 carries none: the per-file path. + if [[ -n "$ART_BUNDLE_SHA" ]]; then + install_root_files_bundle + else + log_warn " the vouched agent v$ART_AGENT_VER carries no config bundle (older than 0.143.0) — fetching the root files one by one" + install_root_files_legacy + install_mgmt_watchdog + # H1: dedicated felhom-sshd OOB instance + static belt (appliance default since v1.25.0; --no-oob opts out). + install_oob + fi + + _state_mark agent_install +} + +# install_root_files_bundle (1.31.0, R-840): fetch the vouched config bundle, verify it against the hub manifest, take +# out its own felhom-os-apply (checked against the bundle's entry), install that, and let it install every root-owned +# file with its checks (visudo, sh/bash -n, python, unit sections, the RuntimeDirectory guard, nft -c), its self-check +# and its undo. Then the parts that are not files: the OOB host key, the belt loader, enabling the units. +install_root_files_bundle() { + local url="$GITEA_BASE/api/packages/$GITEA_OWNER/generic/felhom-agent/$ART_AGENT_VER/felhom-config-bundle.json" + if $DRY_RUN; then + log_dry "fetch $url ; verify sha256=$ART_BUNDLE_SHA ; felhom-os-apply --install-bundle (every root-owned file, one checked unit)" + return 0 + fi + # The bundle writes the OOB belt's files only on a box with the belt: create its directory and user FIRST. + if $ENABLE_OOB; then + install -d -o root -g root -m 0755 /etc/felhom-sshd /etc/felhom-sshd/authorized_keys + id felhom-op >/dev/null 2>&1 || useradd --create-home --shell /bin/bash felhom-op + fi + local btmp ostmp out rc=0 + btmp=$(mktemp -t felhom-bundle.XXXXXX); ostmp=$(mktemp -t felhom-os.XXXXXX) + fetch_verify "$url" "$btmp" "$ART_BUNDLE_SHA" + python3 - "$btmp" "$ostmp" <<'PY' || { rm -f "$btmp" "$ostmp"; die "the config bundle carries no valid felhom-os-apply — refusing"; } +import ast, base64, hashlib, json, sys +b = json.load(open(sys.argv[1])) +e = [f for f in b["files"] if f["path"] == "/usr/local/sbin/felhom-os-apply"][0] +data = base64.b64decode(e["content_b64"]) +assert hashlib.sha256(data).hexdigest() == e["sha256"], "felhom-os-apply does not match its sha in the bundle" +ast.parse(data.decode()) +open(sys.argv[2], "wb").write(data) +PY + install -m 0755 -o root -g root "$ostmp" /usr/local/sbin/felhom-os-apply + rm -f "$ostmp" + # The wrapper refuses --install-bundle from a sudo caller (the agent's route is the signed job); this script is + # root already, so a `sudo bash` run must not look like one. + out=$(env -u SUDO_UID -u SUDO_GID -u SUDO_USER -u SUDO_COMMAND \ + /usr/local/sbin/felhom-os-apply --install-bundle "$btmp" --sha256 "$ART_BUNDLE_SHA" 2>&1) || rc=$? + rm -f "$btmp" + grep '^os-apply: BUNDLE' <<<"$out" | sed 's/^/ /' || true + [[ $rc -eq 0 ]] || die "the config bundle did not install (rc=$rc) — nothing half-done stays (the wrapper put the previous files back): $(grep -E 'REFUSED|FAILED' <<<"$out" | head -2)" + systemctl daemon-reload + systemctl enable felhom-agent >/dev/null 2>&1 || true + log_success " installed every root-owned file from config bundle v$ART_AGENT_VER (sha ${ART_BUNDLE_SHA:0:16}…; checked, previous copies kept)" + if $ENABLE_OOB; then + if [[ ! -f /etc/felhom-sshd/ssh_host_ed25519_key ]]; then + ssh-keygen -t ed25519 -N "" -f /etc/felhom-sshd/ssh_host_ed25519_key -C felhom-sshd-hostkey -q + chmod 600 /etc/felhom-sshd/ssh_host_ed25519_key + fi + systemctl enable --now felhom-oob-nft.service >/dev/null 2>&1 || true # load the static belt now + systemctl enable felhom-sshd >/dev/null 2>&1 || true # NOT start — the agent renders the config first + log_success " OOB felhom-sshd instance + static belt from the bundle (agent renders config + fills sets once oob.enabled)" + else + log_skip " OOB (felhom-sshd) off (byo, or appliance --no-oob) — the bundle skipped its files" + fi +} + +# install_root_files_legacy is the per-file path (before 1.31.0, and for a vouched agent older than 0.143.0). +install_root_files_legacy() { # Guarded-mkfs wrapper (Impl-1 Part B) — the ONLY mkfs path the sudoers permits. Install it BEFORE # the sudoers (which allowlists it), 0755 root:root under /usr/local/sbin. bash -n before install. if $DRY_RUN; then @@ -2503,12 +2579,6 @@ step_agent_install() { # Non-fatal if the agent repo predates them (raw fetch 404s → break-glass just stays manual). # HARD GUARD: refuse ANY fetched unit that declares RuntimeDirectory= — that directive is the very # incident G1 closes (a second sshd's `RuntimeDirectory=sshd` removed the shared /run/sshd). - install_mgmt_watchdog - - # H1: dedicated felhom-sshd OOB instance + static belt (appliance default since v1.25.0; --no-oob opts out). - install_oob - - _state_mark agent_install } # install_mgmt_watchdog fetches + installs the G1 break-glass host artifacts (idempotent; enables the