From 88b0a2e761402f09306a52bc3405c7705835032e Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 4 Oct 2026 13:05:00 +0200 Subject: [PATCH] =?UTF-8?q?hub=20v0.131.0:=20the=20tunnel=20status=20is=20?= =?UTF-8?q?true=20(R-841:=20three=20states,=20tunnel=5Fdown=20after=20two?= =?UTF-8?q?=20not=5Frunning=20reports,=20unknown=20never=20alarms);=20OS?= =?UTF-8?q?=20updates=20per=20layer=20(guest/host=20separate=20approved=20?= =?UTF-8?q?sets,=20host=20candidate=20leaves=20out=20kernel/boot/firmware,?= =?UTF-8?q?=20host=5Frelease=20in=20the=20box=20block),=20the=20fleet=20vi?= =?UTF-8?q?ew=20and=20four=20hourly=20operator=20alarms=20(11=20=C2=A78=20?= =?UTF-8?q?steps=203+4);=20red-proofs=20in=20audits/os-host-lane-2026-10-0?= =?UTF-8?q?4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../partA/agent-redproofs.txt | 9 + .../partA/controller-redproof.txt | 3 + .../partA/hub-redproofs.txt | 17 + .../partB/agent-leg-redproofs.txt | 22 + .../partB/wrapper-redproof.txt | 20 + .../partC/agent-golden-redproof.txt | 8 + .../partC/hub-redproofs.txt | 30 + .../os-host-lane-2026-10-04/partC/hub_rp.py | 44 ++ hub/CHANGELOG.md | 29 + hub/cmd/hub/main.go | 46 +- hub/internal/api/handler.go | 41 +- hub/internal/api/os_updates_test.go | 14 +- .../desired-state-osupdate.golden.json | 9 +- hub/internal/api/tunnel_test.go | 51 ++ hub/internal/notify/dispatcher.go | 8 + .../notify/os_alarms_operator_test.go | 23 + hub/internal/osupdates/alarms_test.go | 306 ++++++++++ hub/internal/osupdates/service.go | 529 ++++++++++++++---- hub/internal/osupdates/service_test.go | 29 +- hub/internal/store/os_updates.go | 140 ++++- hub/internal/store/store.go | 18 + hub/internal/web/hosts.go | 3 + .../web/templates/host_detail_body.html | 2 +- hub/internal/web/templates/hosts.html | 2 +- 24 files changed, 1254 insertions(+), 149 deletions(-) create mode 100644 documentation/audits/os-host-lane-2026-10-04/partA/agent-redproofs.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partA/controller-redproof.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partB/agent-leg-redproofs.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partB/wrapper-redproof.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partC/agent-golden-redproof.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partC/hub-redproofs.txt create mode 100644 documentation/audits/os-host-lane-2026-10-04/partC/hub_rp.py create mode 100644 hub/internal/api/tunnel_test.go create mode 100644 hub/internal/notify/os_alarms_operator_test.go create mode 100644 hub/internal/osupdates/alarms_test.go diff --git a/documentation/audits/os-host-lane-2026-10-04/partA/agent-redproofs.txt b/documentation/audits/os-host-lane-2026-10-04/partA/agent-redproofs.txt new file mode 100644 index 00000000..ad39b056 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partA/agent-redproofs.txt @@ -0,0 +1,9 @@ +# agent v0.141.0 tunnel-probe red-proofs (each mutation compiles; result line printed) +== mutation: container-state-only + cloudflared_test.go:32: running but not connected: got "running" (container running but the tunnel is NOT connected (cloudflared /ready fails)), want "not_running" (…NOT connected…) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.009s +== mutation: unknown-as-down + cloudflared_test.go:32: guest not running: got "not_running" (could not ask the guest: CT 9201 not running), want "unknown" (…could not ask…) + cloudflared_test.go:32: sudo refused: got "not_running" (could not ask the guest: sudo: a password is required), want "unknown" (…could not ask…) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.009s +ok gitea.dooplex.hu/admin/felhom-agent/internal/hub (cached) diff --git a/documentation/audits/os-host-lane-2026-10-04/partA/controller-redproof.txt b/documentation/audits/os-host-lane-2026-10-04/partA/controller-redproof.txt new file mode 100644 index 00000000..879e2b0b --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partA/controller-redproof.txt @@ -0,0 +1,3 @@ +== RP (controller): the healthcheck block removed from the cloudflared template + infra_tunnel_test.go:300: tunnel=false: compose lacks "test: [\"CMD\", \"cloudflared\", \"tunnel\", \"--metrics\", \"localhost:20241\", \"ready\"]": +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.007s diff --git a/documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt b/documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt new file mode 100644 index 00000000..22016693 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt @@ -0,0 +1,17 @@ +# hub v0.131.0 tunnel-alarm red-proofs, 2026-10-04 (each mutation compiles; the package result line is printed) +== mutation: one-report + --- FAIL: TestTunnelAlarm_Sequences/a_single_bad_report_does_not_alarm (0.02s) + tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0 + --- FAIL: TestTunnelAlarm_Sequences/unknown_never_alarms_and_breaks_the_run (0.03s) + tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0 +== mutation: unknown-counts-as-down + --- FAIL: TestTunnelAlarm_Sequences/unknown_never_alarms_and_breaks_the_run (0.03s) + tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0 +FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.164s +== mutation: no-alarm + --- FAIL: TestTunnelAlarm_Sequences/two_in_a_row_alarm_once (0.03s) + tunnel_test.go:47: tunnel_down=0 tunnel_recovered=0, want 1 / 0 + --- FAIL: TestTunnelAlarm_Sequences/recovery_after_an_alarm (0.04s) + tunnel_test.go:47: tunnel_down=0 tunnel_recovered=1, want 1 / 1 +== reverted +ok gitea.dooplex.hu/admin/felhom-hub/internal/api 0.168s diff --git a/documentation/audits/os-host-lane-2026-10-04/partB/agent-leg-redproofs.txt b/documentation/audits/os-host-lane-2026-10-04/partB/agent-leg-redproofs.txt new file mode 100644 index 00000000..5556f4e8 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partB/agent-leg-redproofs.txt @@ -0,0 +1,22 @@ +# agent v0.141.0 leg red-proofs (each mutation compiles; result line printed) +== mutation: byo-gets-host +--- FAIL: TestBYO_NoHostPlan (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.440s +== mutation: host-after-failed-guest +--- FAIL: TestGuestFailure_SkipsTheHost (0.00s) +--- FAIL: TestHealth_FailsAfterTheWait (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.420s +== mutation: host-ignores-services +--- FAIL: TestHostHealthVerdict (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.441s +== mutation: host-ignores-guest +--- FAIL: TestHostHealthVerdict (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.410s +== mutation: host-ignores-tunnel +--- FAIL: TestHost_TunnelDownFailsTheHostStep (0.00s) +--- FAIL: TestHostHealthVerdict (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.427s +== mutation: host-uses-guest-release +--- FAIL: TestRing1_EachLayerItsOwnRelease (0.00s) +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.461s +ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate (cached) diff --git a/documentation/audits/os-host-lane-2026-10-04/partB/wrapper-redproof.txt b/documentation/audits/os-host-lane-2026-10-04/partB/wrapper-redproof.txt new file mode 100644 index 00000000..cb8f7021 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partB/wrapper-redproof.txt @@ -0,0 +1,20 @@ +# felhom-os-apply (agent v0.141.0) red-proof, 2026-10-04: each mutation is applied to a COPY, syntax-checked, and the suite run against it. +R1 (7 raise(s) removed): compiles=True suite rc=1; failing: test_R1_not_owned_by_the_agent, test_R1_path_outside_the_plan_dir, test_R1_symlink; expected test failed: YES +R2 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R2_non_debian_origin_in_the_plan, test_R2_non_debian_origin_in_the_simulation; expected test failed: YES +R3 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R3_slow_lane; expected test failed: YES +R4 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R4_removal; expected test failed: YES +R5 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R5_downgrade_exact; expected test failed: YES +R6 (4 raise(s) removed): compiles=True suite rc=1; failing: test_R6_allow_new_is_slow_lane, test_R6_new_package, test_R6_unlisted_package; expected test failed: YES +R7 (7 raise(s) removed): compiles=True suite rc=1; failing: test_R7_not_downloadable_and_no_snapshot, test_snapshot_does_not_have_it_either; expected test failed: YES +R8 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R8_free_space; expected test failed: YES +R9 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R9_apt_lock_held, test_R9_guest_locked_by_a_backup; expected test failed: YES +R10 (4 raise(s) removed): compiles=True suite rc=1; failing: test_R10_bind_only_in_a_snapshot_section, test_R10_not_running, test_R10_not_the_boxs_own_guest, test_R10_reserved_vmid; expected test failed: YES +R11 (12 raise(s) removed): compiles=True suite rc=1; failing: test_R11_bad_name, test_R11_bad_version_string, test_R11_duplicate; expected test failed: YES +R12 (5 raise(s) removed): compiles=True suite rc=1; failing: test_R12_host_on_a_byo_box, test_R12_host_record_not_root_owned, test_R12_host_without_an_install_record, test_R12_unknown_layer; expected test failed: YES +R13 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R13_repair_does_not_fix_it; expected test failed: YES +R14 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R14_kernel_package_in_a_host_plan, test_R14_kernel_package_pulled_by_the_simulation; expected test failed: YES +host layer runs in the guest instead: compiles=True suite rc=1; failing: test_host_runs_on_the_host_not_in_the_guest; expected test failed: YES +pending-fast takes every pending upgrade: compiles=True suite rc=1; failing: test_pending_fast_skips_proxmox_docker_and_kernel; expected test failed: YES +version checks in the target (pct exec per package): compiles=True suite rc=1; failing: test_no_per_package_guest_calls; expected test failed: YES +lxc-start does not mean reboot: compiles=True suite rc=0; failing: NONE; expected test failed: NO +lxc-start does not mean reboot (after adding test_reboot_needed_for_lxc_start_alone): suite rc=1; failing: test_reboot_needed_for_lxc_start_alone diff --git a/documentation/audits/os-host-lane-2026-10-04/partC/agent-golden-redproof.txt b/documentation/audits/os-host-lane-2026-10-04/partC/agent-golden-redproof.txt new file mode 100644 index 00000000..86052876 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partC/agent-golden-redproof.txt @@ -0,0 +1,8 @@ +# Agent red-proof: the host_release wire tag (contract with the hub golden), 2026-10-04T11:04:25Z +# mutation: report.go json:"host_release" -> json:"hostrelease"; restored after +=== RUN TestOSUpdateGolden_Decodes + osupdate_contract_test.go:34: host_release = +--- FAIL: TestOSUpdateGolden_Decodes (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.007s +FAIL diff --git a/documentation/audits/os-host-lane-2026-10-04/partC/hub-redproofs.txt b/documentation/audits/os-host-lane-2026-10-04/partC/hub-redproofs.txt new file mode 100644 index 00000000..99c1a909 --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partC/hub-redproofs.txt @@ -0,0 +1,30 @@ +# Hub red-proofs, Parts B+C (hub v0.131.0), 2026-10-04T11:03:52Z +# script: each mutation applied, the named test run with -v, file restored. +[CAUGHT] host candidate leaves out kernel/boot/firmware: mutated internal/osupdates/service.go; TestCandidate_HostLeavesOutKernelBootFirmware ran=True failed=True rc=1 + alarms_test.go:27: host candidate = map[firmware-realtek:{Name:firmware-realtek Version:20250410-2 Origin:Debian} grub-efi-amd64:{Name:grub-efi-amd64 Version:2.12-9 Origin:Debian} intel-microcode:{Name:intel-microcode Ve + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.034s +[CAUGHT] host nights counted per layer: mutated internal/osupdates/service.go; TestLayers_SeparateSets ran=True failed=True rc=1 + alarms_test.go:60: host approved on a GUEST night: &{ID:os-host-20261005-130000 Layer:host Fingerprint:535006df2f78491d ApprovedAt:2026-10-05 13:00:00 +0000 UTC ApprovedBy:auto PackagesJSON:[{"name":"libssl3t64","version + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.042s +[CAUGHT] alarm 1 stale threshold: mutated internal/osupdates/service.go; TestAlarm_StaleLeg ran=True failed=True rc=1 + alarms_test.go:151: no alarm at 8 days: [] + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.041s +[CAUGHT] alarm 1 switch OFF not watched: mutated internal/osupdates/service.go; TestAlarm_StaleLeg ran=True failed=True rc=1 + alarms_test.go:168: alarmed with the switch OFF + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.054s +[CAUGHT] alarm 2 reboot-needed counts from the FIRST report: mutated internal/osupdates/service.go; TestAlarm_RebootNeeded ran=True failed=True rc=1 + alarms_test.go:211: reboot-needed since = 2026-10-17 12:00:00 +0000 UTC + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.045s +[CAUGHT] alarm 3 counts only fast-lane pending: mutated internal/osupdates/service.go; TestAlarm_Ring0Stalled ran=True failed=True rc=1 + alarms_test.go:235: a pending KERNEL counted as a stalled fast lane + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.040s +[CAUGHT] alarm 4 ring 0 never alarms: mutated internal/osupdates/service.go; TestAlarm_NotCovered ran=True failed=True rc=1 + alarms_test.go:278: not-covered alarms = ["OS updates (guest): cust1 has had 1 fast-lane package(s) no approved release covers since 2026-10-04: libsomething-hw. Ring 0 does not run them (other hardware?) — add matching + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.060s +[CAUGHT] alarm events are operator-only: mutated internal/notify/dispatcher.go; TestOSUpdateEvents_OperatorOnlyExceptApplied ran=True failed=True rc=1 + os_alarms_operator_test.go:17: os_ring0_stalled is not operator-only + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/notify 0.009s +[CAUGHT] host_release served to ring 1: mutated internal/osupdates/service.go; TestOSUpdate_DesiredBlockMatchesTheGolden ran=True failed=True rc=1 + os_updates_test.go:59: served os_update diverged from the golden: + FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.055s +after restore: ok gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.544s | ok gitea.dooplex.hu/admin/felhom-hub/internal/notify 1.169s | ok gitea.dooplex.hu/admin/felhom-hub/internal/api 4.690s rc= 0 diff --git a/documentation/audits/os-host-lane-2026-10-04/partC/hub_rp.py b/documentation/audits/os-host-lane-2026-10-04/partC/hub_rp.py new file mode 100644 index 00000000..1bd5131a --- /dev/null +++ b/documentation/audits/os-host-lane-2026-10-04/partC/hub_rp.py @@ -0,0 +1,44 @@ +import subprocess, sys, shutil +M = [ + ("host candidate leaves out kernel/boot/firmware", "internal/osupdates/service.go", + "if layer == LayerHost && hostSlowRE.MatchString(p.Name) {\n\t\t\t\tcontinue", "if false && hostSlowRE.MatchString(p.Name) {\n\t\t\t\tcontinue", + "./internal/osupdates/", "TestCandidate_HostLeavesOutKernelBootFirmware"), + ("host nights counted per layer", "internal/osupdates/service.go", + "reps, err := s.Store.OSReportsSince(h, layer, first)", "reps, err := s.Store.OSReportsSince(h, LayerGuest, first)", + "./internal/osupdates/", "TestLayers_SeparateSets"), + ("alarm 1 stale threshold", "internal/osupdates/service.go", + "now.Sub(ref) >= stale", "now.Sub(ref) >= 10*stale", "./internal/osupdates/", "TestAlarm_StaleLeg"), + ("alarm 1 switch OFF not watched", "internal/osupdates/service.go", + "holds := st.Enabled && canRun", "holds := canRun", "./internal/osupdates/", "TestAlarm_StaleLeg"), + ("alarm 2 reboot-needed counts from the FIRST report", "internal/osupdates/service.go", + "\t\tsince = rep.ReceivedAt\n", "\t\tif since.IsZero() {\n\t\t\tsince = rep.ReceivedAt\n\t\t}\n", "./internal/osupdates/", "TestAlarm_RebootNeeded"), + ("alarm 3 counts only fast-lane pending", "internal/osupdates/service.go", + "pendingAny += len(fastPending(layer, r.Pending))", "pendingAny += len(r.Pending)", "./internal/osupdates/", "TestAlarm_Ring0Stalled"), + ("alarm 4 ring 0 never alarms", "internal/osupdates/service.go", + "\tif ring == 0 {\n\t\treturn nil\n\t}\n", "", "./internal/osupdates/", "TestAlarm_NotCovered"), + ("alarm events are operator-only", "internal/notify/dispatcher.go", + '\t"os_ring0_stalled": true,\n', "", "./internal/notify/", "TestOSUpdateEvents_OperatorOnlyExceptApplied"), + ("host_release served to ring 1", "internal/osupdates/service.go", + "b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost)", "b.Release = s.releaseBlock(LayerGuest)", + "./internal/api/", "TestOSUpdate_DesiredBlockMatchesTheGolden"), +] +ok = True +for name, f, old, new, pkg, test in M: + src = open(f).read() + assert src.count(old) == 1, (name, src.count(old)) + shutil.copy(f, f + ".bak") + open(f, "w").write(src.replace(old, new)) + r = subprocess.run(["go", "test", pkg, "-run", "^" + test + "$", "-v", "-count=1"], capture_output=True, text=True) + shutil.move(f + ".bak", f) + out = r.stdout + r.stderr + ran = ("=== RUN " + test) in out + failed = ("--- FAIL: " + test) in out + verdict = "CAUGHT" if (ran and failed and r.returncode != 0) else "NOT CAUGHT" + if verdict != "CAUGHT": ok = False + print(f"[{verdict}] {name}: mutated {f}; {test} ran={ran} failed={failed} rc={r.returncode}") + for l in out.splitlines(): + if "_test.go:" in l or l.startswith("FAIL\t") or "build failed" in l or l.startswith("#"): + print(" " + l.strip()[:220]) +r = subprocess.run(["go", "test", "./internal/osupdates/", "./internal/notify/", "./internal/api/", "-count=1"], capture_output=True, text=True) +print("after restore:", r.stdout.strip().replace("\n", " | "), "rc=", r.returncode) +sys.exit(0 if ok and r.returncode == 0 else 1) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index cd826b14..a7f30aa0 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,32 @@ +## v0.131.0 — the tunnel status is true (R-841); OS updates: the host fast lane's own approved set, the fleet view and four alarms (`11` §8 steps 3+4) (2026-10-04) + +**Needs agent v0.141.0** for the tunnel's three states and the host layer. An older agent still reports +`cloudflared.status` (read as before) and never sends a host-layer report — nothing breaks. + +- **R-841 — three tunnel states.** The agent now reports `running` / `not_running` / `unknown` plus a short `detail` + (agent v0.141.0 reads the guest's cloudflared container and its readiness check). The host list and the host page + show the state and the detail. **`unknown` never alarms.** A tunnel `not_running` in the newest TWO reports (more + than one report cycle) raises `tunnel_down` (error, operator-only, mailed once per run); the first `running` after + that raises `tunnel_recovered` (info). An `unknown` report breaks the run. Pinned by `api/tunnel_test.go`. +- **Two layers, two approved sets.** OS reports and OS releases carry a `layer` (`guest` | `host`; old rows read as + `guest`). The candidate, the 24 h + 1 night rule and "every ring-0 box" are applied PER LAYER: a guest night run + does not count toward a host approval. Release ids are `os-guest-…` / `os-host-…`. The host candidate leaves out + kernel, boot and firmware names (the host's slow lane; the same pattern as the wrapper's R14 refusal). + Ring 1 receives `os_update.host_release` beside `release` — golden updated in BOTH repos. +- **The fleet view** (`GET /os/fleet`, operator): one line per box — ring, switch, the tunnel, and per layer: the + release, last outcome, last successful leg, pending, not-covered, restart-needed, reboot-needed-since (the first + report that said so) and the wrapper's own seconds. +- **Four alarms**, checked hourly, each operator-only, raised at most once a week and cleared when the condition goes + away: `os_update_stale` (no successful OS leg for 7 days while the switch is ON; agents ≥ 0.140.0 only; the mail + names the likely reason), `os_reboot_needed` (host reboot needed for 14 days), `os_ring0_stalled` (ring 0 approved + nothing for 7 days while it has pending fast-lane updates — a pending kernel does not count), `os_not_covered` + (a ring-1 box has fast-lane packages no release covers for 14 days). The numbers are configuration + (`OS_ALARM_STALE_AFTER`, `OS_ALARM_REBOOT_AFTER`, `OS_ALARM_RING0_STALL_AFTER`, `OS_ALARM_NOT_COVERED_AFTER`), + logged at start; recorded in `11` §8.3 as decided by CC unattended — operator may reverse. +- **Host health mail** names the host undo runbook (`runbooks/os-updates-host-undo.md`), not the guest backup. +- Red-proofs: `documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt` (tunnel) and + `partC/hub-redproofs.txt` (9 mutations, all caught). + ## v0.130.0 — OS updates, guest fast lane: rings, the per-box switch, OS releases approved from ring 0 (`11` §8 step 2) - **The OS release.** Ring-0 boxes (the demo boxes) report every OS-leg run to `POST /api/v1/hosts/{id}/os-report` diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 08cd7e81..4b9e07b3 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -414,23 +414,57 @@ func main() { logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired) apiHandler.SetOSUpdateService(osSvc) webServer.SetOSUpdateAdmin(osSvc) + // `11` §8.3: the four alarm thresholds are configuration (decided by CC unattended — operator may reverse). + for _, a := range []struct { + env string + dst *time.Duration + def time.Duration + }{ + {"OS_ALARM_STALE_AFTER", &osSvc.StaleAfter, 7 * 24 * time.Hour}, + {"OS_ALARM_REBOOT_AFTER", &osSvc.RebootAfter, 14 * 24 * time.Hour}, + {"OS_ALARM_RING0_STALL_AFTER", &osSvc.Ring0StallAfter, 7 * 24 * time.Hour}, + {"OS_ALARM_NOT_COVERED_AFTER", &osSvc.NotCoveredAfter, 14 * 24 * time.Hour}, + } { + *a.dst = a.def + if v := os.Getenv(a.env); v != "" { + if d, derr := time.ParseDuration(v); derr == nil && d > 0 { + *a.dst = d + } else { + logger.Printf("[ERROR] %s=%q invalid — keeping %s", a.env, v, a.def) + } + } + } + logger.Printf("[INFO] osupdates: alarms = no OS leg %s, reboot needed %s, ring 0 stalled %s, not covered %s", + osSvc.StaleAfter, osSvc.RebootAfter, osSvc.Ring0StallAfter, osSvc.NotCoveredAfter) go func() { tk := time.NewTicker(60 * time.Second) defer tk.Stop() - last := "" + last := map[string]string{} + var lastAlarms time.Time for { select { case <-ctx.Done(): return case <-tk.C: - st, err := osSvc.Evaluate() + sts, err := osSvc.Evaluate() if err != nil { logger.Printf("[WARN] osupdates: evaluate: %v", err) - continue } - if msg := st.Fingerprint + "|" + st.Waiting; msg != last { - last = msg - logger.Printf("[INFO] osupdates: candidate %s (%d packages, first seen %s): %s", st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""]) + for _, st := range sts { + if msg := st.Fingerprint + "|" + st.Waiting; msg != last[st.Layer] { + last[st.Layer] = msg + logger.Printf("[INFO] osupdates: %s candidate %s (%d packages, first seen %s): %s", st.Layer, st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""]) + } + } + if time.Since(lastAlarms) >= time.Hour { + lastAlarms = time.Now() + start := time.Now() + sent, aerr := osSvc.Alarms() + if aerr != nil { + logger.Printf("[WARN] osupdates: alarms: %v", aerr) + } else { + logger.Printf("[INFO] osupdates: alarm check done in %s, %d alarm(s) sent %v", time.Since(start).Round(time.Millisecond), len(sent), sent) + } } } } diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 9ef39b02..86c4b0f8 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -689,7 +689,8 @@ type hostReportPayload struct { RestoreTests []hostRestoreTest `json:"restore_tests"` // slice 6 PBSSnapshots []hostPBSSnapshot `json:"pbs_snapshots"` // slice 6 Phase B Cloudflared struct { - Status string `json:"status"` + Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown (older: active | inactive | …) + Detail string `json:"detail,omitempty"` } `json:"cloudflared"` // DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim, // ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half. @@ -861,6 +862,8 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) { return } + h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail) + for _, g := range rep.Guests { status := g.Status if status == "" { @@ -2835,3 +2838,39 @@ func (h *Handler) handleAssetFile(w http.ResponseWriter, r *http.Request, filena h.assetsMgr.ServeFile(w, r, filename) } + +// Tunnel alarm (R-841, hub v0.131.0). The agent (>= 0.141.0) reports the box's tunnel as running | not_running | +// unknown, read from the cloudflared container's own readiness check. `not_running` in TWO reports in a row — more +// than one report cycle — is an operator alarm (`tunnel_down`, error); the first `running` after that is +// `tunnel_recovered`. `unknown` never alarms and breaks a run of not_running (R-96 rule 3: a probe that could not +// ask is not evidence of down). Pinned by TestTunnelAlarm_*. +const ( + eventTunnelDown = "tunnel_down" + eventTunnelRecovered = "tunnel_recovered" +) + +func (h *Handler) checkTunnel(hostID, custID, status, detail string) { + st, err := h.store.RecentCloudflaredStatuses(hostID, 3) + if err != nil || len(st) < 2 { + return + } + down := func(i int) bool { return i < len(st) && st[i] == "not_running" } + var typ, sev, msg string + switch { + case down(0) && down(1) && !down(2): + typ, sev = eventTunnelDown, "error" + msg = fmt.Sprintf("The tunnel of %s is NOT running in two reports in a row (%s). The box is not reachable from outside its home.", hostID, detail) + case st[0] == "running" && down(1) && down(2): + typ, sev = eventTunnelRecovered, "info" + msg = fmt.Sprintf("The tunnel of %s is running again.", hostID) + default: + return + } + h.logger.Printf("[WARN] host %s tunnel: %s (%s)", hostID, typ, detail) + details, _ := json.Marshal(map[string]any{"host_id": hostID, "status": status, "detail": detail}) + if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil { + h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr) + } else if h.dispatcher != nil { + go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub") + } +} diff --git a/hub/internal/api/os_updates_test.go b/hub/internal/api/os_updates_test.go index 04aad426..b892802e 100644 --- a/hub/internal/api/os_updates_test.go +++ b/hub/internal/api/os_updates_test.go @@ -21,7 +21,11 @@ func TestOSUpdate_DesiredBlockMatchesTheGolden(t *testing.T) { svc := &osupdates.Service{Store: st, ApproveAfter: 0, NightsRequired: 0, Logger: log.New(os.Stderr, "", 0)} h.SetOSUpdateService(svc) rel := `[{"name":"libc6","version":"2.41-12+deb13u4","origin":"Debian"},{"name":"openssl","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]` - if err := st.SaveOSRelease(storeRelease("os-20261004-120000", rel)); err != nil { + if err := st.SaveOSRelease(storeRelease("os-guest-20261004-120000", "guest", rel)); err != nil { + t.Fatal(err) + } + hrel := `[{"name":"libssl3t64","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]` + if err := st.SaveOSRelease(storeRelease("os-host-20261004-120000", "host", hrel)); err != nil { t.Fatal(err) } rr := do(h, http.MethodGet, "/hosts/h1/desired-state", "HKEY1", "") @@ -66,18 +70,18 @@ func TestOSReport_SelfScoped(t *testing.T) { if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY2", body); rr.Code != http.StatusForbidden { t.Fatalf("cross-host report → %d, want 403", rr.Code) } - if r, _ := st.LatestOSReport("h1"); r != nil { + if r, _ := st.LatestOSReport("h1", "guest"); r != nil { t.Fatal("a refused report was stored") } if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY1", body); rr.Code != http.StatusOK { t.Fatalf("own report → %d %s", rr.Code, rr.Body.String()) } - if r, _ := st.LatestOSReport("h1"); r == nil || r.Outcome != "applied" { + if r, _ := st.LatestOSReport("h1", "guest"); r == nil || r.Outcome != "applied" { t.Fatalf("report not stored: %+v", r) } } -func storeRelease(id, pkgs string) store.OSRelease { +func storeRelease(id, layer, pkgs string) store.OSRelease { at, _ := time.Parse(time.RFC3339, "2026-10-04T12:00:00Z") - return store.OSRelease{ID: id, Fingerprint: "fp", ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs} + return store.OSRelease{ID: id, Layer: layer, Fingerprint: "fp-" + layer, ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs} } diff --git a/hub/internal/api/testdata/desired-state-osupdate.golden.json b/hub/internal/api/testdata/desired-state-osupdate.golden.json index da28c48c..cd3cbae7 100644 --- a/hub/internal/api/testdata/desired-state-osupdate.golden.json +++ b/hub/internal/api/testdata/desired-state-osupdate.golden.json @@ -5,12 +5,19 @@ "ring": 1, "enabled": true, "release": { - "id": "os-20261004-120000", + "id": "os-guest-20261004-120000", "snapshot": "20261004T120000Z", "packages": [ {"name": "libc6", "version": "2.41-12+deb13u4", "origin": "Debian"}, {"name": "openssl", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"} ] + }, + "host_release": { + "id": "os-host-20261004-120000", + "snapshot": "20261004T120000Z", + "packages": [ + {"name": "libssl3t64", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"} + ] } } } diff --git a/hub/internal/api/tunnel_test.go b/hub/internal/api/tunnel_test.go new file mode 100644 index 00000000..00a48889 --- /dev/null +++ b/hub/internal/api/tunnel_test.go @@ -0,0 +1,51 @@ +package api + +import ( + "net/http" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +func tunnelReport(status, detail string) string { + return strings.Replace(validReportBody("h1"), `"cloudflared":{"status":"active"}`, + `"cloudflared":{"status":"`+status+`","detail":"`+detail+`"}`, 1) +} + +// R-841: the tunnel alarm fires when the tunnel is not running in TWO reports in a row; running never alarms; +// one bad report never alarms; `unknown` never alarms and breaks a run; recovery is announced once. +// Red-proof: make checkTunnel alarm on ONE not_running (drop down(1)) and "a single bad report" fails. +func TestTunnelAlarm_Sequences(t *testing.T) { + cases := []struct { + name string + seq []string + down int + recov int + }{ + {"running never alarms", []string{"running", "running", "running"}, 0, 0}, + {"a single bad report does not alarm", []string{"running", "not_running", "running"}, 0, 0}, + {"two in a row alarm once", []string{"running", "not_running", "not_running", "not_running"}, 1, 0}, + {"unknown never alarms and breaks the run", []string{"not_running", "unknown", "not_running", "unknown"}, 0, 0}, + {"recovery after an alarm", []string{"not_running", "not_running", "running"}, 1, 1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + h, st, _ := newTestHandler(t) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"}) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + for _, s := range c.seq { + if rr := do(h, http.MethodPost, "/host-report", "HKEY", tunnelReport(s, "exited, exit code 1")); rr.Code != 200 { + t.Fatalf("report: %d %s", rr.Code, rr.Body.String()) + } + } + since := time.Now().Add(-time.Hour) + d, _ := st.GetEventsByType("c1", eventTunnelDown, since) + r, _ := st.GetEventsByType("c1", eventTunnelRecovered, since) + if len(d) != c.down || len(r) != c.recov { + t.Fatalf("tunnel_down=%d tunnel_recovered=%d, want %d / %d", len(d), len(r), c.down, c.recov) + } + }) + } +} diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index d222965a..2e2b1698 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -687,6 +687,14 @@ var operatorOnlyEvents = map[string]bool{ "os_release_approved": true, "os_release_approved_now": true, "os_update_settings_changed": true, + // R-841 (hub v0.131.0): the tunnel alarm — a box fact the household can do nothing about from inside. + "tunnel_down": true, + "tunnel_recovered": true, + // `11` §8.3 (hub v0.131.0): the four OS-update alarms — fleet facts only the operator can act on. + "os_update_stale": true, + "os_reboot_needed": true, + "os_ring0_stalled": true, + "os_not_covered": true, // R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow // blobs. A customer can take no action on it — the remedy is the operator's inspection of the // off-site tier — and the text is operator-grade English naming host ids and retained-blob diff --git a/hub/internal/notify/os_alarms_operator_test.go b/hub/internal/notify/os_alarms_operator_test.go new file mode 100644 index 00000000..ca67bc3b --- /dev/null +++ b/hub/internal/notify/os_alarms_operator_test.go @@ -0,0 +1,23 @@ +package notify + +import ( + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" +) + +// Every OS-update event except the household's own line is operator-only (`11` §5.7, §8.3). An operator alarm left +// off this list reaches the customer as raw operator English. Red-proof: delete one alarm line from +// operatorOnlyEvents and this fails naming it. +func TestOSUpdateEvents_OperatorOnlyExceptApplied(t *testing.T) { + for _, e := range []string{osupdates.EventFailed, osupdates.EventHealthFailed, osupdates.EventReleaseApprove, + osupdates.EventApprovedNow, osupdates.EventSettings, osupdates.EventStale, osupdates.EventRebootNeeded, + osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered"} { + if !operatorOnlyEvents[e] { + t.Errorf("%s is not operator-only", e) + } + } + if operatorOnlyEvents[osupdates.EventApplied] { + t.Errorf("%s is the household's line and must NOT be operator-only", osupdates.EventApplied) + } +} diff --git a/hub/internal/osupdates/alarms_test.go b/hub/internal/osupdates/alarms_test.go new file mode 100644 index 00000000..39c73ac1 --- /dev/null +++ b/hub/internal/osupdates/alarms_test.go @@ -0,0 +1,306 @@ +package osupdates + +import ( + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// ---------- the host layer (`11` §8 step 3) ---------- + +// The host release never carries a kernel, boot or firmware package (the host's slow lane, C3): the wrapper would +// refuse the whole plan (R14). Red-proof: drop the hostSlowRE check in candidate() and this fails. +func TestCandidate_HostLeavesOutKernelBootFirmware(t *testing.T) { + f := newFix(t) + set := []Package{pk("libssl3t64", "3.5.7-1~deb13u3"), pk("proxmox-kernel-6.14.11-4-pve-signed", "6.14.11-4"), + pk("grub-efi-amd64", "2.12-9"), pk("firmware-realtek", "20250410-2"), pk("intel-microcode", "3.20250512.1"), + pk("shim-signed", "1.46"), pk("pve-firmware", "3.16-3")} + f.reportL(t, "hp", LayerHost, "night", true, set...) + f.reportL(t, "n100", LayerHost, "night", true, set...) + cand, err := f.s.candidate(LayerHost, []string{"hp", "n100"}) + if err != nil { + t.Fatal(err) + } + if len(cand) != 1 || cand["libssl3t64"].Version != "3.5.7-1~deb13u3" { + t.Fatalf("host candidate = %+v, want only libssl3t64", cand) + } + // The guest layer has no such rule: a guest has no kernel of its own, nothing to leave out. + f.reportL(t, "hp", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1")) + f.reportL(t, "n100", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1")) + if g, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"}); len(g) != 1 { + t.Fatalf("guest candidate = %+v", g) + } +} + +// Host and guest are SEPARATE approved sets: a guest night run does not count toward a host approval, and each layer +// gets its own release id. Red-proof: count nights across layers (OSReportsSince without layer) and this fails. +func TestLayers_SeparateSets(t *testing.T) { + f := newFix(t) + g := []Package{pk("libc6", "2.41-12+deb13u4")} + h := []Package{pk("libssl3t64", "3.5.7-1~deb13u3")} + for _, b := range []string{"hp", "n100"} { + f.reportL(t, b, LayerGuest, "debug", true, g...) + f.reportL(t, b, LayerHost, "debug", true, h...) + } + f.s.Evaluate() + f.now = f.now.Add(25 * time.Hour) + for _, b := range []string{"hp", "n100"} { + f.reportL(t, b, LayerGuest, "night", true, g...) // a GUEST night only + } + sts, err := f.s.Evaluate() + if err != nil || len(sts) != 2 { + t.Fatalf("%v %+v", err, sts) + } + if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil || !strings.HasPrefix(gr.ID, "os-guest-") { + t.Fatalf("guest release = %+v", gr) + } + if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil { + t.Fatalf("host approved on a GUEST night: %+v", hr) + } + if !strings.Contains(sts[1].Waiting, "night run") || sts[1].Layer != LayerHost { + t.Fatalf("host status = %+v", sts[1]) + } + for _, b := range []string{"hp", "n100"} { + f.reportL(t, b, LayerHost, "night", true, h...) + } + f.s.Evaluate() + hr, _ := f.s.Store.LatestOSRelease(LayerHost) + if hr == nil || !strings.HasPrefix(hr.ID, "os-host-") { + t.Fatalf("host release = %+v", hr) + } + b := f.s.DesiredBlock("cust1") + if b.Release == nil || b.HostRelease == nil || b.HostRelease.Packages[0].Name != "libssl3t64" || b.Release.Packages[0].Name != "libc6" { + t.Fatalf("ring-1 block = %+v", b) + } +} + +// An unhealthy HOST run blocks only the host approval. The undo text points to the host runbook, not the guest backup. +func TestHostUnhealthy_BlocksHostOnly(t *testing.T) { + f := newFix(t) + for _, b := range []string{"hp", "n100"} { + f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1")) + f.reportL(t, b, LayerHost, "night", true, pk("libssl3t64", "1")) + } + f.s.Evaluate() + f.now = f.now.Add(25 * time.Hour) + var msgs []string + f.s.Emit = func(_, typ, _, msg, _, _ string) { f.events = append(f.events, typ); msgs = append(msgs, msg) } + for _, b := range []string{"hp", "n100"} { + f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1")) + } + f.reportL(t, "hp", LayerHost, "night", false, pk("libssl3t64", "1")) + f.reportL(t, "n100", LayerHost, "night", true, pk("libssl3t64", "1")) + f.s.Evaluate() + if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil { + t.Fatal("an unhealthy HOST run blocked the GUEST approval") + } + if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil { + t.Fatalf("host approved over an unhealthy host run: %+v", hr) + } + if !strings.Contains(strings.Join(msgs, "\n"), "os-updates-host-undo.md") { + t.Fatalf("host health mail does not name the host undo runbook: %q", msgs) + } +} + +// ---------- the four alarms (`11` §8.3) ---------- + +func (f *fix) count(typ string) int { + n := 0 + for _, e := range f.events { + if e == typ { + n++ + } + } + return n +} + +func (f *fix) setAgent(t *testing.T, host, ver string) { + t.Helper() + if err := f.s.Store.SaveHostReport(host, "c-"+host, []byte(`{}`), store.HostReportDenorm{AgentVersion: ver}); err != nil { + t.Fatal(err) + } +} + +func (f *fix) ingest(t *testing.T, host string, r Report) { + t.Helper() + if r.RunID == "" { + r.RunID = host + f.now.String() + } + if err := f.s.Ingest(host, r); err != nil { + t.Fatal(err) + } +} + +// Alarm 1: no completed OS leg for 7 days while the switch is ON, naming the likely reason; once a week at most; +// cleared by a good leg. Red-proof: compare against a fixed 70 days (or drop the Enabled check) and a sub-step fails. +func TestAlarm_StaleLeg(t *testing.T) { + f := newFix(t) + f.s.Now = func() time.Time { return f.now } + f.now = time.Now().UTC() + f.setAgent(t, "cust1", "0.141.0") + f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) + f.now = f.now.Add(6 * 24 * time.Hour) + if sent, _ := f.s.Alarms(); f.count(EventStale) != 0 { + t.Fatalf("alarm at 6 days: %v", sent) + } + f.now = f.now.Add(2 * 24 * time.Hour) + f.s.Alarms() + if f.count(EventStale) != 1 { + t.Fatalf("no alarm at 8 days: %v", f.events) + } + f.now = f.now.Add(time.Hour) + f.s.Alarms() + if f.count(EventStale) != 1 { + t.Fatal("re-alarmed within the week") + } + f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true}) + f.s.Alarms() + if raised := f.s.Store.OSAlarmRaised("stale:cust1"); !raised.IsZero() { + t.Fatal("a good leg did not clear the alarm") + } + // The switch OFF: the leg is not expected, no alarm. + _ = f.s.Store.SetOSEnabled("cust1", false) + f.now = f.now.Add(30 * 24 * time.Hour) + f.s.Alarms() + if f.count(EventStale) != 1 { + t.Fatal("alarmed with the switch OFF") + } +} + +// The stale alarm's reason: a failed leg is named; a box that runs an agent too old for the leg is not watched. +func TestAlarm_StaleNamesTheReason(t *testing.T) { + f := newFix(t) + var msgs []string + f.s.Emit = func(_, typ, _, msg, _, _ string) { + if typ == EventStale { + msgs = append(msgs, msg) + } + } + // The leg reports are dated 8 days back; the host reports (real clock) are fresh — the box IS reporting. + f.now = time.Now().UTC().Add(-8*24*time.Hour - time.Hour) + f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) + f.now = f.now.Add(time.Hour) + f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "refused", HealthReason: "R6 dpkg lock"}) + f.setAgent(t, "cust1", "0.141.0") + f.setAgent(t, "hp", "0.139.0") // too old to run the leg: never watched + f.now = time.Now().UTC() + f.s.Alarms() + if len(msgs) != 1 || !strings.Contains(msgs[0], "ended refused") || !strings.Contains(msgs[0], "Likely reason") { + t.Fatalf("stale alarms = %q", msgs) + } +} + +// Alarm 2: reboot needed for more than 14 days (host layer), from the FIRST report that said so. +// Red-proof: take `since` from the newest report instead of the oldest in the run and the 15-day step stays silent. +func TestAlarm_RebootNeeded(t *testing.T) { + f := newFix(t) + up := []Package{pk("libc6", "2")} + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true}) + for d := 1; d <= 13; d++ { + f.now = f.now.Add(24 * time.Hour) + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true}) + } + f.s.Alarms() + if f.count(EventRebootNeeded) != 0 { + t.Fatal("alarm at 13 days") + } + if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() || f.now.Sub(l.RebootNeededSince) != 13*24*time.Hour { + t.Fatalf("reboot-needed since = %v", l.RebootNeededSince) + } + f.now = f.now.Add(2 * 24 * time.Hour) + f.s.Alarms() + if f.count(EventRebootNeeded) != 1 { + t.Fatalf("no alarm at 15 days: %v", f.events) + } + // A guest-layer reboot flag never feeds the host alarm. + if l := f.s.layerLine("hp", LayerGuest, 0); !l.RebootNeededSince.IsZero() { + t.Fatal("guest line shows a reboot need") + } +} + +// Alarm 3: ring 0 approved nothing for 7 days while it has pending fast-lane updates. Pending SLOW-lane packages +// (a new kernel) do not count. Red-proof: count all pending (not fastPending) and the kernel-only step alarms. +func TestAlarm_Ring0Stalled(t *testing.T) { + f := newFix(t) + kernel := []PendingPkg{{Name: "proxmox-kernel-6.17", From: "", To: "6.17.1", Origin: []string{"Debian"}}} + for _, b := range []string{"hp", "n100"} { + f.ingest(t, b, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: kernel}) + } + f.now = f.now.Add(8 * 24 * time.Hour) + f.s.Alarms() + if f.count(EventRing0Stalled) != 0 { + t.Fatal("a pending KERNEL counted as a stalled fast lane") + } + fast := []PendingPkg{{Name: "libssl3t64", From: "1", To: "2", Origin: []string{"Debian-Security"}}} + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "failed", Healthy: false, Pending: fast}) + f.s.Alarms() + if f.count(EventRing0Stalled) != 1 { + t.Fatalf("no stall alarm: %v", f.events) + } + // A release clears it. + _ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-host-x", Layer: LayerHost, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", PackagesJSON: "[]"}) + f.s.Alarms() + if !f.s.Store.OSAlarmRaised("ring0stall:host").IsZero() { + t.Fatal("a new release did not clear the stall alarm") + } +} + +// Alarm 4: a ring-1 box with fast-lane packages no release covers for more than 14 days. Ring 0 never alarms. +// Red-proof: drop the ring==0 early return in notCoveredFast and the ring-0 step alarms. +func TestAlarm_NotCovered(t *testing.T) { + f := newFix(t) + odd := []PendingPkg{{Name: "libsomething-hw", From: "1", To: "2", Origin: []string{"Debian"}}} + _ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-guest-x", Layer: LayerGuest, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", + PackagesJSON: `[{"name":"libc6","version":"2","origin":"Debian"}]`}) + for d := 0; d <= 15; d++ { + f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd}) + f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd}) + if d == 13 { + f.s.Alarms() + if f.count(EventNotCovered) != 0 { + t.Fatal("alarm at 13 days") + } + } + f.now = f.now.Add(24 * time.Hour) + } + var msgs []string + f.s.Emit = func(_, typ, _, msg, _, _ string) { + f.events = append(f.events, typ) + if typ == EventNotCovered { + msgs = append(msgs, msg) + } + } + f.s.Alarms() + if len(msgs) != 1 || !strings.Contains(msgs[0], "cust1") || !strings.Contains(msgs[0], "libsomething-hw") { + t.Fatalf("not-covered alarms = %q (a ring-0 box must never raise one)", msgs) + } +} + +// The fleet view: one line per box, both layers, the tunnel. +func TestFleet_OneLinePerBoxBothLayers(t *testing.T) { + f := newFix(t) + if err := f.s.Store.SaveHostReport("hp", "c-hp", []byte(`{}`), store.HostReportDenorm{AgentVersion: "0.141.0", CloudflaredStatus: "running"}); err != nil { + t.Fatal(err) + } + f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: []Package{pk("a", "1")}, PassSeconds: 8.5}) + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, + Pending: []PendingPkg{{Name: "proxmox-kernel-6.17", Origin: []string{"Debian"}}}}) + lines, err := f.s.Fleet() + if err != nil || len(lines) != 3 { + t.Fatalf("%v %d lines", err, len(lines)) + } + for _, l := range lines { + if l.HostID != "hp" { + continue + } + if l.Tunnel != "running" || l.Guest.LastOutcome != "applied" || l.Guest.WrapperPassSeconds != 8.5 || + l.Host.LastOutcome != "nothing" || l.Host.Pending != 1 || l.Host.LastSuccessfulLeg.IsZero() { + t.Fatalf("hp line = %+v", l) + } + return + } + t.Fatal("no hp line") +} diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go index c0fd1d02..8be30a9d 100644 --- a/hub/internal/osupdates/service.go +++ b/hub/internal/osupdates/service.go @@ -1,14 +1,19 @@ -// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0). +// Package osupdates is the hub half of the OS-update lanes (`11-os-updates.md` §5.3, §8 steps 2–4). +// +// Two LAYERS, each with its own approved set (hub v0.131.0): the customer guest's Debian packages and the Proxmox +// host's Debian packages. A version approved for one layer is NOT approved for the other by that fact alone. // // Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed -// set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that -// package agree on. A candidate is APPROVED when, since it was first seen: -// - every ring-0 box runs it (its newest report matches the candidate for every package it has), -// - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and -// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs. +// set per layer (C9). Per layer, the CANDIDATE is every Debian-origin package=version that all ring-0 boxes having it +// agree on (the host set leaves out kernel / boot / firmware names — the host's slow lane, `11` C3). A candidate is +// APPROVED when, since it was first seen: +// - every ring-0 box runs it (its newest report of that layer matches the candidate for every package it has), +// - ApproveAfter (default 24 h) has passed with every ring-0 report of that layer healthy, and +// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs of that layer. // -// The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state -// and installs exactly those versions. Rules pinned by service_test.go. +// The operator can approve at once ("approve now"). Ring 1 gets each layer's newest release in its desired state. +// The fleet view and the four alarms (`11` §8 step 4) live here too. Rules pinned by service_test.go and +// alarms_test.go. package osupdates import ( @@ -17,21 +22,44 @@ import ( "encoding/json" "fmt" "log" + "regexp" "sort" "strings" "time" + "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) +// Layers. +const ( + LayerGuest = "guest" + LayerHost = "host" +) + +// Layers lists every layer the hub approves. +var Layers = []string{LayerGuest, LayerHost} + +// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware +// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14). +// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware. +var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`) + +// LegMinAgent is the first agent that runs the OS leg; the stale-leg alarm only watches boxes at or above it. +const LegMinAgent = "0.140.0" + // Event types — operator-only except EventApplied, the household's line (`11` §5.7). const ( EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed" EventFailed = "os_update_failed" // error, operator: the run failed or was refused - EventHealthFailed = "os_update_health_failed" // error, operator: the guest was not healthy after the run + EventHealthFailed = "os_update_health_failed" // error, operator: not healthy after the run EventReleaseApprove = "os_release_approved" // info, operator EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed + EventStale = "os_update_stale" // warning, operator: no successful OS leg for StaleAfter + EventRebootNeeded = "os_reboot_needed" // warning, operator: reboot needed for RebootAfter + EventRing0Stalled = "os_ring0_stalled" // error, operator: ring 0 approved nothing for Ring0StallAfter + EventNotCovered = "os_not_covered" // warning, operator: fast-lane packages no release covers ) // Package is one name=version with its origin ("Debian" | "Debian-Security"). @@ -41,11 +69,12 @@ type Package struct { Origin string `json:"origin"` } -// Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict). +// Report is what the agent POSTs after every run of a layer. type Report struct { RunID string `json:"run_id"` - Trigger string `json:"trigger"` // night | debug - Mode string `json:"mode"` // apply | inventory + Layer string `json:"layer"` // guest | host ("" from agent 0.140.0 = guest) + Trigger string `json:"trigger"` + Mode string `json:"mode"` Ring int `json:"ring"` ReleaseID string `json:"release_id"` Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed @@ -60,10 +89,10 @@ type Report struct { DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"` RebootNeeded bool `json:"reboot_needed,omitempty"` Refused json.RawMessage `json:"refused,omitempty"` - Log []string `json:"log,omitempty"` + PassSeconds float64 `json:"pass_seconds,omitempty"` } -// PendingPkg is one update the guest's sources offer. +// PendingPkg is one update the sources offer. type PendingPkg struct { Name string `json:"name"` From string `json:"from"` @@ -71,14 +100,38 @@ type PendingPkg struct { Origin []string `json:"origin"` } -// Block is what a box receives in its desired state (`os_update`). -type Block struct { - Ring int `json:"ring"` - Enabled bool `json:"enabled"` - Release *ReleaseBlock `json:"release,omitempty"` +func isFast(origins []string) bool { + if len(origins) == 0 { + return false + } + for _, o := range origins { + if o != "Debian" && o != "Debian-Security" { + return false + } + } + return true } -// ReleaseBlock is the newest approved release, for ring 1. +// fastPending is the pending updates a fast-lane release COULD carry for that layer. +func fastPending(layer string, p []PendingPkg) []PendingPkg { + var out []PendingPkg + for _, x := range p { + if isFast(x.Origin) && !(layer == LayerHost && hostSlowRE.MatchString(x.Name)) { + out = append(out, x) + } + } + return out +} + +// Block is what a box receives in its desired state (`os_update`). +type Block struct { + Ring int `json:"ring"` + Enabled bool `json:"enabled"` + Release *ReleaseBlock `json:"release,omitempty"` + HostRelease *ReleaseBlock `json:"host_release,omitempty"` +} + +// ReleaseBlock is a layer's newest approved release, for ring 1. type ReleaseBlock struct { ID string `json:"id"` Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org) @@ -88,12 +141,17 @@ type ReleaseBlock struct { // Service ties the store, the events and the clock together. type Service struct { Store *store.Store - Emit func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests + Emit func(customerID, eventType, severity, message, details, source string) ApproveAfter time.Duration NightsRequired int - Logger *log.Logger - Now func() time.Time - Bump func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests + // Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default. + StaleAfter time.Duration // 7 d + RebootAfter time.Duration // 14 d + Ring0StallAfter time.Duration // 7 d + NotCoveredAfter time.Duration // 14 d + Logger *log.Logger + Now func() time.Time + Bump func(hostID string) } func (s *Service) now() time.Time { @@ -109,6 +167,13 @@ func (s *Service) logf(f string, a ...any) { } } +func dflt(d, def time.Duration) time.Duration { + if d <= 0 { + return def + } + return d +} + func (s *Service) event(customerID, typ, sev, msg string, details any) { dj := "" if details != nil { @@ -124,58 +189,76 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) { } } +func layerOf(r Report) string { + if r.Layer == LayerHost { + return LayerHost + } + return LayerGuest +} + // Ingest stores a run and raises its events. The household gets one line per run that installed something. func (s *Service) Ingest(hostID string, r Report) error { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } + layer := layerOf(r) + r.Layer = layer raw, _ := json.Marshal(r) - if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome, - Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil { + if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, Layer: layer, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, + Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil { return err } - s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d", - hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)) - details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, + s.logf("[INFO] osupdates: %s reported %s run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d reboot-needed=%v wrapper=%.1fs", + hostID, layer, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.RebootNeeded, r.PassSeconds) + details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, "upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason} + where := "the box" + if layer == LayerHost { + where = "the box's base system" + } switch r.Outcome { case "applied", "health_failed": s.event(h.CustomerID, EventApplied, "info", - fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details) + fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details) if !r.Healthy || r.Outcome == "health_failed" { + undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)" + if layer == LayerHost { + undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)" + } s.event(h.CustomerID, EventHealthFailed, "error", - fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.", - hostID, len(r.Upgraded), r.HealthReason), details) + fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.", + layer, hostID, len(r.Upgraded), r.HealthReason, undo), details) } case "refused", "failed": s.event(h.CustomerID, EventFailed, "error", - fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) + fmt.Sprintf("OS update (%s) on %s %s: %s", layer, hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) } return nil } -// candidate derives the version set every ring-0 box agrees on, from each box's newest report. -func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) { - latest := map[string]*store.OSReport{} +// candidate derives, for one layer, the version set every ring-0 box agrees on, from each box's newest report. +func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, error) { byPkg := map[string]map[string]Package{} // name -> host -> pkg for _, h := range ring0 { - rep, err := s.Store.LatestOSReport(h) + rep, err := s.Store.LatestOSReport(h, layer) if err != nil { - return nil, nil, err + return nil, err } if rep == nil { - return nil, nil, nil // a ring-0 box that never reported: no candidate + return nil, nil // a ring-0 box that never reported this layer: no candidate } - latest[h] = rep var r Report if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil { - return nil, nil, err + return nil, err } for _, p := range r.Installed { if p.Origin != "Debian" && p.Origin != "Debian-Security" { continue } + if layer == LayerHost && hostSlowRE.MatchString(p.Name) { + continue + } if byPkg[p.Name] == nil { byPkg[p.Name] = map[string]Package{} } @@ -198,16 +281,17 @@ func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*sto cand[name] = pick } } - return cand, latest, nil + return cand, nil } -func fingerprint(c map[string]Package) (string, []Package) { +func fingerprint(layer string, c map[string]Package) (string, []Package) { var list []Package for _, p := range c { list = append(list, p) } sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) h := sha256.New() + fmt.Fprintf(h, "layer=%s\n", layer) for _, p := range list { fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version) } @@ -230,33 +314,45 @@ func (s *Service) ring0Hosts() ([]string, error) { return out, nil } -// Status explains the approval state (for the log and the fleet page). +// Status explains the approval state of one layer (for the log and the fleet page). type Status struct { + Layer string Fingerprint string FirstSeen time.Time Packages int Waiting string // why not approved yet ("" = approved or nothing to do) } -// Evaluate checks the approval rule and approves when it holds. Called every minute. -func (s *Service) Evaluate() (Status, error) { +// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute. +func (s *Service) Evaluate() ([]Status, error) { + var out []Status + for _, layer := range Layers { + st, err := s.evaluateLayer(layer) + if err != nil { + return out, err + } + out = append(out, st) + } + return out, nil +} + +func (s *Service) evaluateLayer(layer string) (Status, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { - return Status{Waiting: "no ring-0 box"}, err + return Status{Layer: layer, Waiting: "no ring-0 box"}, err } - cand, _, err := s.candidate(ring0) + cand, err := s.candidate(layer, ring0) if err != nil || cand == nil { - return Status{Waiting: "a ring-0 box has not reported"}, err + return Status{Layer: layer, Waiting: "a ring-0 box has not reported this layer"}, err } - fp, list := fingerprint(cand) + fp, list := fingerprint(layer, cand) pj, _ := json.Marshal(list) first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) if err != nil { return Status{}, err } - st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)} - if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { - st.Waiting = "" + st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)} + if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp { return st, nil // already approved } if age := s.now().Sub(first); age < s.ApproveAfter { @@ -264,7 +360,7 @@ func (s *Service) Evaluate() (Status, error) { return st, nil } for _, h := range ring0 { - reps, err := s.Store.OSReportsSince(h, first) + reps, err := s.Store.OSReportsSince(h, layer, first) if err != nil { return st, err } @@ -283,49 +379,54 @@ func (s *Service) Evaluate() (Status, error) { return st, nil } } - if err := s.approve(fp, list, "auto"); err != nil { - return st, err - } - return st, nil + return st, s.approve(layer, fp, list, "auto") } -// ApproveNow approves the current candidate at once (operator, urgent fix). +// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids. func (s *Service) ApproveNow() (string, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return "", fmt.Errorf("osupdates: no ring-0 box") } - cand, _, err := s.candidate(ring0) - if err != nil || cand == nil { + var ids []string + for _, layer := range Layers { + cand, err := s.candidate(layer, ring0) + if err != nil || cand == nil { + continue + } + fp, list := fingerprint(layer, cand) + pj, _ := json.Marshal(list) + if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil { + return "", err + } + if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp { + ids = append(ids, rel.ID) + continue + } + if err := s.approve(layer, fp, list, "operator"); err != nil { + return "", err + } + rel, _ := s.Store.LatestOSRelease(layer) + s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s (%s) at once (%d packages), without the wait.", rel.ID, layer, len(list)), + map[string]any{"release_id": rel.ID, "layer": layer, "packages": len(list)}) + ids = append(ids, rel.ID) + } + if len(ids) == 0 { return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet") } - fp, list := fingerprint(cand) - pj, _ := json.Marshal(list) - if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil { - return "", err - } - if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { - return rel.ID, nil - } - if err := s.approve(fp, list, "operator"); err != nil { - return "", err - } - rel, _ := s.Store.LatestOSRelease() - s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)), - map[string]any{"release_id": rel.ID, "packages": len(list)}) - return rel.ID, nil + return strings.Join(ids, ","), nil } -func (s *Service) approve(fp string, list []Package, by string) error { +func (s *Service) approve(layer, fp string, list []Package, by string) error { at := s.now().UTC().Truncate(time.Second) - id := "os-" + at.Format("20060102-150405") + id := "os-" + layer + "-" + at.Format("20060102-150405") pj, _ := json.Marshal(list) - if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil { + if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil { return err } - s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp) - s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)), - map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp}) + s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp) + s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)), + map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp}) if s.Bump != nil { hosts, _ := s.Store.ListHosts() for _, h := range hosts { @@ -337,17 +438,24 @@ func (s *Service) approve(fp string, list []Package, by string) error { return nil } +func (s *Service) releaseBlock(layer string) *ReleaseBlock { + rel, err := s.Store.LatestOSRelease(layer) + if err != nil || rel == nil { + return nil + } + var list []Package + if json.Unmarshal([]byte(rel.PackagesJSON), &list) != nil { + return nil + } + return &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list} +} + // DesiredBlock is the `os_update` block a box receives. func (s *Service) DesiredBlock(hostID string) Block { st := s.Store.GetOSHostSettings(hostID) b := Block{Ring: st.Ring, Enabled: st.Enabled} if st.Ring == 1 { - if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil { - var list []Package - if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil { - b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list} - } - } + b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost) } return b } @@ -387,17 +495,98 @@ func (s *Service) SetEnabled(hostID string, on bool) error { return nil } -// FleetLine is one box on the fleet page. +// LayerLine is one layer of one box on the fleet view. +type LayerLine struct { + ReleaseID string + LastOutcome string + LastAt time.Time + LastSuccessfulLeg time.Time + Pending int + NotCovered int + NotCoveredFast int // fast-lane packages no approved release covers (the alarm's input) + RestartNeeded int + RebootNeededSince time.Time // zero = not needed + WrapperPassSeconds float64 +} + +// FleetLine is one box on the fleet view. type FleetLine struct { - HostID string - Ring int - Enabled bool - ReleaseID string - LastOutcome string - LastAt time.Time - Pending int - NotCovered int - RestartNeeded int + HostID string + Ring int + Enabled bool + Tunnel string + Guest LayerLine + Host LayerLine +} + +func (s *Service) layerLine(hostID, layer string, ring int) LayerLine { + var l LayerLine + rep, _ := s.Store.LatestOSReport(hostID, layer) + if rep == nil { + return l + } + var r Report + _ = json.Unmarshal([]byte(rep.ReportJSON), &r) + l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt + l.Pending, l.NotCovered, l.RestartNeeded, l.WrapperPassSeconds = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.PassSeconds + if ok, _ := s.Store.LatestHealthyOSReport(hostID, layer, "applied", "nothing"); ok != nil { + l.LastSuccessfulLeg = ok.ReceivedAt + } + l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r)) + l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) { + if len(x.Upgraded) == 0 && x.Outcome != "applied" { + return false, false // a report without an install says nothing about reboot-needed: skip it + } + return x.RebootNeeded, true + }) + return l +} + +// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0 +// installs every fast-lane update itself, so it never has any. +func (s *Service) notCoveredFast(layer string, ring int, r Report) []string { + if ring == 0 { + return nil + } + named := map[string]bool{} + if rb := s.releaseBlock(layer); rb != nil { + for _, p := range rb.Packages { + named[p.Name] = true + } + } + var out []string + for _, p := range fastPending(layer, r.Pending) { + if !named[p.Name] { + out = append(out, p.Name) + } + } + return out +} + +// sinceContinuously walks the box's reports of a layer newest-first and returns the time of the OLDEST report in the +// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result +// false = the report does not count either way (skipped). +func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time { + reps, err := s.Store.OSReportsDesc(hostID, layer, 400) + if err != nil { + return time.Time{} + } + var since time.Time + for _, rep := range reps { + var r Report + if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil { + continue + } + hold, counts := pred(r) + if !counts { + continue + } + if !hold { + break + } + since = rep.ReceivedAt + } + return since } // Fleet lists every box. @@ -409,13 +598,12 @@ func (s *Service) Fleet() ([]FleetLine, error) { var out []FleetLine for _, h := range hosts { st := s.Store.GetOSHostSettings(h.HostID) - l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled} - if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil { - var r Report - _ = json.Unmarshal([]byte(rep.ReportJSON), &r) - l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt - l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded) + l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled, Tunnel: "unknown"} + if t, _ := s.Store.RecentCloudflaredStatuses(h.HostID, 1); len(t) == 1 && t[0] != "" { + l.Tunnel = t[0] } + l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring) + l.Host = s.layerLine(h.HostID, LayerHost, st.Ring) out = append(out, l) } return out, nil @@ -427,10 +615,137 @@ func (s *Service) FleetJSON() (any, error) { if err != nil { return nil, err } - rel, _ := s.Store.LatestOSRelease() out := map[string]any{"boxes": lines} - if rel != nil { - out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} + for _, layer := range Layers { + if rel, _ := s.Store.LatestOSRelease(layer); rel != nil { + out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} + } } return out, nil } + +// ---------------- the alarms (`11` §8.3) ---------------- + +// raise sends an alarm when its condition holds and it was not raised in the last week; clears it when it does not +// hold, so the next occurrence is announced again. Returns true when an event was sent. +func (s *Service) raise(key string, holds bool, customerID, typ, sev, msg string, details any) bool { + last := s.Store.OSAlarmRaised(key) + if !holds { + if !last.IsZero() { + _ = s.Store.SetOSAlarmRaised(key, time.Time{}) + } + return false + } + if !last.IsZero() && s.now().Sub(last) < 7*24*time.Hour { + return false + } + s.event(customerID, typ, sev, msg, details) + _ = s.Store.SetOSAlarmRaised(key, s.now()) + return true +} + +// Alarms evaluates the four OS-update alarms. Called hourly. Returns the event types it sent (for the log/tests). +func (s *Service) Alarms() ([]string, error) { + now := s.now() + stale, reboot, stall, nc := dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), + dflt(s.Ring0StallAfter, 7*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour) + hosts, err := s.Store.ListHosts() + if err != nil { + return nil, err + } + var sent []string + for _, h := range hosts { + st := s.Store.GetOSHostSettings(h.HostID) + // 1. No successful OS leg for `stale` while the switch is ON — the leg runs only after a good night backup, + // so a box with a broken backup silently stops updating. Only boxes whose agent CAN run the leg. + canRun := h.AgentVersion != "" && semver.Valid(h.AgentVersion) && semver.Compare(h.AgentVersion, LegMinAgent) >= 0 + var ref time.Time + if ok, _ := s.Store.LatestHealthyOSReport(h.HostID, LayerGuest, "applied", "nothing"); ok != nil { + ref = ok.ReceivedAt + } else if seen, _ := s.Store.HostAgentVersionsSeen(h.HostID); len(seen) > 0 { + for v, at := range seen { + if semver.Valid(v) && semver.Compare(v, LegMinAgent) >= 0 && (ref.IsZero() || at.Before(ref)) { + ref = at + } + } + } + holds := st.Enabled && canRun && !ref.IsZero() && now.Sub(ref) >= stale + if s.raise("stale:"+h.HostID, holds, h.CustomerID, EventStale, "warning", + fmt.Sprintf("OS updates: %s has not completed an OS update leg for %s (since %s). Likely reason: %s.", + h.HostID, now.Sub(ref).Round(time.Hour), ref.UTC().Format("2006-01-02 15:04"), s.staleReason(h)), + map[string]any{"host_id": h.HostID, "since": ref}) { + sent = append(sent, EventStale) + } + // 2. Reboot needed for `reboot` (host layer): a fix is installed but not fully in force. + since := s.layerLine(h.HostID, LayerHost, st.Ring).RebootNeededSince + if s.raise("reboot:"+h.HostID, !since.IsZero() && now.Sub(since) >= reboot, h.CustomerID, EventRebootNeeded, "warning", + fmt.Sprintf("OS updates (for information): %s has needed a reboot since %s — host fixes are installed but not fully in force (PID 1 or lxc-start still run the old libraries). Nothing reboots automatically until the kernel lane exists.", + h.HostID, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "since": since}) { + sent = append(sent, EventRebootNeeded) + } + // 4. Not-covered fast-lane packages for `nc` (ring 1: hardware ring 0 does not have, `11` edge case 9). + for _, layer := range Layers { + since := s.sinceContinuously(h.HostID, layer, func(r Report) (bool, bool) { + return len(s.notCoveredFast(layer, st.Ring, r)) > 0, true + }) + var names []string + if rep, _ := s.Store.LatestOSReport(h.HostID, layer); rep != nil { + var r Report + _ = json.Unmarshal([]byte(rep.ReportJSON), &r) + names = s.notCoveredFast(layer, st.Ring, r) + } + if s.raise("notcovered:"+layer+":"+h.HostID, !since.IsZero() && now.Sub(since) >= nc, h.CustomerID, EventNotCovered, "warning", + fmt.Sprintf("OS updates (%s): %s has had %d fast-lane package(s) no approved release covers since %s: %s. Ring 0 does not run them (other hardware?) — add matching hardware to ring 0, or approve by hand.", + layer, h.HostID, len(names), since.UTC().Format("2006-01-02"), strings.Join(names, ", ")), + map[string]any{"host_id": h.HostID, "layer": layer, "packages": names}) { + sent = append(sent, EventNotCovered) + } + } + } + // 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped + // getting fixes. + ring0, _ := s.ring0Hosts() + for _, layer := range Layers { + var last time.Time + if rel, _ := s.Store.LatestOSRelease(layer); rel != nil { + last = rel.ApprovedAt + } else if first, _ := s.Store.FirstOSReport(layer, ring0); first != nil { + last = first.ReceivedAt + } + pendingAny := 0 + for _, hID := range ring0 { + if rep, _ := s.Store.LatestOSReport(hID, layer); rep != nil { + var r Report + _ = json.Unmarshal([]byte(rep.ReportJSON), &r) + pendingAny += len(fastPending(layer, r.Pending)) + } + } + holds := len(ring0) > 0 && !last.IsZero() && now.Sub(last) >= stall && pendingAny > 0 + if s.raise("ring0stall:"+layer, holds, "", EventRing0Stalled, "error", + fmt.Sprintf("OS updates (%s): ring 0 has approved nothing since %s while it has %d pending fast-lane update(s). The demo boxes are broken or unhealthy, and the whole fleet has stopped getting fixes.", + layer, last.UTC().Format("2006-01-02 15:04"), pendingAny), map[string]any{"layer": layer, "pending": pendingAny}) { + sent = append(sent, EventRing0Stalled) + } + } + return sent, nil +} + +// staleReason names the likely reason a box stopped completing the OS leg (for the operator's mail). +func (s *Service) staleReason(h store.Host) string { + if h.LastReportAt == nil || s.now().Sub(*h.LastReportAt) > 24*time.Hour { + return "the box is off or not reporting (last host report " + fmtTime(h.LastReportAt) + ")" + } + if rep, _ := s.Store.LatestOSReport(h.HostID, LayerGuest); rep != nil && (rep.Outcome == "failed" || rep.Outcome == "refused" || rep.Outcome == "health_failed" || !rep.Healthy) { + var r Report + _ = json.Unmarshal([]byte(rep.ReportJSON), &r) + return fmt.Sprintf("the last OS leg ended %s (%s) on %s", rep.Outcome, strings.TrimSpace(r.HealthReason+" "+string(r.Refused)), rep.ReceivedAt.UTC().Format("2006-01-02")) + } + return "no successful whole-guest backup reached the OS leg (it runs only after one) — check the box's night backups" +} + +func fmtTime(t *time.Time) string { + if t == nil { + return "never" + } + return t.UTC().Format("2006-01-02 15:04") +} diff --git a/hub/internal/osupdates/service_test.go b/hub/internal/osupdates/service_test.go index 0308f6e5..4f05104b 100644 --- a/hub/internal/osupdates/service_test.go +++ b/hub/internal/osupdates/service_test.go @@ -42,12 +42,17 @@ func newFix(t *testing.T) *fix { func pk(name, ver string) Package { return Package{Name: name, Version: ver, Origin: "Debian"} } func (f *fix) report(t *testing.T, host, trigger string, healthy bool, pkgs ...Package) { + t.Helper() + f.reportL(t, host, LayerGuest, trigger, healthy, pkgs...) +} + +func (f *fix) reportL(t *testing.T, host, layer, trigger string, healthy bool, pkgs ...Package) { t.Helper() outcome := "applied" if !healthy { outcome = "health_failed" } - if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Trigger: trigger, Mode: "apply", Outcome: outcome, + if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Layer: layer, Trigger: trigger, Mode: "apply", Outcome: outcome, Healthy: healthy, Installed: pkgs, Upgraded: pkgs[:1]}); err != nil { t.Fatal(err) } @@ -60,12 +65,14 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) { set := []Package{pk("libc6", "2.41-12+deb13u4"), pk("openssl", "3.5.7-1~deb13u3")} f.report(t, "hp", "debug", true, set...) f.report(t, "n100", "debug", true, set...) - st, _ := f.s.Evaluate() + sts, _ := f.s.Evaluate() + st := sts[0] if !strings.HasPrefix(st.Waiting, "healthy for") { t.Fatalf("fresh set approved or wrong reason: %+v", st) } f.now = f.now.Add(25 * time.Hour) - st, _ = f.s.Evaluate() + sts, _ = f.s.Evaluate() + st = sts[0] if !strings.Contains(st.Waiting, "night run") { t.Fatalf("approved without a night run: %+v", st) } @@ -74,7 +81,7 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) { if _, err := f.s.Evaluate(); err != nil { t.Fatal(err) } - rel, _ := f.s.Store.LatestOSRelease() + rel, _ := f.s.Store.LatestOSRelease(LayerGuest) if rel == nil || rel.ApprovedBy != "auto" { t.Fatalf("not approved: %+v", rel) } @@ -85,6 +92,9 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) { if b.Ring != 1 || !b.Enabled || b.Release == nil || len(b.Release.Packages) != 2 || b.Release.Snapshot != rel.ApprovedAt.UTC().Format("20060102T150405Z") { t.Fatalf("ring-1 block = %+v", b) } + if b.HostRelease != nil { + t.Fatalf("a guest approval must not create a host release (separate sets): %+v", b.HostRelease) + } if hb := f.s.DesiredBlock("hp"); hb.Ring != 0 || hb.Release != nil { t.Fatalf("a ring-0 box must not get a release (it installs everything pending): %+v", hb) } @@ -99,8 +109,9 @@ func TestApproval_UnhealthyRunBlocks(t *testing.T) { f.now = f.now.Add(25 * time.Hour) f.report(t, "hp", "night", false, set...) f.report(t, "n100", "night", true, set...) - st, _ := f.s.Evaluate() - if rel, _ := f.s.Store.LatestOSRelease(); rel != nil { + sts, _ := f.s.Evaluate() + st := sts[0] + if rel, _ := f.s.Store.LatestOSRelease(LayerGuest); rel != nil { t.Fatalf("approved although a ring-0 run was not healthy: %+v", st) } if !strings.Contains(st.Waiting, "healthy=false") { @@ -120,7 +131,7 @@ func TestCandidate_DisagreementLeftOut(t *testing.T) { f := newFix(t) f.report(t, "hp", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u5")) f.report(t, "n100", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u4")) - cand, _, err := f.s.candidate([]string{"hp", "n100"}) + cand, err := f.s.candidate(LayerGuest, []string{"hp", "n100"}) if err != nil { t.Fatal(err) } @@ -135,7 +146,7 @@ func TestCandidate_OnlyDebianOrigins(t *testing.T) { d := Package{Name: "docker-ce", Version: "5:29.8.2", Origin: "Docker"} f.report(t, "hp", "night", true, pk("libc6", "x1"), d) f.report(t, "n100", "night", true, pk("libc6", "x1"), d) - cand, _, _ := f.s.candidate([]string{"hp", "n100"}) + cand, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"}) if _, ok := cand["docker-ce"]; ok { t.Fatal("a Docker package entered the candidate") } @@ -149,7 +160,7 @@ func TestApproveNow_IsAnOperatorEvent(t *testing.T) { if err != nil || id == "" { t.Fatalf("%q %v", id, err) } - rel, _ := f.s.Store.LatestOSRelease() + rel, _ := f.s.Store.LatestOSRelease(LayerGuest) if rel.ApprovedBy != "operator" { t.Fatalf("approved_by = %q", rel.ApprovedBy) } diff --git a/hub/internal/store/os_updates.go b/hub/internal/store/os_updates.go index 15988a22..1ac9cacc 100644 --- a/hub/internal/store/os_updates.go +++ b/hub/internal/store/os_updates.go @@ -2,6 +2,7 @@ package store import ( "database/sql" + "strings" "time" ) @@ -46,7 +47,13 @@ func (s *Store) migrateOSUpdates() error { packages_json TEXT NOT NULL ); `) - return err + if err != nil { + return err + } + // Host fast lane (hub v0.131.0): every report and release belongs to a LAYER; old rows are the guest's. + s.db.Exec(`ALTER TABLE os_reports ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`) + s.db.Exec(`ALTER TABLE os_releases ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`) + return nil } // OSHostSettings is one box's ring and switch. @@ -88,6 +95,7 @@ func (s *Store) SetOSEnabled(hostID string, on bool) error { type OSReport struct { ID int64 HostID string + Layer string // guest | host ReceivedAt time.Time Trigger string Mode string @@ -109,8 +117,12 @@ func (s *Store) SaveOSReport(r OSReport) (int64, error) { } // received_at comes from the CALLER's clock: the approval rule compares it with first_seen, which the // service stamps with its own clock — two clocks would make "since first seen" miss reports. - res, err := s.db.Exec(`INSERT INTO os_reports (host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - r.HostID, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON) + layer := r.Layer + if layer == "" { + layer = "guest" + } + res, err := s.db.Exec(`INSERT INTO os_reports (host_id, layer, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + r.HostID, layer, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON) if err != nil { return 0, err } @@ -124,7 +136,7 @@ func scanOSReports(rows *sql.Rows) ([]OSReport, error) { var r OSReport var at string var h int - if err := rows.Scan(&r.ID, &r.HostID, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil { + if err := rows.Scan(&r.ID, &r.HostID, &r.Layer, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil { return nil, err } r.ReceivedAt, r.Healthy = parseSQLiteTime(at), h == 1 @@ -133,11 +145,11 @@ func scanOSReports(rows *sql.Rows) ([]OSReport, error) { return out, rows.Err() } -const osReportCols = `id, host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json` +const osReportCols = `id, host_id, layer, received_at, trigger, mode, outcome, healthy, release_id, report_json` -// LatestOSReport returns the box's newest run, or nil. -func (s *Store) LatestOSReport(hostID string) (*OSReport, error) { - rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? ORDER BY id DESC LIMIT 1`, hostID) +// LatestOSReport returns the box's newest run for a layer, or nil. +func (s *Store) LatestOSReport(hostID, layer string) (*OSReport, error) { + rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? ORDER BY id DESC LIMIT 1`, hostID, layer) if err != nil { return nil, err } @@ -148,10 +160,10 @@ func (s *Store) LatestOSReport(hostID string) (*OSReport, error) { return &rs[0], nil } -// OSReportsSince returns the box's runs received at or after t, oldest first. -func (s *Store) OSReportsSince(hostID string, t time.Time) ([]OSReport, error) { - rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND received_at >= ? ORDER BY id`, - hostID, t.UTC().Format("2006-01-02 15:04:05")) +// OSReportsSince returns the box's runs of a layer received at or after t, oldest first. +func (s *Store) OSReportsSince(hostID, layer string, t time.Time) ([]OSReport, error) { + rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? AND received_at >= ? ORDER BY id`, + hostID, layer, t.UTC().Format("2006-01-02 15:04:05")) if err != nil { return nil, err } @@ -174,6 +186,7 @@ func (s *Store) OSCandidateFirstSeen(fingerprint, packagesJSON string, now time. // OSRelease is one approved version set. type OSRelease struct { ID string + Layer string Fingerprint string ApprovedAt time.Time ApprovedBy string @@ -182,17 +195,21 @@ type OSRelease struct { // SaveOSRelease stores an approved release. func (s *Store) SaveOSRelease(r OSRelease) error { - _, err := s.db.Exec(`INSERT INTO os_releases (id, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?)`, - r.ID, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON) + layer := r.Layer + if layer == "" { + layer = "guest" + } + _, err := s.db.Exec(`INSERT INTO os_releases (id, layer, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?, ?)`, + r.ID, layer, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON) return err } -// LatestOSRelease returns the newest approved release, or nil. -func (s *Store) LatestOSRelease() (*OSRelease, error) { +// LatestOSRelease returns the newest approved release of a layer, or nil. +func (s *Store) LatestOSRelease(layer string) (*OSRelease, error) { var r OSRelease var at string - err := s.db.QueryRow(`SELECT id, fingerprint, approved_at, approved_by, packages_json FROM os_releases ORDER BY approved_at DESC, id DESC LIMIT 1`). - Scan(&r.ID, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON) + err := s.db.QueryRow(`SELECT id, layer, fingerprint, approved_at, approved_by, packages_json FROM os_releases WHERE layer = ? ORDER BY approved_at DESC, id DESC LIMIT 1`, layer). + Scan(&r.ID, &r.Layer, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON) if err == sql.ErrNoRows { return nil, nil } @@ -209,3 +226,90 @@ func (s *Store) BackdateOSCandidateForTest(fingerprint string, by time.Duration) time.Now().Add(-by).UTC().Format("2006-01-02 15:04:05"), fingerprint) return err } + +// OSAlarmRaised returns when an OS alarm (by key) was last raised; zero = not raised / cleared (hub v0.131.0). +func (s *Store) OSAlarmRaised(key string) time.Time { + v := s.getSetting("os_alarm:" + key) + if v == "" { + return time.Time{} + } + t, _ := time.Parse(time.RFC3339, v) + return t +} + +// SetOSAlarmRaised records (t non-zero) or clears (zero t) an OS alarm. +func (s *Store) SetOSAlarmRaised(key string, t time.Time) error { + v := "" + if !t.IsZero() { + v = t.UTC().Format(time.RFC3339) + } + return s.setSetting("os_alarm:"+key, v) +} + +// HostAgentVersionsSeen returns, per agent version a host ever reported, the first time it was seen (the stale-leg +// alarm's start point for a box whose agent CAN run the OS leg but never reported one). +func (s *Store) HostAgentVersionsSeen(hostID string) (map[string]time.Time, error) { + rows, err := s.db.Query(`SELECT COALESCE(agent_version, ''), MIN(received_at) FROM host_reports WHERE host_id = ? GROUP BY agent_version`, hostID) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]time.Time{} + for rows.Next() { + var v, at string + if err := rows.Scan(&v, &at); err != nil { + return nil, err + } + out[v] = parseSQLiteTime(at) + } + return out, rows.Err() +} + +// LatestOSReportWhere returns the box's newest report of a layer whose outcome is one of outcomes and healthy, or nil. +func (s *Store) LatestHealthyOSReport(hostID, layer string, outcomes ...string) (*OSReport, error) { + q := `SELECT ` + osReportCols + ` FROM os_reports WHERE host_id = ? AND layer = ? AND healthy = 1 AND outcome IN (?` + + strings.Repeat(",?", len(outcomes)-1) + `) ORDER BY id DESC LIMIT 1` + args := []any{hostID, layer} + for _, o := range outcomes { + args = append(args, o) + } + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + rs, err := scanOSReports(rows) + if err != nil || len(rs) == 0 { + return nil, err + } + return &rs[0], nil +} + +// OSReportsDesc returns the box's newest n reports of a layer, newest first. +func (s *Store) OSReportsDesc(hostID, layer string, n int) ([]OSReport, error) { + rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? ORDER BY id DESC LIMIT ?`, hostID, layer, n) + if err != nil { + return nil, err + } + return scanOSReports(rows) +} + +// FirstOSReport returns the oldest report of a layer from any of the given hosts, or nil. +func (s *Store) FirstOSReport(layer string, hosts []string) (*OSReport, error) { + if len(hosts) == 0 { + return nil, nil + } + q := `SELECT ` + osReportCols + ` FROM os_reports WHERE layer = ? AND host_id IN (?` + strings.Repeat(",?", len(hosts)-1) + `) ORDER BY id LIMIT 1` + args := []any{layer} + for _, h := range hosts { + args = append(args, h) + } + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + rs, err := scanOSReports(rows) + if err != nil || len(rs) == 0 { + return nil, err + } + return &rs[0], nil +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 1693f106..f56bf43b 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -3442,6 +3442,24 @@ func (s *Store) GetHostDRBundle(hostID string) (*HostDRBundle, error) { return &b, nil } +// RecentCloudflaredStatuses returns the host's last n reported tunnel states, newest first (R-841). +func (s *Store) RecentCloudflaredStatuses(hostID string, n int) ([]string, error) { + rows, err := s.db.Query(`SELECT COALESCE(cloudflared_status, '') FROM host_reports WHERE host_id = ? ORDER BY id DESC LIMIT ?`, hostID, n) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var st string + if err := rows.Scan(&st); err != nil { + return nil, err + } + out = append(out, st) + } + return out, rows.Err() +} + // SaveHostReport inserts a host_reports row and bumps the host's reality columns // (agent_version/last_report_at/updated_at) — never the inert intent columns. func (s *Store) SaveHostReport(hostID, customerID string, reportJSON []byte, d HostReportDenorm) error { diff --git a/hub/internal/web/hosts.go b/hub/internal/web/hosts.go index b2503863..6fd66843 100644 --- a/hub/internal/web/hosts.go +++ b/hub/internal/web/hosts.go @@ -79,6 +79,7 @@ type hostVitals struct { MemoryPercent float64 DiskPercent float64 CloudflaredStatus string + CloudflaredDetail string // agent >= 0.141.0: why not_running (container state, exit code, readiness) } // parseHostVitals extracts the vitals block from a host-report body. A missing/malformed @@ -96,6 +97,7 @@ func parseHostVitals(reportJSON string) hostVitals { } `json:"host"` Cloudflared struct { Status string `json:"status"` + Detail string `json:"detail"` } `json:"cloudflared"` } if err := json.Unmarshal([]byte(reportJSON), &body); err != nil { @@ -105,6 +107,7 @@ func parseHostVitals(reportJSON string) hostVitals { v.MemoryPercent = body.Host.MemoryPercent v.DiskPercent = body.Host.DiskPercent v.CloudflaredStatus = body.Cloudflared.Status + v.CloudflaredDetail = body.Cloudflared.Detail return v } diff --git a/hub/internal/web/templates/host_detail_body.html b/hub/internal/web/templates/host_detail_body.html index e4ab00ea..eb99b92c 100644 --- a/hub/internal/web/templates/host_detail_body.html +++ b/hub/internal/web/templates/host_detail_body.html @@ -73,7 +73,7 @@
Cloudflared - {{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{else}}—{{end}} + {{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{if .Vitals.CloudflaredDetail}} ({{.Vitals.CloudflaredDetail}}){{end}}{{else}}—{{end}}
Guests diff --git a/hub/internal/web/templates/hosts.html b/hub/internal/web/templates/hosts.html index b8c9075f..68ae2a20 100644 --- a/hub/internal/web/templates/hosts.html +++ b/hub/internal/web/templates/hosts.html @@ -105,7 +105,7 @@ {{if .HasReport}}{{formatFloat .Vitals.CPUPercent}}%{{else}}—{{end}} {{if .HasReport}}{{formatFloat .Vitals.MemoryPercent}}%{{else}}—{{end}} {{if .HasReport}}{{formatFloat .Vitals.DiskPercent}}%{{else}}—{{end}} - {{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{else}}—{{end}} + {{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{if .Vitals.CloudflaredDetail}} ({{.Vitals.CloudflaredDetail}}){{end}}{{else}}—{{end}} {{if .HasStorage}}{{formatFloat .WorstFillPct}}% {{.WorstFillName}}{{else}}—{{end}} {{end}}