diff --git a/documentation/audits/os-docker-crash-2026-10-04/partA/hub-redproofs.txt b/documentation/audits/os-docker-crash-2026-10-04/partA/hub-redproofs.txt new file mode 100644 index 00000000..cc403223 --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partA/hub-redproofs.txt @@ -0,0 +1,31 @@ +# Hub v0.132.0 red-proofs, 2026-10-04T14:04:22Z +# First run: 2 of 10 NOT CAUGHT (the parser test never fed empty facts fields; the page test counted the word 'unknown' but not its amber class). Both tests strengthened; the rerun below catches all 10. +[CAUGHT] unknown, never empty (parser): internal/sysfacts/sysfacts.go mutated; TestParse_AbsentAndPartialAreUnknown ran=True failed=True + sysfacts_test.go:38: empty facts fields: {Present:true PVEVersion:x KernelVersion:unknown VMID:0 FactsError: ReadAt: Host:{Debian:unknown KernelRunning:unknown KernelNextBoot:unknown KernelNextBootSou +[CAUGHT] Docker never auto-approved: internal/osupdates/service.go mutated; TestDocker_NeverAutoApproved ran=True failed=True + docker_test.go:33: a Docker set was auto-approved: &{ID:os-docker-20261006-140000 Layer:docker Fingerprint:a4ec54235cdd17a5 ApprovedAt:2026-10-06 14:00:00 +0000 UTC ApprovedBy:auto PackagesJSON:[{"nam +[CAUGHT] Docker button needs 2 nights: internal/osupdates/service.go mutated; TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box ran=True failed=True + docker_test.go:45: approved after one night: +[CAUGHT] a Docker approval nudges no box: internal/osupdates/service.go mutated; TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box ran=True failed=True + docker_test.go:62: a Docker approval must nudge no box (ring 1 takes it by a signed job): [cust1] +[CAUGHT] crash event once per boot: internal/api/crash.go mutated; TestCheckCrash_EventsOncePerFact ran=True failed=True + crash_test.go:45: restart events: op=4 household=4 +[CAUGHT] first sighting is quiet: internal/api/crash.go mutated; TestCheckCrash_EventsOncePerFact ran=True failed=True + crash_test.go:39: first sighting flooded 1 events +[CAUGHT] page: unknown cell: internal/web/system.go mutated; TestSystemPage_FullPartialUnknown ran=True failed=True + system_test.go:65: an unknown value must be shown amber with its reason, got 0 +[CAUGHT] page: Docker button gate: internal/web/templates/system.html mutated; TestSystemPage_DockerButtonOnlyWhenReady ran=True failed=True + system_test.go:87: button shown after ONE night +[CAUGHT] buttons need the operator login: internal/web/server.go mutated; TestSystemButtons_NeedTheOperatorLogin ran=True failed=True + system_test.go:117: POST /os/ring/full-1 with auth "" = 303 — must be refused + system_test.go:117: POST /os/ring/full-1 with auth "Bearer k-full-1" = 303 — must be refused + system_test.go:117: POST /os/enabled/full-1 with auth "" = 303 — must be refused + system_test.go:117: POST /os/enabled/full-1 with auth "Bearer k-full-1" = 303 — must be refused + system_test.go:117: POST /os/approve-now with auth "" = 303 — must be refused + system_test.go:117: POST /os/approve-now with auth "Bearer k-full-1" = 303 — must be refused + system_test.go:117: POST /os/approve-docker with auth "" = 303 — must be refused + system_test.go:117: POST /os/approve-docker with auth "Bearer k-full-1" = 303 — must be refused + system_test.go:122: an unauthenticated POST changed a box +[CAUGHT] Hosts column: internal/web/templates/hosts.html mutated; TestHostsPage_ProxmoxKernelColumn ran=True failed=True + system_test.go:153: hosts column missing: +after restore: ok gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts 0.003s | ok gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.682s | ok gitea.dooplex.hu/admin/felhom-hub/internal/api 4.765s | ok gitea.dooplex.hu/admin/felhom-hub/internal/web 15.831s rc= 0 diff --git a/documentation/audits/os-docker-crash-2026-10-04/partA/hub_rp2.py b/documentation/audits/os-docker-crash-2026-10-04/partA/hub_rp2.py new file mode 100644 index 00000000..aab47987 --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partA/hub_rp2.py @@ -0,0 +1,54 @@ +import subprocess, shutil, sys +M = [ + ("unknown, never empty (parser)", "internal/sysfacts/sysfacts.go", + "out.Guest.Containerd, out.Guest.LiveRestore = orUnknown(out.Guest.Containerd), orUnknown(out.Guest.LiveRestore)", + "out.Guest.Containerd, out.Guest.LiveRestore = out.Guest.Containerd, out.Guest.LiveRestore", + "./internal/sysfacts/", "TestParse_AbsentAndPartialAreUnknown"), + ("Docker never auto-approved", "internal/osupdates/service.go", + "var Layers = []string{LayerGuest, LayerHost}", "var Layers = []string{LayerGuest, LayerHost, LayerDocker}", + "./internal/osupdates/", "TestDocker_NeverAutoApproved"), + ("Docker button needs 2 nights", "internal/osupdates/service.go", + "\tcase need == 0:\n\t\tneed = 2", "\tcase need == 0:\n\t\tneed = 1", + "./internal/osupdates/", "TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box"), + ("a Docker approval nudges no box", "internal/osupdates/service.go", + "if s.Bump != nil && layer != LayerDocker {", "if s.Bump != nil {", + "./internal/osupdates/", "TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box"), + ("crash event once per boot", "internal/api/crash.go", + "\t\tif known[b] {\n\t\t\tcontinue\n\t\t}", "\t\tif false {\n\t\t\tcontinue\n\t\t}", + "./internal/api/", "TestCheckCrash_EventsOncePerFact"), + ("first sighting is quiet", "internal/api/crash.go", + "\t\tif first {\n\t\t\treturn\n\t\t}\n\t\td := map", "\t\td := map", + "./internal/api/", "TestCheckCrash_EventsOncePerFact"), + ("page: unknown cell", "internal/web/system.go", + 'if s == "" || s == sysfacts.Unknown {', 'if false {', + "./internal/web/", "TestSystemPage_FullPartialUnknown"), + ("page: Docker button gate", "internal/web/templates/system.html", + '(not .Approved) (eq .Waiting "")}}', '(not .Approved)}}', + "./internal/web/", "TestSystemPage_DockerButtonOnlyWhenReady"), + ("buttons need the operator login", "internal/web/server.go", + 'if r.URL.Path == "/login" || isPublicBindPath(r.URL.Path) {', 'if r.URL.Path == "/login" || isPublicBindPath(r.URL.Path) || strings.HasPrefix(r.URL.Path, "/os/") {', + "./internal/web/", "TestSystemButtons_NeedTheOperatorLogin"), + ("Hosts column", "internal/web/templates/hosts.html", + "Proxmox / kernel", "Proxmox", + "./internal/web/", "TestHostsPage_ProxmoxKernelColumn"), +] +ok = True +for name, f, old, new, pkg, test in M: + src = open(f).read() + assert src.count(old) == 1, (name, src.count(old)) + shutil.copy(f, f + ".bak") + open(f, "w").write(src.replace(old, new)) + r = subprocess.run(["go", "test", pkg, "-run", "^" + test + "$", "-v", "-count=1"], capture_output=True, text=True) + shutil.move(f + ".bak", f) + out = r.stdout + r.stderr + ran = ("=== RUN " + test) in out + failed = ("--- FAIL: " + test) in out + v = "CAUGHT" if ran and failed else "NOT CAUGHT" + ok &= v == "CAUGHT" + print(f"[{v}] {name}: {f} mutated; {test} ran={ran} failed={failed}") + for l in out.splitlines(): + if "_test.go:" in l or "build failed" in l or l.startswith("#"): + print(" " + l.strip()[:200]) +r = subprocess.run(["go", "test", "./internal/sysfacts/", "./internal/osupdates/", "./internal/api/", "./internal/web/", "-count=1"], capture_output=True, text=True) +print("after restore:", r.stdout.strip().replace("\n", " | "), "rc=", r.returncode) +sys.exit(0 if ok and r.returncode == 0 else 1) diff --git a/documentation/audits/os-docker-crash-2026-10-04/partA/live/facts-demo-hp-rc1.txt b/documentation/audits/os-docker-crash-2026-10-04/partA/live/facts-demo-hp-rc1.txt new file mode 100644 index 00000000..790d99ee --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partA/live/facts-demo-hp-rc1.txt @@ -0,0 +1,39 @@ + --- facts (vmid 9201, 2.1s) --- + { + "guest": { + "containerd": "2.3.3-1~debian.13~trixie", + "debian": "13.7", + "docker_engine": "29.7.2", + "live_restore": "on" + }, + "host": { + "crash_guard": { + "armed": true, + "boot_id": "0defc390-c295-4e9f-aa6e-13d69a1f24c3", + "config": { + "LIMIT": 3, + "PANIC_SECONDS": 10, + "REARM_HOURS": 24, + "WINDOW_MINUTES": 60 + }, + "kernel_panic": 10, + "last_boot_at": "2026-10-04T13:39:06Z", + "last_boot_unclean": false, + "tripped": false, + "unclean_boots": [], + "unclean_boots_24h": 0, + "unclean_boots_in_window": 0, + "updated_at": "2026-10-04T14:08:23Z", + "version": 1 + }, + "debian": "13.7", + "held": [], + "kernel_next_boot": "7.0.14-20-pve", + "kernel_next_boot_source": "saved default", + "kernel_panic": 10, + "kernel_running": "7.0.14-20-pve", + "oops_this_boot": false, + "tainted": 4097, + "warn_this_boot": false + } + } diff --git a/documentation/audits/os-docker-crash-2026-10-04/partB/agent-redproofs.txt b/documentation/audits/os-docker-crash-2026-10-04/partB/agent-redproofs.txt new file mode 100644 index 00000000..4db25bed --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partB/agent-redproofs.txt @@ -0,0 +1,106 @@ +# Agent v0.142.0 red-proofs, 2026-10-04T13:54:18Z +[CAUGHT] docker pkg in a fast plan refused (R2): configs/felhom-os-apply mutated; DockerLane.test_docker_package_in_a_fast_plan_is_refused ran=True rc=1 + test_docker_package_in_a_fast_plan_is_refused (__main__.DockerLane.test_docker_package_in_a_fast_plan_is_refused) ... FAIL + FAIL: test_docker_package_in_a_fast_plan_is_refused (__main__.DockerLane.test_docker_package_in_a_fast_plan_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'running'}, 'felhom-controller': {'health': 'healthy', ' + FAILED (failures=1) +[CAUGHT] slow lane without authority refused (R3): configs/felhom-os-apply mutated; DockerLane.test_no_authority_is_refused ran=True rc=1 + test_no_authority_is_refused (__main__.DockerLane.test_no_authority_is_refused) ... FAIL + FAIL: test_no_authority_is_refused (__main__.DockerLane.test_no_authority_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'authority': 'ring0', 'docker_engine': '29.7.2', 'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'runnin + FAILED (failures=1) +[CAUGHT] bad signature refused (R3): configs/felhom-os-apply mutated; DockerLane.test_bad_signature_is_refused ran=True rc=1 + test_bad_signature_is_refused (__main__.DockerLane.test_bad_signature_is_refused) ... FAIL + FAIL: test_bad_signature_is_refused (__main__.DockerLane.test_bad_signature_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'authority': 'signed', 'docker_engine': '29.7.2', 'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'runni + FAILED (failures=1) +[CAUGHT] replay refused (R3): configs/felhom-os-apply mutated; DockerLane.test_replayed_signed_job_is_refused ran=True rc=1 + test_replayed_signed_job_is_refused (__main__.DockerLane.test_replayed_signed_job_is_refused) ... FAIL + FAIL: test_replayed_signed_job_is_refused (__main__.DockerLane.test_replayed_signed_job_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'authority': 'signed', 'docker_engine': '29.7.2', 'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'runni + FAILED (failures=1) +[CAUGHT] agent-writable trust file refused (R3): configs/felhom-os-apply mutated; DockerLane.test_agent_writable_trust_file_is_refused ran=True rc=1 + test_agent_writable_trust_file_is_refused (__main__.DockerLane.test_agent_writable_trust_file_is_refused) ... FAIL + FAIL: test_agent_writable_trust_file_is_refused (__main__.DockerLane.test_agent_writable_trust_file_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'authority': 'ring0', 'docker_engine': '29.7.2', 'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'runnin + FAILED (failures=1) +[CAUGHT] live-restore off refused (R15): configs/felhom-os-apply mutated; DockerLane.test_live_restore_off_is_refused ran=True rc=1 + test_live_restore_off_is_refused (__main__.DockerLane.test_live_restore_off_is_refused) ... FAIL + FAIL: test_live_restore_off_is_refused (__main__.DockerLane.test_live_restore_off_is_refused) + self.assertEqual(rc, 2, rep) + AssertionError: 0 != 2 : {'authority': 'signed', 'docker_engine': '29.7.2', 'docker_restart_needed': False, 'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'runni + FAILED (failures=1) +[CAUGHT] live-restore uses reload, never restart: configs/felhom-os-apply mutated; LiveRestore.test_turns_it_on_with_a_reload_never_a_restart ran=True rc=1 + test_turns_it_on_with_a_reload_never_a_restart (__main__.LiveRestore.test_turns_it_on_with_a_reload_never_a_restart) ... FAIL + FAIL: test_turns_it_on_with_a_reload_never_a_restart (__main__.LiveRestore.test_turns_it_on_with_a_reload_never_a_restart) + self.assertEqual(rc, 0, rep) + AssertionError: 3 != 0 : {'failed': {'rc': 3, 'reason': '', 'step': 'live-restore'}, 'layer': 'guest', 'live_restore': {'containers_before': 2, 'reload_rc': 0, 'result': 'failed', 'same_ids': True}, ' + FAILED (failures=1) +[CAUGHT] guest scanned every pass (R-849): configs/felhom-os-apply mutated; HostLayer.test_every_layer_scans_every_pass ran=True rc=1 + test_every_layer_scans_every_pass (__main__.HostLayer.test_every_layer_scans_every_pass) ... FAIL + FAIL: test_every_layer_scans_every_pass (__main__.HostLayer.test_every_layer_scans_every_pass) + self.assertTrue(rep["reboot_scanned"] and rep["reboot_needed"], (layer, rep)) + AssertionError: False is not true : ('guest', {'health_after': {'containers': {'app': {'health': 'healthy', 'id': 'bbb222', 'state': 'running'}, 'felhom-controller': {'health': 'healthy', 'id': 'aaa11 + FAILED (failures=1) +[CAUGHT] facts: next-boot by dpkg order: configs/felhom-os-apply mutated; Facts.test_next_entry_wins_and_default_zero_is_the_newest ran=True rc=1 + test_next_entry_wins_and_default_zero_is_the_newest (__main__.Facts.test_next_entry_wins_and_default_zero_is_the_newest) ... FAIL + FAIL: test_next_entry_wins_and_default_zero_is_the_newest (__main__.Facts.test_next_entry_wins_and_default_zero_is_the_newest) + self.assertEqual(fa["host"]["kernel_next_boot"], "7.0.14-20-pve", "dpkg order, not string order (7.0.2 < 7.0.14)") + AssertionError: '7.0.2-6-pve' != '7.0.14-20-pve' + FAILED (failures=1) +[CAUGHT] crash guard: 3rd crash leaves the box off: configs/felhom-crash-guard mutated; Guard.test_third_crash_in_an_hour_leaves_the_box_off ran=True rc=1 + FAIL + FAIL: test_third_crash_in_an_hour_leaves_the_box_off (__main__.Guard.test_third_crash_in_an_hour_leaves_the_box_off) + self.assertTrue(s["tripped"], s) + AssertionError: False is not true : {'armed': True, 'boot_id': 'unknown', 'config': {'LIMIT': 3, 'PANIC_SECONDS': 10, 'REARM_HOURS': 24, 'WINDOW_MINUTES': 60}, 'kernel_panic': 10, 'last_boot_at': '202 + FAILED (failures=1) +[CAUGHT] crash guard: clean reboots never count: configs/felhom-crash-guard mutated; Guard.test_clean_reboots_never_count ran=True rc=1 + FAIL + FAIL: test_clean_reboots_never_count (__main__.Guard.test_clean_reboots_never_count) + self.assertEqual(s["unclean_boots_in_window"], 0) + AssertionError: 5 != 0 + FAILED (failures=1) +[CAUGHT] crash guard: 24 h re-arm not earlier: configs/felhom-crash-guard mutated; Guard.test_rearms_after_24h_of_normal_running ran=True rc=1 + FAIL + FAIL: test_rearms_after_24h_of_normal_running (__main__.Guard.test_rearms_after_24h_of_normal_running) + self.assertTrue(self.e.state()["tripped"], "not before 24 h") + AssertionError: False is not true : not before 24 h + FAILED (failures=1) +[CAUGHT] docker health: changed id fails: internal/osupdate/leg.go mutated; TestDockerHealthVerdict ran=True failed=True + leg_test.go:442: a changed container id passed — live-restore failed and the apps restarted + --- FAIL: TestDockerHealthVerdict (0.00s) + FAIL + FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.008s + FAIL +[CAUGHT] docker health: changed id -> health_failed: internal/osupdate/leg.go mutated; TestDocker_ChangedIDIsHealthFailed ran=True failed=True + leg_test.go:465: docker = {RunID:20261004T040000Z Layer:docker Trigger:night Mode:apply Ring:0 ReleaseID:ring0-20261004T040000Z Outcome:applied Healthy:true HealthReason: VMID:9201 Upgraded:[{Name:doc + --- FAIL: TestDocker_ChangedIDIsHealthFailed (0.00s) + FAIL + FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.011s + FAIL +[CAUGHT] docker health: engine must move: internal/osupdate/leg.go mutated; TestDockerHealthVerdict ran=True failed=True + leg_test.go:445: the engine did not move and the step passed + --- FAIL: TestDockerHealthVerdict (0.00s) + FAIL + FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.007s + FAIL +[CAUGHT] ring 1 never steps in the night leg: internal/osupdate/leg.go mutated; TestDocker_Ring1NightLegNeverSteps ran=True failed=True + leg_test.go:390: ring 1 took a docker step: guest:inventory,host:inventory,guest:live-restore-on,docker:apply + --- FAIL: TestDocker_Ring1NightLegNeverSteps (0.00s) + FAIL + FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.009s + FAIL +after restore: ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.628s rc=0 + +[executor] signed envelope reaches the plan — mutation: drop plan["signed"] +=== RUN TestDockerStepExecutor_PassesTheSignedEnvelope + dockerjob_test.go:31: docker plan = map[lane:slow layer:docker mode:apply packages:[map[name:docker-ce origin:Docker CE version:5:29.8.2-1~debian.13~trixie]] release_id:os-docker-1 select:listed s +--- FAIL: TestDockerStepExecutor_PassesTheSignedEnvelope (0.00s) +FAIL +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.009s +FAIL diff --git a/documentation/audits/os-docker-crash-2026-10-04/partB/b1-live-restore-9202.txt b/documentation/audits/os-docker-crash-2026-10-04/partB/b1-live-restore-9202.txt new file mode 100644 index 00000000..9479f4af --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partB/b1-live-restore-9202.txt @@ -0,0 +1,80 @@ ++ pct exec 9202 -- docker ps --no-trunc --format '{{.ID}} {{.Names}} {{.Status}}' ++ sort ++ cat /root/9202-ids-before.txt ++ cut -c1-12,65- +1b8dc9032f36 paperless-webserver Up 27 minutes (healthy) +2d6019eb0042 paperless-redis Up 27 minutes (healthy) +3f8f9651ee32 felhom-controller Up 27 minutes (healthy) +468aa784345b paperless-postgres Up 27 minutes (healthy) +7504fa621b39 filebrowser Up 27 minutes (healthy) +8d8778d4dcf8 traefik Up 27 minutes ++ pct exec 9202 -- docker info --format 'live={{.LiveRestoreEnabled}}' +live=false ++ pct exec 9202 -- cat /etc/docker/daemon.json +{ + "features": { "containerd-snapshotter": false }, + "log-driver": "json-file", + "log-opts": { "max-size": "10m", "max-file": "3" } +} +++ date +%s.%N ++ S=1791122916.729550605 ++ sudo -u felhom-agent /usr/local/bin/felhom-agent-0.142.0-rc1 --config /etc/felhom-agent/agent.json --selftest=live-restore -vmid 9202 ++ grep -E 'LIVE-RESTORE|live-restore|error|refused' +selftest=live-restore: refused: {"code": "R10", "reason": "vmid 9202 does not bind /mnt/felhom-drives \u2014 it is not this box's customer guest"} +++ bc ++++ date +%s.%N +++ echo 1791122916.827586880-1791122916.729550605 ++ echo WALL=.098036275 +WALL=.098036275 ++ pct exec 9202 -- docker ps --no-trunc --format '{{.ID}} {{.Names}} {{.Status}}' ++ sort ++ cut '-d ' -f1,2 /root/9202-ids-before.txt ++ cut '-d ' -f1,2 /root/9202-ids-after.txt ++ diff /tmp/a /tmp/b +++ wc -l +IDS IDENTICAL (6 containers) ++ echo 'IDS IDENTICAL (6 containers)' ++ pct exec 9202 -- docker info --format 'live={{.LiveRestoreEnabled}}' +live=false ++ pct exec 9202 -- cat /etc/docker/daemon.json +{ + "features": { "containerd-snapshotter": false }, + "log-driver": "json-file", + "log-opts": { "max-size": "10m", "max-file": "3" } +} ++ pct exec 9202 -- systemctl show docker -p ActiveEnterTimestamp -p NRestarts +NRestarts=0 +ActiveEnterTimestamp=Sun 2026-10-04 13:40:57 UTC ++ pct exec 9202 -- cp -p /etc/docker/daemon.json /root/daemon.json.bak-2026-10-04 ++ pct exec 9202 -- python3 -c ' +import json; p="/etc/docker/daemon.json"; d=json.load(open(p)); d["live-restore"]=True; open(p,"w").write(json.dumps(d,indent=2,sort_keys=True)+"\n")' +++ date +%s.%N ++ S=1791122938.796302354 ++ pct exec 9202 -- systemctl reload docker +++ bc ++++ date +%s.%N +++ echo 1791122939.730342275-1791122938.796302354 +reload_rc=0 took=.934039921 ++ echo 'reload_rc=0 took=.934039921' ++ sleep 3 ++ pct exec 9202 -- docker info --format 'live={{.LiveRestoreEnabled}}' +live=true ++ pct exec 9202 -- docker ps --no-trunc --format '{{.ID}} {{.Names}} {{.Status}}' ++ sort ++ cut '-d ' -f1,2 /root/9202-ids-before.txt ++ cut '-d ' -f1,2 /root/9202-ids-after.txt ++ diff /tmp/a /tmp/b +++ wc -l +IDS IDENTICAL (6 containers) ++ echo 'IDS IDENTICAL (6 containers)' ++ cut -c1-12,65- /root/9202-ids-after.txt +1b8dc9032f36 paperless-webserver Up 28 minutes (healthy) +2d6019eb0042 paperless-redis Up 28 minutes (healthy) +3f8f9651ee32 felhom-controller Up 28 minutes (healthy) +468aa784345b paperless-postgres Up 28 minutes (healthy) +7504fa621b39 filebrowser Up 28 minutes (healthy) +8d8778d4dcf8 traefik Up 28 minutes ++ pct exec 9202 -- systemctl show docker -p ActiveEnterTimestamp -p NRestarts -p ExecMainPID +NRestarts=0 +ExecMainPID=227 +ActiveEnterTimestamp=Sun 2026-10-04 13:40:57 UTC diff --git a/documentation/audits/os-docker-crash-2026-10-04/partB/b2-live-restore-demo-hp-9201.txt b/documentation/audits/os-docker-crash-2026-10-04/partB/b2-live-restore-demo-hp-9201.txt new file mode 100644 index 00000000..ed30004f --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partB/b2-live-restore-demo-hp-9201.txt @@ -0,0 +1,7 @@ +before: 24 containers, live=false +os-apply: LIVE-RESTORE reload_rc=0 state=true containers=24 same-ids=yes +live-restore: on (see the wrapper's LIVE-RESTORE line above for the container ids) +WALL=7.532254689 +IDS IDENTICAL (24 containers) +after: live=true +NRestarts=0 ActiveEnterTimestamp=Sun 2026-10-04 13:41:02 UTC \ No newline at end of file diff --git a/documentation/audits/os-docker-crash-2026-10-04/partB/b3-ring0-docker-demo-hp.log b/documentation/audits/os-docker-crash-2026-10-04/partB/b3-ring0-docker-demo-hp.log new file mode 100644 index 00000000..561f80a2 --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partB/b3-ring0-docker-demo-hp.log @@ -0,0 +1,202 @@ +=== felhom-agent 0.142.0-rc1 selftest=os-update vmid=9201 ring=0 enabled=true guest-release=false host-release=false appliance=true === +time=2026-10-04T16:09:47.878+02:00 level=INFO msg="osupdate: START" run=20261004T140947Z layer=guest vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T140947Z +time=2026-10-04T16:10:04.518+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T140947Z layer=guest:9201 lane=fast mode=apply select=pending-fast packages=0" +time=2026-10-04T16:10:04.518+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:10:04.518+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=0 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:10:04.518+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)" +time=2026-10-04T16:10:04.519+02:00 level=INFO msg="osupdate: DONE" run=20261004T140947Z layer=guest vmid=9201 ring=0 trigger=debug outcome=nothing healthy=true reason="" upgraded=0 pending=6 not_covered=6 restart_needed=0 reboot_needed=false wrapper_seconds=16.6 +time=2026-10-04T16:10:04.530+02:00 level=INFO msg="osupdate: START" run=20261004T140947Z layer=host vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T140947Z +time=2026-10-04T16:10:19.645+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T140947Z layer=host lane=fast mode=apply select=pending-fast packages=0" +time=2026-10-04T16:10:19.645+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:10:19.645+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=0 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:10:19.645+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)" +time=2026-10-04T16:10:20.691+02:00 level=INFO msg="osupdate: DONE" run=20261004T140947Z layer=host vmid=9201 ring=0 trigger=debug outcome=nothing healthy=true reason="" upgraded=0 pending=78 not_covered=78 restart_needed=0 reboot_needed=false wrapper_seconds=15 +time=2026-10-04T16:10:22.649+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: LIVE-RESTORE already on" +time=2026-10-04T16:10:22.649+02:00 level=INFO msg="osupdate: live-restore" vmid=9201 result="{\"result\": \"already on\"}" +time=2026-10-04T16:10:22.649+02:00 level=INFO msg="osupdate: START" run=20261004T140947Z layer=docker vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T140947Z +time=2026-10-04T16:11:14.270+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T140947Z layer=docker:9201 lane=slow mode=apply select=pending-docker packages=0 authority=ring0" +time=2026-10-04T16:11:14.270+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:11:14.270+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=6 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:11:14.271+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=24.4 upgraded=6 restart-needed=containerd-shim reboot-needed=no" +time=2026-10-04T16:11:50.070+02:00 level=INFO msg="osupdate: DONE" run=20261004T140947Z layer=docker vmid=9201 ring=0 trigger=debug outcome=applied healthy=true reason="" upgraded=6 pending=0 not_covered=0 restart_needed=1 reboot_needed=false wrapper_seconds=51.5 + --- os-update report (guest) --- + { + "authority": "", + "docker_engine": "", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": [ + "docker-ce-cli", + "containerd.io", + "docker-ce", + "docker-buildx-plugin", + "docker-ce-rootless-extras", + "docker-compose-plugin" + ], + "outcome": "nothing", + "pending": 6, + "reboot_needed": false, + "refused": null, + "release_id": "ring0-20261004T140947Z", + "restart_needed": [], + "ring": 0, + "run_id": "20261004T140947Z", + "upgraded": [], + "wrapper_seconds": 16.6 + } + --- os-update report (host) --- + { + "authority": "", + "docker_engine": "", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": [ + "frr", + "shim-signed-common", + "proxmox-secure-boot-support", + "shim-unsigned", + "shim-helpers-amd64-signed", + "shim-signed", + "amd64-microcode", + "libradosstriper1", + "librgw2", + "ceph-common", + "librbd1", + "librados2", + "python3-cephfs", + "libcephfs2", + "python3-rgw", + "python3-rados", + "python3-ceph-argparse", + "python3-ceph-common", + "python3-rbd", + "ceph-fuse", + "chrony", + "libcorosync-common4", + "libcfg7", + "libcmap4", + "libcpg4", + "libknet1t64", + "libnozzle1t64", + "libquorum5", + "libvotequorum8", + "corosync", + "frr-pythontools", + "libjs-extjs", + "libnvpair3linux", + "libproxmox-acme-plugins", + "libproxmox-backup-qemu0", + "pve-qemu-kvm", + "libpve-notify-perl", + "libpve-cluster-api-perl", + "libpve-cluster-perl", + "pve-cluster", + "libpve-access-control", + "libpve-apiclient-perl", + "librados2-perl", + "proxmox-backup-client", + "proxmox-backup-file-restore", + "pve-manager", + "libproxmox-acme-perl", + "libpve-common-perl", + "libpve-guest-common-perl", + "qemu-server", + "libpve-storage-perl", + "pve-edk2-firmware-legacy", + "pve-edk2-firmware-ovmf", + "libpve-network-api-perl", + "libpve-network-perl", + "proxmox-firewall-data", + "pve-firewall", + "pve-container", + "pve-ha-manager", + "novnc-pve", + "proxmox-enterprise-support-keyring", + "proxmox-mini-journalreader", + "proxmox-widget-toolkit", + "pve-docs", + "pve-i18n", + "pve-xtermjs", + "pve-yew-mobile-i18n", + "pve-yew-mobile-gui", + "libuutil3linux", + "libzfs7linux", + "libzpool7linux", + "proxmox-kernel-helper", + "pve-edk2-firmware-aarch64", + "pve-edk2-firmware", + "pve-firmware", + "zfs-initramfs", + "zfsutils-linux", + "zfs-zed" + ], + "outcome": "nothing", + "pending": 78, + "reboot_needed": false, + "refused": null, + "release_id": "ring0-20261004T140947Z", + "restart_needed": [], + "ring": 0, + "run_id": "20261004T140947Z", + "upgraded": [], + "wrapper_seconds": 15 + } + --- os-update report (docker) --- + { + "authority": "ring0", + "docker_engine": "29.8.2", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": null, + "outcome": "applied", + "pending": 0, + "reboot_needed": false, + "refused": null, + "release_id": "ring0-20261004T140947Z", + "restart_needed": [ + "containerd-shim" + ], + "ring": 0, + "run_id": "20261004T140947Z", + "upgraded": [ + { + "name": "docker-ce-cli", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "containerd.io", + "version": "2.3.6-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-ce", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-buildx-plugin", + "version": "0.37.1-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-ce-rootless-extras", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-compose-plugin", + "version": "5.6.0-1~debian.13~trixie", + "origin": "" + } + ], + "wrapper_seconds": 51.5 + } + pass took 2m2.2s +WALL=122.327970106 +IDS IDENTICAL (24 containers) +engine=29.8.2 +live=true diff --git a/documentation/audits/os-docker-crash-2026-10-04/partB/b4-ring0-docker-demo-felhom.log b/documentation/audits/os-docker-crash-2026-10-04/partB/b4-ring0-docker-demo-felhom.log new file mode 100644 index 00000000..1b3cf4b5 --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partB/b4-ring0-docker-demo-felhom.log @@ -0,0 +1,259 @@ +before: 5 containers, engine=29.7.1, live=false +=== felhom-agent 0.142.0-rc1 selftest=os-update vmid=9201 ring=0 enabled=true guest-release=false host-release=false appliance=true === +time=2026-10-04T16:12:37.832+02:00 level=INFO msg="osupdate: START" run=20261004T141237Z layer=guest vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T141237Z +time=2026-10-04T16:12:50.352+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T141237Z layer=guest:9201 lane=fast mode=apply select=pending-fast packages=0" +time=2026-10-04T16:12:50.352+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:12:50.352+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=0 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:12:50.352+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)" +time=2026-10-04T16:12:50.352+02:00 level=INFO msg="osupdate: DONE" run=20261004T141237Z layer=guest vmid=9201 ring=0 trigger=debug outcome=nothing healthy=true reason="" upgraded=0 pending=6 not_covered=6 restart_needed=10 reboot_needed=true wrapper_seconds=12.5 +time=2026-10-04T16:12:50.364+02:00 level=INFO msg="osupdate: START" run=20261004T141237Z layer=host vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T141237Z +time=2026-10-04T16:13:00.773+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T141237Z layer=host lane=fast mode=apply select=pending-fast packages=0" +time=2026-10-04T16:13:00.773+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:13:00.773+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=0 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:13:00.773+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)" +time=2026-10-04T16:13:01.549+02:00 level=INFO msg="osupdate: DONE" run=20261004T141237Z layer=host vmid=9201 ring=0 trigger=debug outcome=nothing healthy=true reason="" upgraded=0 pending=80 not_covered=80 restart_needed=34 reboot_needed=true wrapper_seconds=10.4 +time=2026-10-04T16:13:08.002+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: LIVE-RESTORE reload_rc=0 state=true containers=5 same-ids=yes" +time=2026-10-04T16:13:08.002+02:00 level=INFO msg="osupdate: live-restore" vmid=9201 result="{\"containers_before\": 5, \"reload_rc\": 0, \"result\": \"on\", \"same_ids\": true}" +time=2026-10-04T16:13:08.002+02:00 level=INFO msg="osupdate: START" run=20261004T141237Z layer=docker vmid=9201 ring=0 trigger=debug enabled=true release=ring0-20261004T141237Z +time=2026-10-04T16:13:42.395+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: START release=ring0-20261004T141237Z layer=docker:9201 lane=slow mode=apply select=pending-docker packages=0 authority=ring0" +time=2026-10-04T16:13:42.395+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: REPAIR configured=0 fixed=0" +time=2026-10-04T16:13:42.395+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: PLAN upgrade=6 already=0 not-installed=0 from-snapshot=0" +time=2026-10-04T16:13:42.395+02:00 level=INFO msg="osupdate: wrapper" line="os-apply: DONE rc=0 seconds=13.7 upgraded=6 restart-needed=agetty,containerd-shim,cron,dbus-daemon,dhclient,sshd,systemd,systemd-journal,systemd-logind,systemd-network reboot-needed=yes" +time=2026-10-04T16:14:16.982+02:00 level=INFO msg="osupdate: DONE" run=20261004T141237Z layer=docker vmid=9201 ring=0 trigger=debug outcome=applied healthy=true reason="" upgraded=6 pending=0 not_covered=0 restart_needed=10 reboot_needed=true wrapper_seconds=34.3 + --- os-update report (guest) --- + { + "authority": "", + "docker_engine": "", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": [ + "docker-ce-cli", + "containerd.io", + "docker-ce", + "docker-buildx-plugin", + "docker-ce-rootless-extras", + "docker-compose-plugin" + ], + "outcome": "nothing", + "pending": 6, + "reboot_needed": true, + "refused": null, + "release_id": "ring0-20261004T141237Z", + "restart_needed": [ + "agetty", + "containerd", + "cron", + "dbus-daemon", + "dhclient", + "sshd", + "systemd", + "systemd-journal", + "systemd-logind", + "systemd-network" + ], + "ring": 0, + "run_id": "20261004T141237Z", + "upgraded": [], + "wrapper_seconds": 12.5 + } + --- os-update report (host) --- + { + "authority": "", + "docker_engine": "", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": [ + "frr", + "shim-signed-common", + "shim-unsigned", + "shim-helpers-amd64-signed", + "shim-signed", + "libradosstriper1", + "librgw2", + "ceph-common", + "librbd1", + "librados2", + "python3-cephfs", + "libcephfs2", + "python3-rgw", + "python3-rados", + "python3-ceph-argparse", + "python3-ceph-common", + "python3-rbd", + "ceph-fuse", + "chrony", + "libcorosync-common4", + "libcfg7", + "libcmap4", + "libcpg4", + "libknet1t64", + "libnozzle1t64", + "libquorum5", + "libvotequorum8", + "corosync", + "frr-pythontools", + "libjs-extjs", + "libnvpair3linux", + "libproxmox-acme-plugins", + "libproxmox-backup-qemu0", + "pve-qemu-kvm", + "libpve-notify-perl", + "libpve-cluster-api-perl", + "libpve-cluster-perl", + "pve-cluster", + "libpve-access-control", + "libpve-apiclient-perl", + "librados2-perl", + "proxmox-backup-client", + "proxmox-backup-file-restore", + "pve-manager", + "libproxmox-acme-perl", + "libpve-common-perl", + "libpve-guest-common-perl", + "qemu-server", + "libpve-storage-perl", + "pve-edk2-firmware-legacy", + "pve-edk2-firmware-ovmf", + "libpve-network-api-perl", + "libpve-network-perl", + "proxmox-firewall-data", + "pve-firewall", + "pve-container", + "pve-ha-manager", + "novnc-pve", + "proxmox-enterprise-support-keyring", + "proxmox-mini-journalreader", + "proxmox-widget-toolkit", + "pve-docs", + "pve-i18n", + "pve-xtermjs", + "pve-yew-mobile-i18n", + "pve-yew-mobile-gui", + "libuutil3linux", + "libzfs7linux", + "libzpool7linux", + "proxmox-first-boot", + "pve-firmware", + "proxmox-kernel-7.0.14-20-pve-signed", + "proxmox-kernel-7.0", + "proxmox-kernel-helper", + "pve-edk2-firmware-aarch64", + "pve-edk2-firmware", + "zfs-initramfs", + "zfsutils-linux", + "zfs-zed", + "tailscale" + ], + "outcome": "nothing", + "pending": 80, + "reboot_needed": true, + "refused": null, + "release_id": "ring0-20261004T141237Z", + "restart_needed": [ + "agetty", + "blkmapd", + "chronyd", + "cron", + "dbus-daemon", + "dmeventd", + "ksmtuned", + "lxc-monitord", + "lxc-start", + "lxcfs", + "pmxcfs", + "proxmox-firewal", + "pve-firewall", + "pve-ha-crm", + "pve-ha-lrm", + "pve-lxc-syscall", + "pvedaemon", + "pvedaemon worke", + "pvefw-logger", + "pveproxy", + "pveproxy worker", + "pvescheduler", + "pvestatd", + "qmeventd", + "rpcbind", + "rrdcached", + "smartd", + "spiceproxy", + "spiceproxy work", + "sshd", + "systemd-logind", + "systemd-udevd", + "watchdog-mux", + "zed" + ], + "ring": 0, + "run_id": "20261004T141237Z", + "upgraded": [], + "wrapper_seconds": 10.4 + } + --- os-update report (docker) --- + { + "authority": "ring0", + "docker_engine": "29.8.2", + "health_reason": "", + "healthy": true, + "mode": "apply", + "not_covered": null, + "outcome": "applied", + "pending": 0, + "reboot_needed": true, + "refused": null, + "release_id": "ring0-20261004T141237Z", + "restart_needed": [ + "agetty", + "containerd-shim", + "cron", + "dbus-daemon", + "dhclient", + "sshd", + "systemd", + "systemd-journal", + "systemd-logind", + "systemd-network" + ], + "ring": 0, + "run_id": "20261004T141237Z", + "upgraded": [ + { + "name": "docker-ce-cli", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "containerd.io", + "version": "2.3.6-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-ce", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-buildx-plugin", + "version": "0.37.1-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-ce-rootless-extras", + "version": "5:29.8.2-1~debian.13~trixie", + "origin": "" + }, + { + "name": "docker-compose-plugin", + "version": "5.6.0-1~debian.13~trixie", + "origin": "" + } + ], + "wrapper_seconds": 34.3 + } + pass took 1m39.2s +WALL=99.343682135 +IDS IDENTICAL (5 containers) +after: engine=29.8.2 live=true diff --git a/documentation/audits/os-docker-crash-2026-10-04/partC/c0-readonly-demo-hp.txt b/documentation/audits/os-docker-crash-2026-10-04/partC/c0-readonly-demo-hp.txt new file mode 100644 index 00000000..e00b82c7 --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partC/c0-readonly-demo-hp.txt @@ -0,0 +1,53 @@ ++ sysctl kernel.panic kernel.panic_on_oops kernel.sysrq kernel.panic_on_warn kernel.softlockup_panic kernel.hung_task_panic +kernel.panic = 0 +kernel.panic_on_oops = 0 +kernel.sysrq = 438 +kernel.panic_on_warn = 0 +kernel.softlockup_panic = 0 +kernel.hung_task_panic = 0 ++ ls -la /sys/fs/pstore/ /var/lib/systemd/pstore/ +/sys/fs/pstore/: +total 0 +drwxr-x--- 2 root root 0 Oct 4 14:36 . +drwxr-xr-x 10 root root 0 Oct 4 14:36 .. + +/var/lib/systemd/pstore/: +total 8 +drwxr-xr-x 2 root root 4096 May 21 13:32 . +drwxr-xr-x 12 root root 4096 Aug 21 17:44 .. ++ mount ++ grep pstore +none on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) ++ cat /sys/module/pstore/parameters/backend +efi_pstore ++ dmesg ++ grep -iE 'pstore|efi_pstore|ramoops|erst' ++ head +[ 4.129612] systemd[1]: Starting modprobe@efi_pstore.service - Load Kernel Module efi_pstore... +[ 4.153714] pstore: Using crash dump compression: deflate +[ 4.206148] pstore: Registered efi_pstore as persistent store backend +[ 4.207174] systemd[1]: modprobe@efi_pstore.service: Deactivated successfully. +[ 4.207514] systemd[1]: Finished modprobe@efi_pstore.service - Load Kernel Module efi_pstore. +[ 4.210265] systemd[1]: systemd-pstore.service - Platform Persistent Storage Archival skipped, unmet condition check ConditionDirectoryNotEmpty=/sys/fs/pstore ++ grep -E '^#?Storage' /etc/systemd/journald.conf +#Storage=auto ++ ls /var/log/journal ++ head -2 +9756a908fef046fda6a56428d3fefe3c ++ journalctl --list-boots --no-pager ++ tail -4 + -3 9cfa86adcf9f4dec99e324f5e2645032 Sun 2026-10-04 09:44:31 CEST Sun 2026-10-04 09:45:46 CEST + -2 1407ed1ad3ba4a37ac84ddae7cb05b3b Sun 2026-10-04 09:46:18 CEST Sun 2026-10-04 14:26:34 CEST + -1 ffd2f9aa02a344399338755a7958219f Sun 2026-10-04 14:27:08 CEST Sun 2026-10-04 14:35:43 CEST + 0 7e320e5daf1f4ee79760c2de1aaf39ff Sun 2026-10-04 14:36:16 CEST Sun 2026-10-04 15:26:56 CEST ++ systemctl is-enabled systemd-pstore.service +enabled ++ lsmod ++ grep -iE 'efi_pstore|ramoops' +efi_pstore 12288 0 ++ ls /sys/firmware/efi/efivars ++ grep -ci dump +0 ++ which kdump-config ++ cat /proc/cmdline +BOOT_IMAGE=/boot/vmlinuz-7.0.14-20-pve root=/dev/mapper/pve-root ro quiet diff --git a/documentation/audits/os-docker-crash-2026-10-04/partC/c1-crash1.txt b/documentation/audits/os-docker-crash-2026-10-04/partC/c1-crash1.txt new file mode 100644 index 00000000..737eb03e --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partC/c1-crash1.txt @@ -0,0 +1,85 @@ +== before crash 1, 2026-10-04T13:38:26Z +kernel.panic = 10 +7e320e5d-af1f-4ee7-9760-c2de1aaf39ff +7.0.14-20-pve +2026-10-04 14:36:11 +crash issued 2026-10-04T13:38:27Z +== after crash 1, 2026-10-04T13:39:27Z ++ uname -r +7.0.14-20-pve ++ uptime -s +2026-10-04 15:39:06 ++ cat /proc/sys/kernel/random/boot_id +0defc390-c295-4e9f-aa6e-13d69a1f24c3 ++ sysctl kernel.panic +kernel.panic = 0 ++ ls -la /sys/fs/pstore/ /var/lib/systemd/pstore/ +/sys/fs/pstore/: +total 0 +drwxr-x--- 2 root root 0 Oct 4 15:39 . +drwxr-xr-x 10 root root 0 Oct 4 15:39 .. + +/var/lib/systemd/pstore/: +total 8 +drwxr-xr-x 2 root root 4096 May 21 13:32 . +drwxr-xr-x 12 root root 4096 Aug 21 17:44 .. ++ find /var/lib/systemd/pstore -type f ++ head +++ find /var/lib/systemd/pstore -type f +++ head -2 ++ journalctl -b -1 --no-pager -n 8 -o short-iso +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Listening on gpg-agent.socket - GnuPG cryptographic agent and passphrase cache. +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Listening on ssh-agent.socket - OpenSSH Agent socket. +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Reached target sockets.target - Sockets. +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Reached target basic.target - Basic System. +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Reached target default.target - Main User Target. +2026-10-04T15:38:27+02:00 demo-hp systemd[205455]: Startup finished in 310ms. +2026-10-04T15:38:27+02:00 demo-hp systemd[1]: Started user@0.service - User Manager for UID 0. +2026-10-04T15:38:27+02:00 demo-hp systemd[1]: Started session-15.scope - Session 15 of User root. ++ journalctl -b -2 --no-pager -n 4 -o short-iso +2026-10-04T14:35:43+02:00 demo-hp systemd-shutdown[1]: Syncing filesystems and block devices. +2026-10-04T14:35:43+02:00 demo-hp systemd-shutdown[1]: Sending SIGTERM to remaining processes... +2026-10-04T14:35:43+02:00 demo-hp systemd-journald[349]: Received SIGTERM from PID 1 (systemd-shutdow). +2026-10-04T14:35:43+02:00 demo-hp systemd-journald[349]: Journal stopped ++ journalctl -b 0 -u systemd-pstore --no-pager -o cat ++ head ++ last -x ++ head -5 +reboot system boot 7.0.14-20-pve Sun Oct 4 15:39 - still running +reboot system boot 7.0.14-20-pve Sun Oct 4 14:36 - crash +reboot system boot 7.0.14-20-pve Sun Oct 4 14:27 - 14:35 (00:08) +shutdown system down 7.0.14-20-pve Sun Oct 4 14:35 - 14:36 (00:00) +reboot system boot 7.0.2-6-pve Sun Oct 4 09:46 - 14:26 (04:39) ++ cat /sys/module/efi_pstore/parameters/pstore_disable +N ++ cat /sys/module/pstore/parameters/max_reason +cat: /sys/module/pstore/parameters/max_reason: No such file or directory ++ cat /sys/module/pstore/parameters/kmsg_bytes +10240 +++ uname -r ++ grep -E 'EFI_VARS_PSTORE|PSTORE_DEFAULT|CONFIG_PSTORE=' /boot/config-7.0.14-20-pve +CONFIG_EFI_VARS_PSTORE=m +# CONFIG_EFI_VARS_PSTORE_DEFAULT_DISABLE is not set +CONFIG_PSTORE=y +CONFIG_PSTORE_DEFAULT_KMSG_BYTES=10240 ++ cat /sys/module/printk/parameters/always_kmsg_dump +N ++ journalctl -b 0 -u felhom-agent --no-pager -o short-iso ++ grep -iE 'started|report|hub:' ++ head -5 ++ cut -c1-200 +2026-10-04T15:39:22+02:00 demo-hp systemd[1]: Started felhom-agent.service - Felhom host agent (Proxmox host tier; hub control loop + PBS verify + storage watchdog). +2026-10-04T15:39:23+02:00 demo-hp sudo[1492]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/sbin/lvs --reportformat json --units b -o lv_name,data_percent,metadata_percent -- pve/data +2026-10-04T15:39:24+02:00 demo-hp felhom-agent[1304]: time=2026-10-04T15:39:24.149+02:00 level=INFO msg="guest-power: watchdog started" interval=1m0s max_attempts=3 +2026-10-04T15:39:24+02:00 demo-hp felhom-agent[1304]: time=2026-10-04T15:39:24.149+02:00 level=INFO msg="controller-supervisor: started" interval=30s confirm_sweeps=2 crashloop_max=3 crashloop_window= +2026-10-04T15:39:24+02:00 demo-hp sudo[1585]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/sbin/lvs --reportformat json --units b -o lv_name,data_percent,metadata_percent -- pve/data ++ pct status 9201 +status: running ++ journalctl -k -b -2 --no-pager ++ grep -ciE 'oops|BUG:' +0 ++ journalctl -k --no-pager ++ grep -iE 'Oops|BUG:|Call Trace' ++ head -3 ++ cat /proc/sys/kernel/tainted +4097 diff --git a/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-felhom.txt b/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-felhom.txt new file mode 100644 index 00000000..36ce73aa --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-felhom.txt @@ -0,0 +1,10 @@ +755 root 41c759e4c195d349 /usr/local/sbin/felhom-os-apply +755 root f45b99817fe8ff50 /usr/local/sbin/felhom-crash-guard +644 root 5580ad6cdd51990b /etc/systemd/system/felhom-crash-guard.service +644 root 93d2c4749fe63ab1 /etc/systemd/system/felhom-crash-guard-check.service +644 root b214e79e483c339f /etc/systemd/system/felhom-crash-guard-check.timer +644 root 9b9d305b421f81d2 /etc/felhom/crash-guard.conf +644 root d17cb8e4456575b1 /etc/felhom/operator-signers +644 root 338a564cc8392bad /etc/felhom/os-trust.json +755 root 18acdf69f9c915be /usr/local/bin/felhom-agent-0.142.0-rc1 +kernel.panic = 10 diff --git a/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-hp.txt b/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-hp.txt new file mode 100644 index 00000000..8428e15f --- /dev/null +++ b/documentation/audits/os-docker-crash-2026-10-04/partD/copied-by-hand-demo-hp.txt @@ -0,0 +1,33 @@ +Created symlink '/etc/systemd/system/sysinit.target.wants/felhom-crash-guard.service' -> '/etc/systemd/system/felhom-crash-guard.service'. +Created symlink '/etc/systemd/system/timers.target.wants/felhom-crash-guard-check.timer' -> '/etc/systemd/system/felhom-crash-guard-check.timer'. +755 root 41c759e4c195d349 /usr/local/sbin/felhom-os-apply +755 root f45b99817fe8ff50 /usr/local/sbin/felhom-crash-guard +644 root 5580ad6cdd51990b /etc/systemd/system/felhom-crash-guard.service +644 root 93d2c4749fe63ab1 /etc/systemd/system/felhom-crash-guard-check.service +644 root b214e79e483c339f /etc/systemd/system/felhom-crash-guard-check.timer +644 root 9b9d305b421f81d2 /etc/felhom/crash-guard.conf +644 root d17cb8e4456575b1 /etc/felhom/operator-signers +644 root af0890ef7a5778d0 /etc/felhom/os-trust.json +755 root 18acdf69f9c915be /usr/local/bin/felhom-agent-0.142.0-rc1 +active +active +kernel.panic = 10 +{ + "armed": true, + "boot_id": "0defc390-c295-4e9f-aa6e-13d69a1f24c3", + "config": { + "LIMIT": 3, + "PANIC_SECONDS": 10, + "REARM_HOURS": 24, + "WINDOW_MINUTES": 60 + }, + "kernel_panic": 10, + "last_boot_at": "2026-10-04T13:39:06Z", + "last_boot_unclean": false, + "tripped": false, + "unclean_boots": [], + "unclean_boots_24h": 0, + "unclean_boots_in_window": 0, + "updated_at": "2026-10-04T14:08:23Z", + "version": 1 +} diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 28ea01b5..f14d50bb 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,32 @@ +## v0.132.0 — the System page, the Docker engine release, the crash events (R-852, `09` decisions 87–89) (2026-10-04) + +**Needs agent v0.142.0** for the versions, the Docker step and the crash guard; an older agent shows "no versions +reported" and every value as `unknown`. + +- **The System page** (`/system`, new "System" tab on every page; operator login like every page). One row per box: + ring and updates switch **with buttons**; the tunnel; host: Proxmox, running kernel, next-boot kernel (amber when it + differs), Debian, Felhom OS release, pending, not covered, **held packages (R-848)**, reboot needed since, + `kernel.panic`, kernel oops, crash restarts in 24 h, the crash guard; guest: Debian, release, pending, restart needed; + Docker: engine, containerd, live-restore, release; the last OS leg (when, outcome, seconds). Amber = worth a look; red = + an operator alarm fires (the same thresholds as `08` §6.3); a value the box could not read says `unknown`, amber, with + the reason. Above the table: the newest release per layer (guest, host, Docker), what ring 0 runs now and how long it + still waits, **"Approve now" (guest + host)** with a confirmation, and **"Approve Docker set"** — shown only when the + rule allows it. Page buttons post `return=/system` and come back with a message; the JSON routes are unchanged. +- **The Hosts page** gets a "Proxmox / kernel" column (R-852). +- **The Docker engine set** (`11` §5.8): a third layer, `docker`. It is **never approved automatically**: the operator's + button works only after every ring-0 box ran the set in **2 healthy night Docker steps** (`OS_DOCKER_APPROVE_NIGHTS` + overrides it for a TEST only, logged). An approval nudges no box — ring 1 takes a Docker set only through a signed + operator job. A Docker health failure mail names the undo (a signed `os_docker_step` with `undo`). +- **The crash events** (decision 88, R-851), from the report's crash-guard state: each NEW unclean boot is an operator + mail (`host_crash_restart`, warning) and the household's one line (`host_restarted_after_crash`, info, hu/en); the + guard tripping is an alarm (`host_crash_guard_tripped`, error); a re-arm (`host_crash_guard_rearmed`) and a kernel oops + (`host_kernel_oops`, warning) are announced once. The first report from a box only records its history. +- Package `internal/sysfacts`: the one parser of the `system` stanza. +- Tests: System page (full / partial / unknown, the Docker button per branch, every button behind the login, the + redirect), the Hosts column, Docker approval, crash events. Red-proofs: + `documentation/audits/os-docker-crash-2026-10-04/partA/hub-redproofs.txt` (10 caught; 2 tests had to be strengthened + first). + ## v0.131.1 — a reboot clears "reboot needed" (2026-10-04) **Pairs with agent v0.141.1.** A patch release in the same session as v0.131.0 — a deliberate exception to "one diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 4b9e07b3..e3846814 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -411,7 +411,20 @@ func main() { logger.Printf("[ERROR] OS_APPROVE_NIGHTS=%q invalid — keeping 1", v) } } + osSvc.DockerNights = 2 + if v := os.Getenv("OS_DOCKER_APPROVE_NIGHTS"); v != "" { + if n, nerr := strconv.Atoi(v); nerr == nil && n >= 0 { + osSvc.DockerNights = n + if n == 0 { + osSvc.DockerNights = -1 // "none" (0 in the service means the default) + } + logger.Printf("[WARN] OS_DOCKER_APPROVE_NIGHTS=%s — the Docker button needs %d healthy night(s) instead of 2 (TEST CONFIGURATION)", v, n) + } else { + logger.Printf("[ERROR] OS_DOCKER_APPROVE_NIGHTS=%q invalid — keeping 2", v) + } + } logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired) + logger.Printf("[INFO] osupdates: the Docker engine set is approved only by the operator, after %d healthy ring-0 night(s)", osSvc.DockerNights) apiHandler.SetOSUpdateService(osSvc) webServer.SetOSUpdateAdmin(osSvc) // `11` §8.3: the four alarm thresholds are configuration (decided by CC unattended — operator may reverse). diff --git a/hub/internal/api/crash.go b/hub/internal/api/crash.go new file mode 100644 index 00000000..aa3db4ce --- /dev/null +++ b/hub/internal/api/crash.go @@ -0,0 +1,104 @@ +package api + +import ( + "encoding/json" + "fmt" + + "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" +) + +// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the +// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the +// texts say "an unexpected stop". +const ( + EventHostCrashRestart = "host_crash_restart" // warning, operator: the box restarted after an unclean stop + EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed) + EventCrashGuardTripped = "host_crash_guard_tripped" // error, operator: the next crash leaves the box off + EventCrashGuardRearmed = "host_crash_guard_rearmed" // info, operator + EventKernelOops = "host_kernel_oops" // warning, operator: a kernel oops this boot (no restart) +) + +type crashSeen struct { + Boots []string `json:"boots"` + TrippedAt string `json:"tripped_at"` + RearmedAt string `json:"rearmed_at"` + OopsBoot string `json:"oops_boot"` +} + +// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is +// there (no flood of old history). Pinned by crash_test.go. +func (h *Handler) checkCrash(hostID, custID string, body []byte) { + sys := sysfacts.Parse(string(body)) + cg := sys.Host.CrashGuard + if !sys.Present || cg == nil { + return + } + var seen crashSeen + raw := h.store.CrashSeen(hostID) + first := raw == "" + if !first { + _ = json.Unmarshal([]byte(raw), &seen) + } + known := map[string]bool{} + for _, b := range seen.Boots { + known[b] = true + } + emit := func(typ, sev, msg string, extra map[string]any) { + if first { + return + } + d := map[string]any{"host_id": hostID} + for k, v := range extra { + d[k] = v + } + dj, _ := json.Marshal(d) + h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ) + if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil { + h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err) + } else if h.dispatcher != nil { + go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub") + } + } + for _, b := range cg.UncleanBoots { + if known[b] { + continue + } + emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.", + hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h}) + emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.", + map[string]any{"boot_at": b}) + seen.Boots = append(seen.Boots, b) + } + if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt { + emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.", + hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt}) + seen.TrippedAt = cg.TrippedAt + } + if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt { + emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy), + map[string]any{"rearmed_at": cg.RearmedAt}) + seen.RearmedAt = cg.RearmedAt + } + if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot { + emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID), + map[string]any{"boot_id": cg.BootID}) + seen.OopsBoot = cg.BootID + } + if first { + seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt + } + if len(seen.Boots) > 200 { + seen.Boots = seen.Boots[len(seen.Boots)-200:] + } + sj, _ := json.Marshal(seen) + if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil { + h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err) + } +} + +func intOr(p *int, d string) string { + if p == nil { + return d + } + return fmt.Sprint(*p) +} diff --git a/hub/internal/api/crash_test.go b/hub/internal/api/crash_test.go new file mode 100644 index 00000000..aea55062 --- /dev/null +++ b/hub/internal/api/crash_test.go @@ -0,0 +1,67 @@ +package api + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func crashBody(boots []string, tripped bool, trippedAt, rearmedAt, bootID string, oops bool) []byte { + q := func(ss []string) string { + if len(ss) == 0 { + return "[]" + } + return `["` + strings.Join(ss, `","`) + `"]` + } + return []byte(fmt.Sprintf(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts":{"host":{"oops_this_boot":%v,"kernel_panic":10, +"crash_guard":{"unclean_boots":%s,"unclean_boots_24h":%d,"tripped":%v,"tripped_at":%q,"tripped_reason":"2 unclean boots within 60 minutes", +"rearmed_at":%q,"rearmed_by":"operator","boot_id":%q}},"guest":{}}}}`, oops, q(boots), len(boots), tripped, trippedAt, rearmedAt, bootID)) +} + +func countEv(t *testing.T, h *Handler, cust, typ string) int { + t.Helper() + ev, err := h.store.GetEventsByType(cust, typ, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + return len(ev) +} + +// The first report only records history (no flood); each NEW unclean boot is one operator event + one household line; +// the trip is one alarm; a repeat report is quiet. Red-proof: drop the `known[b]` skip and the repeat report emits again. +func TestCheckCrash_EventsOncePerFact(t *testing.T) { + h, st, _ := newTestHandler(t) + seedHost(t, st, "h1", "c1", "K1") + old := []string{"2026-10-01T10:00:00Z"} + h.checkCrash("h1", "c1", crashBody(old, false, "", "", "b0", false)) + if n := countEv(t, h, "c1", EventHostCrashRestart); n != 0 { + t.Fatalf("first sighting flooded %d events", n) + } + two := append(old, "2026-10-04T15:40:00Z") + h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false)) + h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false)) // the same report again + if countEv(t, h, "c1", EventHostCrashRestart) != 1 || countEv(t, h, "c1", EventHostRestartedAfterCrash) != 1 { + t.Fatalf("restart events: op=%d household=%d", countEv(t, h, "c1", EventHostCrashRestart), countEv(t, h, "c1", EventHostRestartedAfterCrash)) + } + three := append(two, "2026-10-04T15:45:00Z") + h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false)) + h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false)) + if countEv(t, h, "c1", EventCrashGuardTripped) != 1 { + t.Fatalf("tripped events = %d", countEv(t, h, "c1", EventCrashGuardTripped)) + } + h.checkCrash("h1", "c1", crashBody(three, false, "", "2026-10-04T16:10:00Z", "b2", true)) + if countEv(t, h, "c1", EventCrashGuardRearmed) != 1 || countEv(t, h, "c1", EventKernelOops) != 1 { + t.Fatal("re-arm / oops not announced") + } +} + +// An agent older than v0.142.0 (no system stanza) never produces a crash event. +func TestCheckCrash_NoStanzaIsQuiet(t *testing.T) { + h, st, _ := newTestHandler(t) + seedHost(t, st, "h1", "c1", "K1") + h.checkCrash("h1", "c1", []byte(`{"host":{}}`)) + if st.CrashSeen("h1") != "" { + t.Fatal("a report without the stanza recorded crash state") + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 86c4b0f8..a2765ef6 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -863,6 +863,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) { } h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail) + h.checkCrash(hostID, custID, body) for _, g := range rep.Guests { status := g.Status diff --git a/hub/internal/i18n/locales/en.json b/hub/internal/i18n/locales/en.json index e47a1f4c..a71a7deb 100644 --- a/hub/internal/i18n/locales/en.json +++ b/hub/internal/i18n/locales/en.json @@ -87,5 +87,6 @@ "bind.resend.button": "Send me a new link", "bind.resent.lead": "Done.", "bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support.", + "mail.event.host_restarted_after_crash": "Your box restarted by itself after an unexpected stop. You do not need to do anything.", "mail.event.os_update_applied": "System security fixes were installed on your box. You do not need to do anything." } diff --git a/hub/internal/i18n/locales/hu.json b/hub/internal/i18n/locales/hu.json index 90dee775..16b088fc 100644 --- a/hub/internal/i18n/locales/hu.json +++ b/hub/internal/i18n/locales/hu.json @@ -87,5 +87,6 @@ "bind.resend.button": "Új linket kérek", "bind.resent.lead": "Kész.", "bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak.", + "mail.event.host_restarted_after_crash": "A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.", "mail.event.os_update_applied": "Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned." } diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 2e2b1698..68e72f35 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -695,6 +695,11 @@ var operatorOnlyEvents = map[string]bool{ "os_reboot_needed": true, "os_ring0_stalled": true, "os_not_covered": true, + // R-851 (hub v0.132.0): the crash guard. host_restarted_after_crash is deliberately NOT here — the household's line. + "host_crash_restart": true, + "host_crash_guard_tripped": true, + "host_crash_guard_rearmed": true, + "host_kernel_oops": true, // R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow // blobs. A customer can take no action on it — the remedy is the operator's inspection of the // off-site tier — and the text is operator-grade English naming host ids and retained-blob diff --git a/hub/internal/notify/os_alarms_operator_test.go b/hub/internal/notify/os_alarms_operator_test.go index ca67bc3b..6aa56396 100644 --- a/hub/internal/notify/os_alarms_operator_test.go +++ b/hub/internal/notify/os_alarms_operator_test.go @@ -12,11 +12,15 @@ import ( func TestOSUpdateEvents_OperatorOnlyExceptApplied(t *testing.T) { for _, e := range []string{osupdates.EventFailed, osupdates.EventHealthFailed, osupdates.EventReleaseApprove, osupdates.EventApprovedNow, osupdates.EventSettings, osupdates.EventStale, osupdates.EventRebootNeeded, - osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered"} { + osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered", + "host_crash_restart", "host_crash_guard_tripped", "host_crash_guard_rearmed", "host_kernel_oops"} { if !operatorOnlyEvents[e] { t.Errorf("%s is not operator-only", e) } } + if operatorOnlyEvents["host_restarted_after_crash"] { + t.Errorf("host_restarted_after_crash is the household's line and must NOT be operator-only") + } if operatorOnlyEvents[osupdates.EventApplied] { t.Errorf("%s is the household's line and must NOT be operator-only", osupdates.EventApplied) } diff --git a/hub/internal/notify/testdata/mail_goldens/en/customer_event_host_restarted_after_crash.txt b/hub/internal/notify/testdata/mail_goldens/en/customer_event_host_restarted_after_crash.txt new file mode 100644 index 00000000..c22a4498 --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/en/customer_event_host_restarted_after_crash.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Warning: Your box restarted by itself after an unexpected stop. You do not need to do anything. +--- +Dear Customer, + +Your Felhom system sent the following notification: + +Your box restarted by itself after an unexpected stop. You do not need to do anything. + +Details: +- Server: demo-fixture +- Time: 2026-01-15 10:30 +- Level: Warning +- Type: host_restarted_after_crash + +If you have any questions, contact your operator. + +Best regards, +Felhom.eu monitoring diff --git a/hub/internal/notify/testdata/mail_goldens/hu/customer_event_host_restarted_after_crash.txt b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_host_restarted_after_crash.txt new file mode 100644 index 00000000..4b1b2171 --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_host_restarted_after_crash.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Figyelmeztetés: A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned. +--- +Kedves Ügyfél! + +A Felhom rendszered a következő értesítést küldte: + +A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned. + +Részletek: +- Szerver: demo-fixture +- Időpont: 2026-01-15 10:30 +- Szint: Figyelmeztetés +- Típus: host_restarted_after_crash + +Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel. + +Üdvözlettel, +Felhom.eu monitoring diff --git a/hub/internal/osupdates/docker_test.go b/hub/internal/osupdates/docker_test.go new file mode 100644 index 00000000..33bbbd1a --- /dev/null +++ b/hub/internal/osupdates/docker_test.go @@ -0,0 +1,79 @@ +package osupdates + +import ( + "strings" + "testing" + "time" +) + +var engineSet = []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"}, + {Name: "containerd.io", Version: "2.3.6-1~debian.13~trixie", Origin: "Docker"}} + +func (f *fix) dockerNight(t *testing.T, host string, healthy bool) { + t.Helper() + out := "nothing" + if !healthy { + out = "health_failed" + } + f.ingest(t, host, Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy, + Installed: append([]Package{pk("libc6", "x")}, engineSet...)}) +} + +// The Docker set is NEVER approved automatically (`11` §5.8: the operator approves it). Red-proof: add LayerDocker to +// Layers (the auto-approved list) and this fails. +func TestDocker_NeverAutoApproved(t *testing.T) { + f := newFix(t) + for i := 0; i < 3; i++ { + f.dockerNight(t, "hp", true) + f.dockerNight(t, "n100", true) + f.now = f.now.Add(25 * time.Hour) + f.s.Evaluate() + } + if rel, _ := f.s.Store.LatestOSRelease(LayerDocker); rel != nil { + t.Fatalf("a Docker set was auto-approved: %+v", rel) + } +} + +// The operator's button works only after every ring-0 box ran the set 2 healthy nights; an unhealthy step blocks it. +// The candidate is the six engine packages only. Red-proof: compare nights against 1 instead of DockerNights and the +// one-night step approves. +func TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box(t *testing.T) { + f := newFix(t) + f.dockerNight(t, "hp", true) + f.dockerNight(t, "n100", true) + if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "1 of 2") { + t.Fatalf("approved after one night: %v", err) + } + f.now = f.now.Add(24 * time.Hour) + f.dockerNight(t, "hp", true) + if _, err := f.s.ApproveDocker(); err == nil { + t.Fatal("approved while n100 had only one night") + } + f.dockerNight(t, "n100", true) + id, err := f.s.ApproveDocker() + if err != nil || !strings.HasPrefix(id, "os-docker-") { + t.Fatalf("%q %v", id, err) + } + rel, _ := f.s.Store.LatestOSRelease(LayerDocker) + if rel.ApprovedBy != "operator" || !strings.Contains(rel.PackagesJSON, "docker-ce") || strings.Contains(rel.PackagesJSON, "libc6") { + t.Fatalf("release %+v", rel) + } + if len(f.bumps) != 0 { + t.Fatalf("a Docker approval must nudge no box (ring 1 takes it by a signed job): %v", f.bumps) + } + if b := f.s.DesiredBlock("cust1"); b.Release != nil || b.HostRelease != nil { + t.Fatalf("the Docker set leaked into the desired block: %+v", b) + } +} + +func TestDocker_UnhealthyStepBlocksTheButton(t *testing.T) { + f := newFix(t) + f.dockerNight(t, "hp", true) + f.dockerNight(t, "n100", true) + f.now = f.now.Add(24 * time.Hour) + f.dockerNight(t, "hp", false) + f.dockerNight(t, "n100", true) + if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "health_failed") { + t.Fatalf("an unhealthy Docker step did not block: %v", err) + } +} diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go index 8e29fb73..069b1205 100644 --- a/hub/internal/osupdates/service.go +++ b/hub/internal/osupdates/service.go @@ -33,13 +33,21 @@ import ( // Layers. const ( - LayerGuest = "guest" - LayerHost = "host" + LayerGuest = "guest" + LayerHost = "host" + LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0) ) -// Layers lists every layer the hub approves. +// Layers lists the layers the hub approves AUTOMATICALLY (the fast lane). var Layers = []string{LayerGuest, LayerHost} +// AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker). +var AllLayers = []string{LayerGuest, LayerHost, LayerDocker} + +// dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES). +var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true, + "docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true} + // hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware // packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14). // Pinned by TestCandidate_HostLeavesOutKernelBootFirmware. @@ -91,6 +99,9 @@ type Report struct { RebootScanned bool `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass) Refused json.RawMessage `json:"refused,omitempty"` PassSeconds float64 `json:"pass_seconds,omitempty"` + DockerEngine string `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0) + Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed + Undo bool `json:"undo,omitempty"` // docker layer: a signed undo } // PendingPkg is one update the sources offer. @@ -145,6 +156,10 @@ type Service struct { Emit func(customerID, eventType, severity, message, details, source string) ApproveAfter time.Duration NightsRequired int + // DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's + // button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for + // a TEST only, logged. + DockerNights int // Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default. StaleAfter time.Duration // 7 d RebootAfter time.Duration // 14 d @@ -191,8 +206,9 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) { } func layerOf(r Report) string { - if r.Layer == LayerHost { - return LayerHost + switch r.Layer { + case LayerHost, LayerDocker: + return r.Layer } return LayerGuest } @@ -215,8 +231,11 @@ func (s *Service) Ingest(hostID string, r Report) error { details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, "upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason} where := "the box" - if layer == LayerHost { + switch layer { + case LayerHost: where = "the box's base system" + case LayerDocker: + where = "the box's app engine (Docker " + r.DockerEngine + ")" } switch r.Outcome { case "applied", "health_failed": @@ -224,8 +243,11 @@ func (s *Service) Ingest(hostID string, r Report) error { fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details) if !r.Healthy || r.Outcome == "health_failed" { undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)" - if layer == LayerHost { + switch layer { + case LayerHost: undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)" + case LayerDocker: + undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)" } s.event(h.CustomerID, EventHealthFailed, "error", fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.", @@ -254,7 +276,11 @@ func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, e return nil, err } for _, p := range r.Installed { - if p.Origin != "Debian" && p.Origin != "Debian-Security" { + if layer == LayerDocker { + if !dockerNames[p.Name] { + continue + } + } else if p.Origin != "Debian" && p.Origin != "Debian-Security" { continue } if layer == LayerHost && hostSlowRE.MatchString(p.Name) { @@ -322,6 +348,7 @@ type Status struct { FirstSeen time.Time Packages int Waiting string // why not approved yet ("" = approved or nothing to do) + Approved string // the release id when this set is already approved } // Evaluate checks the approval rule of every layer and approves when it holds. Called every minute. @@ -337,7 +364,12 @@ func (s *Service) Evaluate() ([]Status, error) { return out, nil } -func (s *Service) evaluateLayer(layer string) (Status, error) { +func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) } + +// LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves. +func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) } + +func (s *Service) layerStatus(layer string, approve bool) (Status, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return Status{Layer: layer, Waiting: "no ring-0 box"}, err @@ -354,6 +386,7 @@ func (s *Service) evaluateLayer(layer string) (Status, error) { } st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)} if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp { + st.Approved = rel.ID return st, nil // already approved } if age := s.now().Sub(first); age < s.ApproveAfter { @@ -380,9 +413,53 @@ func (s *Service) evaluateLayer(layer string) (Status, error) { return st, nil } } + if !approve { + st.Waiting = "ready — approves at the next minute's check" + return st, nil + } return st, s.approve(layer, fp, list, "auto") } +// ReleaseInfo is one layer's newest approved release, for the System page. +type ReleaseInfo struct { + Layer string + ID string + ApprovedAt time.Time + ApprovedBy string + Packages int +} + +// Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent. +func (s *Service) Releases() []ReleaseInfo { + var out []ReleaseInfo + for _, layer := range AllLayers { + rel, _ := s.Store.LatestOSRelease(layer) + if rel == nil { + continue + } + var list []Package + _ = json.Unmarshal([]byte(rel.PackagesJSON), &list) + out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)}) + } + return out +} + +// Candidates is the read-only candidate status of every layer (the Docker one by its own rule). +func (s *Service) Candidates() []Status { + var out []Status + for _, layer := range Layers { + st, _ := s.LayerStatus(layer) + out = append(out, st) + } + d, _ := s.DockerStatus() + return append(out, d) +} + +// Thresholds are the alarm numbers the System page colours by (the same values the alarms use). +func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) { + return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour) +} + // ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids. func (s *Service) ApproveNow() (string, error) { ring0, err := s.ring0Hosts() @@ -418,6 +495,84 @@ func (s *Service) ApproveNow() (string, error) { return strings.Join(ids, ","), nil } +// DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0 +// box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy. +func (s *Service) DockerStatus() (Status, error) { + st := Status{Layer: LayerDocker} + ring0, err := s.ring0Hosts() + if err != nil || len(ring0) == 0 { + st.Waiting = "no ring-0 box" + return st, err + } + cand, err := s.candidate(LayerDocker, ring0) + if err != nil || len(cand) == 0 { + st.Waiting = "a ring-0 box has not reported a Docker step" + return st, err + } + fp, list := fingerprint(LayerDocker, cand) + pj, _ := json.Marshal(list) + first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) + if err != nil { + return st, err + } + st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list) + if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp { + st.Approved, st.Waiting = rel.ID, "already approved" + return st, nil + } + need := s.DockerNights // 0 = the default 2; negative = none (a TEST override only, logged at start) + switch { + case need == 0: + need = 2 + case need < 0: + need = 0 + } + for _, h := range ring0 { + reps, err := s.Store.OSReportsSince(h, LayerDocker, first) + if err != nil { + return st, err + } + nights := 0 + for _, r := range reps { + if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { + st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) + return st, nil + } + if r.Trigger == "night" { + nights++ + } + } + if nights < need { + st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need) + return st, nil + } + } + return st, nil +} + +// ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows. +// A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing. +func (s *Service) ApproveDocker() (string, error) { + st, err := s.DockerStatus() + if err != nil { + return "", err + } + if st.Waiting != "" { + return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting) + } + ring0, _ := s.ring0Hosts() + cand, err := s.candidate(LayerDocker, ring0) + if err != nil { + return "", err + } + fp, list := fingerprint(LayerDocker, cand) + if err := s.approve(LayerDocker, fp, list, "operator"); err != nil { + return "", err + } + rel, _ := s.Store.LatestOSRelease(LayerDocker) + return rel.ID, nil +} + func (s *Service) approve(layer, fp string, list []Package, by string) error { at := s.now().UTC().Truncate(time.Second) id := "os-" + layer + "-" + at.Format("20060102-150405") @@ -428,7 +583,7 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error { s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp) s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)), map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp}) - if s.Bump != nil { + if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state hosts, _ := s.Store.ListHosts() for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { @@ -518,6 +673,7 @@ type FleetLine struct { Tunnel string Guest LayerLine Host LayerLine + Docker LayerLine } func (s *Service) layerLine(hostID, layer string, ring int) LayerLine { @@ -605,6 +761,7 @@ func (s *Service) Fleet() ([]FleetLine, error) { } l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring) l.Host = s.layerLine(h.HostID, LayerHost, st.Ring) + l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring) out = append(out, l) } return out, nil @@ -617,7 +774,7 @@ func (s *Service) FleetJSON() (any, error) { return nil, err } out := map[string]any{"boxes": lines} - for _, layer := range Layers { + for _, layer := range AllLayers { if rel, _ := s.Store.LatestOSRelease(layer); rel != nil { out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} } diff --git a/hub/internal/store/os_updates.go b/hub/internal/store/os_updates.go index 1ac9cacc..beb7cf7c 100644 --- a/hub/internal/store/os_updates.go +++ b/hub/internal/store/os_updates.go @@ -313,3 +313,9 @@ func (s *Store) FirstOSReport(layer string, hosts []string) (*OSReport, error) { } return &rs[0], nil } + +// CrashSeen / SetCrashSeen keep, per host, what the hub already announced from the box's crash guard (hub v0.132.0): +// the unclean boots, the trip and the re-arm it has turned into events, so a report repeating them stays quiet. +func (s *Store) CrashSeen(hostID string) string { return s.getSetting("crash_seen:" + hostID) } + +func (s *Store) SetCrashSeen(hostID, v string) error { return s.setSetting("crash_seen:"+hostID, v) } diff --git a/hub/internal/sysfacts/sysfacts.go b/hub/internal/sysfacts/sysfacts.go new file mode 100644 index 00000000..4b1577d4 --- /dev/null +++ b/hub/internal/sysfacts/sysfacts.go @@ -0,0 +1,128 @@ +// Package sysfacts reads the agent's `system` stanza (agent v0.142.0, R-852, `09` decision 89): the box's Proxmox, +// kernel, Debian and Docker versions and its crash guard. One parser for the System page, the Hosts page and the +// crash events. A value the box could not read stays "unknown" — never empty, never guessed. Pinned by sysfacts_test.go. +package sysfacts + +import ( + "encoding/json" + "strings" +) + +// Unknown is what a field reads when nobody could read it. +const Unknown = "unknown" + +// CrashGuard is the box's felhom-crash-guard state (`11` §5.9). +type CrashGuard struct { + Armed bool `json:"armed"` + Tripped bool `json:"tripped"` + TrippedAt string `json:"tripped_at"` + TrippedReason string `json:"tripped_reason"` + UncleanBoots []string `json:"unclean_boots"` + InWindow int `json:"unclean_boots_in_window"` + In24h int `json:"unclean_boots_24h"` + LastBootAt string `json:"last_boot_at"` + LastBootUnclean bool `json:"last_boot_unclean"` + KernelPanic *int `json:"kernel_panic"` + BootID string `json:"boot_id"` + RearmedAt string `json:"rearmed_at"` + RearmedBy string `json:"rearmed_by"` +} + +// Host is the Proxmox host's half. +type Host struct { + Debian string `json:"debian"` + KernelRunning string `json:"kernel_running"` + KernelNextBoot string `json:"kernel_next_boot"` + KernelNextBootSource string `json:"kernel_next_boot_source"` + Held []string `json:"held"` // nil = unknown + Tainted *int `json:"tainted"` + OopsThisBoot *bool `json:"oops_this_boot"` + KernelPanic *int `json:"kernel_panic"` + CrashGuard *CrashGuard `json:"crash_guard"` +} + +// Guest is the customer guest's half. +type Guest struct { + Debian string `json:"debian"` + DockerEngine string `json:"docker_engine"` + Containerd string `json:"containerd"` + LiveRestore string `json:"live_restore"` // on | off | unknown + UnknownReason string `json:"unknown_reason"` +} + +// System is the whole stanza. Present is false for a report from an agent older than v0.142.0. +type System struct { + Present bool + PVEVersion string + KernelVersion string + VMID int + FactsError string + ReadAt string + Host Host + Guest Guest +} + +type wire struct { + System *struct { + PVEVersion string `json:"pve_version"` + KernelVersion string `json:"kernel_version"` + VMID int `json:"vmid"` + Facts json.RawMessage `json:"facts"` + FactsError string `json:"facts_error"` + ReadAt string `json:"read_at"` + } `json:"system"` +} + +func orUnknown(s string) string { + if strings.TrimSpace(s) == "" { + return Unknown + } + return s +} + +// Parse reads the stanza from a host-report body. Never fails: an absent or broken stanza is Present=false or unknowns. +func Parse(reportJSON string) System { + out := System{PVEVersion: Unknown, KernelVersion: Unknown, + Host: Host{Debian: Unknown, KernelRunning: Unknown, KernelNextBoot: Unknown}, + Guest: Guest{Debian: Unknown, DockerEngine: Unknown, Containerd: Unknown, LiveRestore: Unknown}} + var w wire + if json.Unmarshal([]byte(reportJSON), &w) != nil || w.System == nil { + return out + } + out.Present = true + out.PVEVersion, out.KernelVersion = orUnknown(w.System.PVEVersion), orUnknown(w.System.KernelVersion) + out.VMID, out.FactsError, out.ReadAt = w.System.VMID, w.System.FactsError, w.System.ReadAt + var f struct { + Host Host `json:"host"` + Guest Guest `json:"guest"` + } + if len(w.System.Facts) > 0 && json.Unmarshal(w.System.Facts, &f) == nil { + out.Host, out.Guest = f.Host, f.Guest + } + out.Host.Debian, out.Host.KernelRunning = orUnknown(out.Host.Debian), orUnknown(out.Host.KernelRunning) + out.Host.KernelNextBoot = orUnknown(out.Host.KernelNextBoot) + out.Guest.Debian, out.Guest.DockerEngine = orUnknown(out.Guest.Debian), orUnknown(out.Guest.DockerEngine) + out.Guest.Containerd, out.Guest.LiveRestore = orUnknown(out.Guest.Containerd), orUnknown(out.Guest.LiveRestore) + return out +} + +// ShortPVE turns "pve-manager/9.0.11/abc123" into "9.0.11". +func ShortPVE(s string) string { + if p := strings.Split(s, "/"); len(p) >= 2 && p[0] == "pve-manager" { + return p[1] + } + return s +} + +// ShortKernel turns "Linux 7.0.14-20-pve #1 SMP …" into "7.0.14-20-pve". +func ShortKernel(s string) string { + if f := strings.Fields(s); len(f) >= 2 && f[0] == "Linux" { + return f[1] + } + return s +} + +// NextBootDiffers is true when the box will boot a kernel other than the one it runs (both known). +func (s System) NextBootDiffers() bool { + return s.Host.KernelRunning != Unknown && s.Host.KernelNextBoot != Unknown && s.Host.KernelRunning != s.Host.KernelNextBoot +} diff --git a/hub/internal/sysfacts/sysfacts_test.go b/hub/internal/sysfacts/sysfacts_test.go new file mode 100644 index 00000000..b9046c54 --- /dev/null +++ b/hub/internal/sysfacts/sysfacts_test.go @@ -0,0 +1,43 @@ +package sysfacts + +import "testing" + +const full = `{"system":{"pve_version":"pve-manager/9.0.11/abc","kernel_version":"Linux 7.0.14-20-pve #1 SMP","vmid":9201, +"read_at":"2026-10-04T15:00:00Z","facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve","kernel_next_boot":"7.0.2-6-pve", +"held":["tzdata"],"tainted":4225,"oops_this_boot":true,"kernel_panic":10,"crash_guard":{"armed":false,"tripped":true, +"tripped_at":"2026-10-04T16:00:00Z","unclean_boots":["2026-10-04T15:40:00Z"],"unclean_boots_24h":1}}, +"guest":{"debian":"13.7","docker_engine":"29.7.2","containerd":"2.3.3-1","live_restore":"on"}}}}` + +func TestParse_Full(t *testing.T) { + s := Parse(full) + if !s.Present || ShortPVE(s.PVEVersion) != "9.0.11" || ShortKernel(s.KernelVersion) != "7.0.14-20-pve" || s.VMID != 9201 { + t.Fatalf("%+v", s) + } + if s.Host.Debian != "13.7" || len(s.Host.Held) != 1 || !*s.Host.OopsThisBoot || !s.Host.CrashGuard.Tripped || !s.NextBootDiffers() { + t.Fatalf("host %+v", s.Host) + } + if s.Guest.DockerEngine != "29.7.2" || s.Guest.LiveRestore != "on" { + t.Fatalf("guest %+v", s.Guest) + } +} + +// An agent older than v0.142.0, or a stanza whose facts failed: unknown everywhere, never empty. +// Red-proof: drop the orUnknown on the guest fields and the partial case fails. +func TestParse_AbsentAndPartialAreUnknown(t *testing.T) { + s := Parse(`{"host":{}}`) + if s.Present || s.PVEVersion != Unknown || s.Guest.DockerEngine != Unknown || s.Host.Held != nil { + t.Fatalf("absent: %+v", s) + } + s = Parse(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts_error":"no running customer guest"}}`) + if !s.Present || s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Host.KernelRunning != Unknown || s.FactsError == "" { + t.Fatalf("partial: %+v", s) + } + // facts present but empty fields (a guest that answered nothing): still unknown, not "" + s = Parse(`{"system":{"pve_version":"x","facts":{"host":{},"guest":{"docker_engine":"","live_restore":"","containerd":""}}}}`) + if s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Guest.Containerd != Unknown || s.Host.Debian != Unknown { + t.Fatalf("empty facts fields: %+v", s) + } + if s.NextBootDiffers() { + t.Fatal("unknown kernels must not read as 'differs'") + } +} diff --git a/hub/internal/web/hosts.go b/hub/internal/web/hosts.go index 6fd66843..148e3b3f 100644 --- a/hub/internal/web/hosts.go +++ b/hub/internal/web/hosts.go @@ -11,6 +11,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" + "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // agentOrUnknown renders an agent version for operator text, mapping the empty (never-reported) @@ -502,6 +503,8 @@ func parseHostStorageTargets(reportJSON string) []storageTargetView { // hostListRow is the per-host view model for the fleet list. type hostListRow struct { + PVEVersion string // R-852 (hub v0.132.0): from the report's system stanza; "unknown" when not reported + KernelRunning string HostID string CustomerID string CustomerName string @@ -590,8 +593,14 @@ func (s *Server) handleHostsList(w http.ResponseWriter, r *http.Request) { } // Vitals from the latest report body. + row.PVEVersion, row.KernelRunning = sysfacts.Unknown, sysfacts.Unknown if reportJSON, _ := s.store.GetLatestHostReportJSON(h.CustomerID); reportJSON != "" { row.Vitals = parseHostVitals(reportJSON) + sf := sysfacts.Parse(reportJSON) + row.PVEVersion, row.KernelRunning = sysfacts.ShortPVE(sf.PVEVersion), sf.Host.KernelRunning + if row.KernelRunning == sysfacts.Unknown { + row.KernelRunning = sysfacts.ShortKernel(sf.KernelVersion) + } } if wf, ok := worstFill[h.HostID]; ok { diff --git a/hub/internal/web/os_updates.go b/hub/internal/web/os_updates.go index 6e9fb591..ddcb9fbe 100644 --- a/hub/internal/web/os_updates.go +++ b/hub/internal/web/os_updates.go @@ -2,7 +2,9 @@ package web import ( "encoding/json" + "fmt" "net/http" + "net/url" "strconv" "strings" ) @@ -12,13 +14,27 @@ import ( // POST /os/ring/ ring=0|1 // POST /os/enabled/ on=1|0 // POST /os/approve-now approve the current ring-0 set at once (an operator event) +// POST /os/approve-docker approve the Docker engine set ring 0 ran 2 healthy nights (`11` §5.8) +// (a form field return=/system makes any POST answer with a redirect to the System page) // GET /os/fleet one line per box (JSON) func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path string) { if s.osUpdates == nil { http.Error(w, "os updates not configured", http.StatusServiceUnavailable) return } + // A button on the System page posts return=/system: answer with a redirect and a flash, never JSON. + fromPage := r.Method == http.MethodPost && r.FormValue("return") == "/system" reply := func(v any, err error) { + if fromPage { + q := "flash=done" + if err != nil { + q = "err=" + url.QueryEscape(err.Error()) + } else if m, ok := v.(map[string]string); ok && m["release_id"] != "" { + q = "flash=" + url.QueryEscape("approved "+m["release_id"]) + } + http.Redirect(w, r, "/system?"+q, http.StatusSeeOther) + return + } if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -46,6 +62,14 @@ func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path stri case r.Method == http.MethodPost && path == "/os/approve-now": id, err := s.osUpdates.ApproveNow() reply(map[string]string{"release_id": id}, err) + case r.Method == http.MethodPost && path == "/os/approve-docker": + view, ok := s.osUpdates.(OSSystemView) + if !ok { + reply(nil, fmt.Errorf("docker approval not available")) + return + } + id, err := view.ApproveDocker() + reply(map[string]string{"release_id": id}, err) default: http.Error(w, "not found", http.StatusNotFound) } diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index 99fe41e2..6288788b 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -457,6 +457,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Hosts — read-only fleet view (audit F-M1) + the v0.46.0 log-bundle actions. case path == "/hosts" || path == "/hosts/": s.handleHostsList(w, r) + // System — every box's versions + OS updates, with the operator's buttons (hub v0.132.0, R-852). + case path == "/system": + s.handleSystem(w, r) // R-21 slice C — unclaimed-appliance operator actions (bind/discard). POST only. case strings.HasPrefix(path, "/appliances/") && strings.HasSuffix(path, "/bind"): if id, ok := parseApplianceID(path, "bind"); ok && r.Method == http.MethodPost { diff --git a/hub/internal/web/system.go b/hub/internal/web/system.go new file mode 100644 index 00000000..ad96de08 --- /dev/null +++ b/hub/internal/web/system.go @@ -0,0 +1,221 @@ +package web + +import ( + "fmt" + "net/http" + "sort" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" + "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" +) + +// The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update +// state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own +// records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines. + +// cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire). +type cell struct { + Text string + Class string + Title string +} + +type systemRow struct { + HostID, CustomerName string + Ring int + Enabled bool + Tunnel cell + HasFacts bool + FactsNote string + // host + PVE, KernelRunning, KernelNextBoot, HostDebian cell + HostRelease, HostPending, HostNotCovered cell + Held, RebootSince, KernelPanic, Oops cell + CrashRestarts24h, Guard cell + // guest + GuestDebian, GuestRelease, GuestPending, GuestRestart cell + // docker + Engine, Containerd, LiveRestore, DockerRelease cell + // last leg + LastLeg cell +} + +// OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service). +type OSSystemView interface { + Fleet() ([]osupdates.FleetLine, error) + Releases() []osupdates.ReleaseInfo + Candidates() []osupdates.Status + Thresholds() (stale, reboot, notCovered time.Duration) + ApproveDocker() (string, error) +} + +func plain(s string) cell { return cell{Text: s} } + +func unknownCell(s string) cell { + if s == "" || s == sysfacts.Unknown { + return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"} + } + return cell{Text: s} +} + +func ago(t time.Time, now time.Time) string { + if t.IsZero() { + return "never" + } + d := now.Sub(t) + switch { + case d < time.Hour: + return fmt.Sprintf("%d min ago", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%d h ago", int(d.Hours())) + } + return fmt.Sprintf("%d days ago", int(d.Hours()/24)) +} + +// buildSystemRows is pure (the render test feeds it directly). +func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string, + stale, reboot, notCov time.Duration, now time.Time) []systemRow { + var rows []systemRow + for _, l := range lines { + f := facts[l.HostID] + r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present} + switch l.Tunnel { + case "running": + r.Tunnel = plain("running") + case "not_running", "inactive": + r.Tunnel = cell{Text: l.Tunnel, Class: "bad"} + default: + r.Tunnel = cell{Text: l.Tunnel, Class: "warn"} + } + if !f.Present { + r.FactsNote = "no versions reported (agent older than v0.142.0)" + } else if f.FactsError != "" { + r.FactsNote = "partial: " + f.FactsError + } + r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion)) + r.KernelRunning = unknownCell(f.Host.KernelRunning) + r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot) + if f.NextBootDiffers() { + r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")" + } + r.HostDebian = unknownCell(f.Host.Debian) + r.HostRelease = plain(orDash(l.Host.ReleaseID)) + r.HostPending = plain(fmt.Sprint(l.Host.Pending)) + r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast)) + if l.Host.NotCoveredFast > 0 { + r.HostNotCovered.Class = "warn" + } + switch { + case f.Host.Held == nil: + r.Held = unknownCell("") + case len(f.Host.Held) == 0: + r.Held = plain("none") + default: + r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"} + } + if l.Host.RebootNeededSince.IsZero() { + r.RebootSince = plain("no") + } else { + r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"} + if now.Sub(l.Host.RebootNeededSince) >= reboot { + r.RebootSince.Class = "bad" + } + } + if f.Host.KernelPanic != nil { + r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic)) + if *f.Host.KernelPanic == 0 { + r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"} + } + } else { + r.KernelPanic = unknownCell("") + } + r.Oops = plain("no") + if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot { + r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"} + } + if cg := f.Host.CrashGuard; cg != nil { + r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h)) + if cg.In24h > 0 { + r.CrashRestarts24h.Class = "warn" + } + if cg.Tripped { + r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason} + } else { + r.Guard = plain("armed") + } + } else { + r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"} + } + r.GuestDebian = unknownCell(f.Guest.Debian) + r.GuestRelease = plain(orDash(l.Guest.ReleaseID)) + r.GuestPending = plain(fmt.Sprint(l.Guest.Pending)) + r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded)) + r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd) + r.LiveRestore = unknownCell(f.Guest.LiveRestore) + if f.Guest.LiveRestore == "off" { + r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)" + } + r.DockerRelease = plain(orDash(l.Docker.ReleaseID)) + last := l.Guest + if l.Host.LastAt.After(last.LastAt) { + last = l.Host + } + if l.Docker.LastAt.After(last.LastAt) { + last = l.Docker + } + ok := l.Guest.LastSuccessfulLeg + r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds), + Title: "last successful leg: " + ago(ok, now)} + if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale { + r.LastLeg.Class = "bad" + } else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" { + r.LastLeg.Class = "warn" + } + rows = append(rows, r) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID }) + return rows +} + +func orDash(s string) string { + if s == "" { + return "—" + } + return s +} + +func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) { + view, ok := s.osUpdates.(OSSystemView) + if s.osUpdates == nil || !ok { + http.Error(w, "os updates not configured", http.StatusServiceUnavailable) + return + } + lines, err := view.Fleet() + if err != nil { + s.logger.Printf("[ERROR] system page: fleet: %v", err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + hosts, _ := s.store.ListHosts() + facts, names := map[string]sysfacts.System{}, map[string]string{} + for _, h := range hosts { + names[h.HostID] = s.customerName(h.CustomerID) + if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" { + facts[h.HostID] = sysfacts.Parse(rj) + } + } + stale, reboot, notCov := view.Thresholds() + data := map[string]interface{}{ + "Rows": buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()), + "Releases": view.Releases(), + "Candidates": view.Candidates(), + "Flash": r.URL.Query().Get("flash"), + "FlashErr": r.URL.Query().Get("err"), + "CSRFToken": s.getCSRFToken(r), + } + if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil { + s.logger.Printf("[ERROR] system.html template: %v", err) + } +} diff --git a/hub/internal/web/system_test.go b/hub/internal/web/system_test.go new file mode 100644 index 00000000..a1b8016f --- /dev/null +++ b/hub/internal/web/system_test.go @@ -0,0 +1,155 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + "golang.org/x/crypto/bcrypt" +) + +const sysFull = `{"host":{"cpu_percent":1},"cloudflared":{"status":"running"},"system":{"pve_version":"pve-manager/9.0.11/abc", +"kernel_version":"Linux 7.0.14-20-pve #1","vmid":9201,"facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve", +"kernel_next_boot":"7.0.2-6-pve","kernel_next_boot_source":"saved default","held":["tzdata"],"kernel_panic":0,"oops_this_boot":false, +"crash_guard":{"armed":false,"tripped":true,"tripped_at":"2026-10-04T16:00:00Z","tripped_reason":"2 unclean boots within 60 minutes","unclean_boots_24h":2}}, +"guest":{"debian":"13.7","docker_engine":"29.8.2","containerd":"2.3.6-1~debian.13~trixie","live_restore":"on"}}}}` +const sysPartial = `{"host":{"cpu_percent":1},"system":{"pve_version":"pve-manager/9.0.10/x","kernel_version":"Linux 7.0.2-6-pve #1","facts_error":"no running customer guest"}}` +const sysOld = `{"host":{"cpu_percent":1}}` + +func systemServer(t *testing.T) (*Server, *store.Store, *osupdates.Service) { + s, st := newTestServer(t) + for _, h := range []struct{ id, cust, body string }{{"full-1", "c-full", sysFull}, {"part-1", "c-part", sysPartial}, {"old-1", "c-old", sysOld}} { + if err := st.UpsertHost(&store.Host{HostID: h.id, CustomerID: h.cust, APIKey: "k-" + h.id}); err != nil { + t.Fatal(err) + } + if err := st.SaveHostReport(h.id, h.cust, []byte(h.body), store.HostReportDenorm{AgentVersion: "0.142.0", CloudflaredStatus: "running"}); err != nil { + t.Fatal(err) + } + } + svc := &osupdates.Service{Store: st, ApproveAfter: 24 * time.Hour, NightsRequired: 1} + s.SetOSUpdateAdmin(svc) + return s, st, svc +} + +func getSystem(t *testing.T, s *Server) string { + t.Helper() + rr := httptest.NewRecorder() + s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/system", nil)) + if rr.Code != 200 { + t.Fatalf("GET /system = %d %s", rr.Code, rr.Body.String()) + } + return rr.Body.String() +} + +// Full, partial and old-agent boxes render; an unreadable value says "unknown", never empty or guessed. +// Red-proof: render KernelRunning with no unknownCell and the partial box shows an empty cell. +func TestSystemPage_FullPartialUnknown(t *testing.T) { + s, _, _ := systemServer(t) + b := getSystem(t, s) + for _, want := range []string{"9.0.11", "7.0.14-20-pve", "29.8.2", "tzdata", "TRIPPED", "0 (stays off)", + "9.0.10", "partial: no running customer guest", "no versions reported (agent older than v0.142.0)", + `action="/os/ring/full-1"`, `action="/os/enabled/part-1"`, `action="/os/approve-now"`} { + if !strings.Contains(b, want) { + t.Errorf("System page lacks %q", want) + } + } + if strings.Count(b, ">unknown<") < 6 { + t.Errorf("the partial and old boxes must read unknown, got %d unknown cells", strings.Count(b, ">unknown<")) + } + if strings.Count(b, `class="c-warn" title="the box could not read it`) < 6 { + t.Errorf("an unknown value must be shown amber with its reason, got %d", strings.Count(b, `class="c-warn" title="the box could not read it`)) + } + if !strings.Contains(b, `class="c-bad" title="2 unclean boots`) { + t.Error("a tripped crash guard must be red") + } +} + +// The "Approve Docker set" button appears ONLY when the rule allows it (seam rule: one render test per branch). +func TestSystemPage_DockerButtonOnlyWhenReady(t *testing.T) { + s, st, svc := systemServer(t) + if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) { + t.Fatal("button shown with no Docker candidate") + } + _ = st.SetOSRing("full-1", 0) + night := func() { + if err := svc.Ingest("full-1", osupdates.Report{RunID: time.Now().String(), Layer: "docker", Trigger: "night", Mode: "apply", + Outcome: "nothing", Healthy: true, Installed: []osupdates.Package{{Name: "docker-ce", Version: "5:29.8.2-1", Origin: "Docker"}}}); err != nil { + t.Fatal(err) + } + } + night() + if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) { + t.Fatal("button shown after ONE night") + } + night() + if !strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) { + t.Fatal("button missing after two healthy nights") + } +} + +// Every button needs the operator login; a box's own API key is not one. Red-proof: route /os/ outside RequireAuth. +func TestSystemButtons_NeedTheOperatorLogin(t *testing.T) { + s, st, _ := systemServer(t) + h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost) + s.configPasswordHash = string(h) + for _, tc := range []struct{ method, path, body string }{ + {http.MethodGet, "/system", ""}, + {http.MethodPost, "/os/ring/full-1", "ring=0&return=%2Fsystem"}, + {http.MethodPost, "/os/enabled/full-1", "on=0&return=%2Fsystem"}, + {http.MethodPost, "/os/approve-now", "return=%2Fsystem"}, + {http.MethodPost, "/os/approve-docker", "return=%2Fsystem"}, + } { + for _, auth := range []string{"", "Bearer k-full-1"} { + req := httptest.NewRequest(tc.method, tc.path, strings.NewReader(tc.body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Requested-With", "XMLHttpRequest") + if auth != "" { + req.Header.Set("Authorization", auth) + } + rr := httptest.NewRecorder() + s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req) + if rr.Code == http.StatusOK || rr.Code == http.StatusSeeOther { + t.Errorf("%s %s with auth %q = %d — must be refused", tc.method, tc.path, auth, rr.Code) + } + } + } + if st.GetOSHostSettings("full-1").Enabled != true || st.GetOSHostSettings("full-1").Ring != 1 { + t.Fatal("an unauthenticated POST changed a box") + } +} + +// With the operator's session a page button changes the box and returns to the page. +func TestSystemButtons_RedirectBackToThePage(t *testing.T) { + s, st, _ := systemServer(t) + h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost) + s.configPasswordHash = string(h) + cookie, csrf := newRevealSession(t, s) + form := url.Values{"on": {"0"}, "return": {"/system"}, "_csrf": {csrf}} + req := httptest.NewRequest(http.MethodPost, "/os/enabled/full-1", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(cookie) + rr := httptest.NewRecorder() + s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req) + if rr.Code != http.StatusSeeOther || !strings.HasPrefix(rr.Header().Get("Location"), "/system?flash=") { + t.Fatalf("= %d %q", rr.Code, rr.Header().Get("Location")) + } + if st.GetOSHostSettings("full-1").Enabled { + t.Fatal("the switch did not change") + } +} + +// The Hosts page shows the Proxmox version and the running kernel (unknown for an old agent). +func TestHostsPage_ProxmoxKernelColumn(t *testing.T) { + s, _, _ := systemServer(t) + rr := httptest.NewRecorder() + s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/hosts", nil)) + b := rr.Body.String() + if !strings.Contains(b, "Proxmox / kernel") || !strings.Contains(b, "9.0.11") || !strings.Contains(b, "7.0.2-6-pve") { + t.Fatalf("hosts column missing:\n%s", b[:min(len(b), 400)]) + } +} diff --git a/hub/internal/web/templates/app_detail.html b/hub/internal/web/templates/app_detail.html index 7431c7bf..cc52a990 100644 --- a/hub/internal/web/templates/app_detail.html +++ b/hub/internal/web/templates/app_detail.html @@ -18,6 +18,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/apps.html b/hub/internal/web/templates/apps.html index 738458be..28561f64 100644 --- a/hub/internal/web/templates/apps.html +++ b/hub/internal/web/templates/apps.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/config_form.html b/hub/internal/web/templates/config_form.html index 53b8a586..ece1a6a1 100644 --- a/hub/internal/web/templates/config_form.html +++ b/hub/internal/web/templates/config_form.html @@ -17,6 +17,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/configs.html b/hub/internal/web/templates/configs.html index 5a7ab757..d689c014 100644 --- a/hub/internal/web/templates/configs.html +++ b/hub/internal/web/templates/configs.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/configuration.html b/hub/internal/web/templates/configuration.html index f726e033..579e5159 100644 --- a/hub/internal/web/templates/configuration.html +++ b/hub/internal/web/templates/configuration.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/customer_unified.html b/hub/internal/web/templates/customer_unified.html index 96974393..08bcfb17 100644 --- a/hub/internal/web/templates/customer_unified.html +++ b/hub/internal/web/templates/customer_unified.html @@ -18,6 +18,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/dashboard.html b/hub/internal/web/templates/dashboard.html index 92cf7d2f..24c9c3a2 100644 --- a/hub/internal/web/templates/dashboard.html +++ b/hub/internal/web/templates/dashboard.html @@ -17,6 +17,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/host_detail.html b/hub/internal/web/templates/host_detail.html index f3e5dc90..ddbaab28 100644 --- a/hub/internal/web/templates/host_detail.html +++ b/hub/internal/web/templates/host_detail.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/hosts.html b/hub/internal/web/templates/hosts.html index 68ae2a20..20e82295 100644 --- a/hub/internal/web/templates/hosts.html +++ b/hub/internal/web/templates/hosts.html @@ -17,6 +17,7 @@ Customers Apps Hosts + System Offsite Configuration @@ -85,6 +86,7 @@ Host Customer Agent + Proxmox / kernel Status Guests CPU @@ -100,6 +102,7 @@ {{.HostID}} {{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}} {{if .AgentVersion}}{{.AgentVersion}}{{else}}—{{end}}{{if .FloorHeld}} floor held{{else if eq .FloorSource "declared"}} floor: declared MinAgent{{end}} + {{.PVEVersion}}
{{.KernelRunning}} {{.StatusLabel}} {{if .HasReport}}{{.GuestRunning}}/{{.GuestTotal}}{{else}}—{{end}} {{if .HasReport}}{{formatFloat .Vitals.CPUPercent}}%{{else}}—{{end}} diff --git a/hub/internal/web/templates/log_tail.html b/hub/internal/web/templates/log_tail.html index b4f57c41..9451eaf5 100644 --- a/hub/internal/web/templates/log_tail.html +++ b/hub/internal/web/templates/log_tail.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/offsite.html b/hub/internal/web/templates/offsite.html index f7b90fdc..bccf6622 100644 --- a/hub/internal/web/templates/offsite.html +++ b/hub/internal/web/templates/offsite.html @@ -16,6 +16,7 @@ Customers Apps Hosts + System Offsite Configuration diff --git a/hub/internal/web/templates/system.html b/hub/internal/web/templates/system.html new file mode 100644 index 00000000..9ab5a816 --- /dev/null +++ b/hub/internal/web/templates/system.html @@ -0,0 +1,128 @@ + + + + + + System — Felhom Hub + + + + + {{template "icon_sprite"}} + {{template "inline_confirm_js"}} +
+
+

Felhom Hub

+ +
+ +

System — versions and OS updates

+ + {{if .Flash}}
{{.Flash}}
{{end}} + {{if .FlashErr}}
{{.FlashErr}}
{{end}} + +
+

Approved releases

+
+ {{range .Releases}} +
{{.Layer}}: {{.ID}}
+ {{.Packages}} packages · {{.ApprovedAt.UTC.Format "2006-01-02 15:04"}} UTC · by {{.ApprovedBy}}
+ {{else}}
No release approved yet.
{{end}} +
+

What ring 0 runs now

+
+ {{range .Candidates}} +
{{.Layer}}: + {{if .Fingerprint}}{{.Packages}} packages, first seen {{.FirstSeen.UTC.Format "2006-01-02 15:04"}} UTC{{else}}—{{end}}
+ {{if .Approved}}approved as {{.Approved}} + {{else if .Waiting}}{{.Waiting}}{{end}} + {{if and (eq .Layer "docker") .Fingerprint (not .Approved) (eq .Waiting "")}} +
+ + +
{{end}} +
+ {{end}} +
+
+ + + An urgent fix only — normally the hub approves after 24 h and one night. +
+
+ + {{if .Rows}} +
+ + + + + + + + + + + + + {{range .Rows}} + + + + {{template "sys_cell" .Tunnel}} + + {{template "sys_cell" .KernelRunning}}{{template "sys_cell" .KernelNextBoot}}{{template "sys_cell" .HostDebian}} + {{template "sys_cell" .HostRelease}}{{template "sys_cell" .HostPending}}{{template "sys_cell" .HostNotCovered}} + {{template "sys_cell" .Held}}{{template "sys_cell" .RebootSince}}{{template "sys_cell" .KernelPanic}}{{template "sys_cell" .Oops}} + {{template "sys_cell" .CrashRestarts24h}}{{template "sys_cell" .Guard}} + + {{template "sys_cell" .GuestRelease}}{{template "sys_cell" .GuestPending}}{{template "sys_cell" .GuestRestart}} + + {{template "sys_cell" .Containerd}}{{template "sys_cell" .LiveRestore}}{{template "sys_cell" .DockerRelease}} + + + {{end}} + +
BoxRing / updatesTunnelProxmoxKernel (running)Kernel (next boot)DebianFelhom releasePendingNot coveredHeldReboot neededkernel.panicOopsCrash restarts 24 hCrash guardGuest DebianFelhom releasePendingRestart neededDockercontainerdlive-restoreDocker releaseLast OS leg
hostguestDocker engine
{{.HostID}}{{if .CustomerName}}
{{.CustomerName}}{{end}} + {{if .FactsNote}}
{{.FactsNote}}{{end}}		 + ring {{.Ring}} +
+ + {{if eq .Ring 0}} + {{else}}{{end}} +

+ updates {{if .Enabled}}ON{{else}}OFF{{end}} +
+ + {{if .Enabled}} + {{else}}{{end}} +
+ 		{{.PVE.Text}}{{.GuestDebian.Text}}{{.Engine.Text}}{{.LastLeg.Text}}
+
+

Amber: worth a look. Red: an operator alarm fires (`08` §6.3). "unknown": the box could not read the value — never a guess.

+ {{else}} +

No boxes yet.

+ {{end}} + +
+ Felhom Hub {{hubVersion}} +
+
+ + +{{define "sys_cell"}}{{.Text}}{{end}}