hub v0.132.0: the System page (versions + OS updates with the ring/switch/approve buttons, R-852), the Hosts Proxmox/kernel column, the operator-approved Docker engine release (2 healthy ring-0 nights), the crash-guard events (R-851); evidence audits/os-docker-crash-2026-10-04

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/api/crash.go

@@ -0,0 +1,104 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
+)
+
+// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the
+// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the
+// texts say "an unexpected stop".
+const (
+	EventHostCrashRestart        = "host_crash_restart"         // warning, operator: the box restarted after an unclean stop
+	EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed)
+	EventCrashGuardTripped       = "host_crash_guard_tripped"   // error, operator: the next crash leaves the box off
+	EventCrashGuardRearmed       = "host_crash_guard_rearmed"   // info, operator
+	EventKernelOops              = "host_kernel_oops"           // warning, operator: a kernel oops this boot (no restart)
+)
+
+type crashSeen struct {
+	Boots     []string `json:"boots"`
+	TrippedAt string   `json:"tripped_at"`
+	RearmedAt string   `json:"rearmed_at"`
+	OopsBoot  string   `json:"oops_boot"`
+}
+
+// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is
+// there (no flood of old history). Pinned by crash_test.go.
+func (h *Handler) checkCrash(hostID, custID string, body []byte) {
+	sys := sysfacts.Parse(string(body))
+	cg := sys.Host.CrashGuard
+	if !sys.Present || cg == nil {
+		return
+	}
+	var seen crashSeen
+	raw := h.store.CrashSeen(hostID)
+	first := raw == ""
+	if !first {
+		_ = json.Unmarshal([]byte(raw), &seen)
+	}
+	known := map[string]bool{}
+	for _, b := range seen.Boots {
+		known[b] = true
+	}
+	emit := func(typ, sev, msg string, extra map[string]any) {
+		if first {
+			return
+		}
+		d := map[string]any{"host_id": hostID}
+		for k, v := range extra {
+			d[k] = v
+		}
+		dj, _ := json.Marshal(d)
+		h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ)
+		if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
+			h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
+		} else if h.dispatcher != nil {
+			go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
+		}
+	}
+	for _, b := range cg.UncleanBoots {
+		if known[b] {
+			continue
+		}
+		emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.",
+			hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h})
+		emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
+			map[string]any{"boot_at": b})
+		seen.Boots = append(seen.Boots, b)
+	}
+	if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt {
+		emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.",
+			hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt})
+		seen.TrippedAt = cg.TrippedAt
+	}
+	if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt {
+		emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy),
+			map[string]any{"rearmed_at": cg.RearmedAt})
+		seen.RearmedAt = cg.RearmedAt
+	}
+	if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot {
+		emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID),
+			map[string]any{"boot_id": cg.BootID})
+		seen.OopsBoot = cg.BootID
+	}
+	if first {
+		seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt
+	}
+	if len(seen.Boots) > 200 {
+		seen.Boots = seen.Boots[len(seen.Boots)-200:]
+	}
+	sj, _ := json.Marshal(seen)
+	if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil {
+		h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err)
+	}
+}
+
+func intOr(p *int, d string) string {
+	if p == nil {
+		return d
+	}
+	return fmt.Sprint(*p)
+}

hub/internal/api/crash_test.go

@@ -0,0 +1,67 @@
+package api
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func crashBody(boots []string, tripped bool, trippedAt, rearmedAt, bootID string, oops bool) []byte {
+	q := func(ss []string) string {
+		if len(ss) == 0 {
+			return "[]"
+		}
+		return `["` + strings.Join(ss, `","`) + `"]`
+	}
+	return []byte(fmt.Sprintf(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts":{"host":{"oops_this_boot":%v,"kernel_panic":10,
+"crash_guard":{"unclean_boots":%s,"unclean_boots_24h":%d,"tripped":%v,"tripped_at":%q,"tripped_reason":"2 unclean boots within 60 minutes",
+"rearmed_at":%q,"rearmed_by":"operator","boot_id":%q}},"guest":{}}}}`, oops, q(boots), len(boots), tripped, trippedAt, rearmedAt, bootID))
+}
+
+func countEv(t *testing.T, h *Handler, cust, typ string) int {
+	t.Helper()
+	ev, err := h.store.GetEventsByType(cust, typ, time.Now().Add(-time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return len(ev)
+}
+
+// The first report only records history (no flood); each NEW unclean boot is one operator event + one household line;
+// the trip is one alarm; a repeat report is quiet. Red-proof: drop the `known[b]` skip and the repeat report emits again.
+func TestCheckCrash_EventsOncePerFact(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	seedHost(t, st, "h1", "c1", "K1")
+	old := []string{"2026-10-01T10:00:00Z"}
+	h.checkCrash("h1", "c1", crashBody(old, false, "", "", "b0", false))
+	if n := countEv(t, h, "c1", EventHostCrashRestart); n != 0 {
+		t.Fatalf("first sighting flooded %d events", n)
+	}
+	two := append(old, "2026-10-04T15:40:00Z")
+	h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false))
+	h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false)) // the same report again
+	if countEv(t, h, "c1", EventHostCrashRestart) != 1 || countEv(t, h, "c1", EventHostRestartedAfterCrash) != 1 {
+		t.Fatalf("restart events: op=%d household=%d", countEv(t, h, "c1", EventHostCrashRestart), countEv(t, h, "c1", EventHostRestartedAfterCrash))
+	}
+	three := append(two, "2026-10-04T15:45:00Z")
+	h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
+	h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
+	if countEv(t, h, "c1", EventCrashGuardTripped) != 1 {
+		t.Fatalf("tripped events = %d", countEv(t, h, "c1", EventCrashGuardTripped))
+	}
+	h.checkCrash("h1", "c1", crashBody(three, false, "", "2026-10-04T16:10:00Z", "b2", true))
+	if countEv(t, h, "c1", EventCrashGuardRearmed) != 1 || countEv(t, h, "c1", EventKernelOops) != 1 {
+		t.Fatal("re-arm / oops not announced")
+	}
+}
+
+// An agent older than v0.142.0 (no system stanza) never produces a crash event.
+func TestCheckCrash_NoStanzaIsQuiet(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	seedHost(t, st, "h1", "c1", "K1")
+	h.checkCrash("h1", "c1", []byte(`{"host":{}}`))
+	if st.CrashSeen("h1") != "" {
+		t.Fatal("a report without the stanza recorded crash state")
+	}
+}

hub/internal/api/handler.go

@@ -863,6 +863,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
+	h.checkCrash(hostID, custID, body)
 
 	for _, g := range rep.Guests {
 		status := g.Status