hub v0.132.0: the System page (versions + OS updates with the ring/switch/approve buttons, R-852), the Hosts Proxmox/kernel column, the operator-approved Docker engine release (2 healthy ring-0 nights), the crash-guard events (R-851); evidence audits/os-docker-crash-2026-10-04

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-04 16:16:02 +02:00
parent bee277ffad
commit 175ecfcdd2
45 changed files with 2198 additions and 12 deletions
@@ -0,0 +1,104 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
+)
+
+// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the
+// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the
+// texts say "an unexpected stop".
+const (
+	EventHostCrashRestart        = "host_crash_restart"         // warning, operator: the box restarted after an unclean stop
+	EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed)
+	EventCrashGuardTripped       = "host_crash_guard_tripped"   // error, operator: the next crash leaves the box off
+	EventCrashGuardRearmed       = "host_crash_guard_rearmed"   // info, operator
+	EventKernelOops              = "host_kernel_oops"           // warning, operator: a kernel oops this boot (no restart)
+)
+
+type crashSeen struct {
+	Boots     []string `json:"boots"`
+	TrippedAt string   `json:"tripped_at"`
+	RearmedAt string   `json:"rearmed_at"`
+	OopsBoot  string   `json:"oops_boot"`
+}
+
+// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is
+// there (no flood of old history). Pinned by crash_test.go.
+func (h *Handler) checkCrash(hostID, custID string, body []byte) {
+	sys := sysfacts.Parse(string(body))
+	cg := sys.Host.CrashGuard
+	if !sys.Present || cg == nil {
+		return
+	}
+	var seen crashSeen
+	raw := h.store.CrashSeen(hostID)
+	first := raw == ""
+	if !first {
+		_ = json.Unmarshal([]byte(raw), &seen)
+	}
+	known := map[string]bool{}
+	for _, b := range seen.Boots {
+		known[b] = true
+	}
+	emit := func(typ, sev, msg string, extra map[string]any) {
+		if first {
+			return
+		}
+		d := map[string]any{"host_id": hostID}
+		for k, v := range extra {
+			d[k] = v
+		}
+		dj, _ := json.Marshal(d)
+		h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ)
+		if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
+			h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
+		} else if h.dispatcher != nil {
+			go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
+		}
+	}
+	for _, b := range cg.UncleanBoots {
+		if known[b] {
+			continue
+		}
+		emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.",
+			hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h})
+		emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
+			map[string]any{"boot_at": b})
+		seen.Boots = append(seen.Boots, b)
+	}
+	if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt {
+		emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.",
+			hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt})
+		seen.TrippedAt = cg.TrippedAt
+	}
+	if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt {
+		emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy),
+			map[string]any{"rearmed_at": cg.RearmedAt})
+		seen.RearmedAt = cg.RearmedAt
+	}
+	if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot {
+		emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID),
+			map[string]any{"boot_id": cg.BootID})
+		seen.OopsBoot = cg.BootID
+	}
+	if first {
+		seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt
+	}
+	if len(seen.Boots) > 200 {
+		seen.Boots = seen.Boots[len(seen.Boots)-200:]
+	}
+	sj, _ := json.Marshal(seen)
+	if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil {
+		h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err)
+	}
+}
+
+func intOr(p *int, d string) string {
+	if p == nil {
+		return d
+	}
+	return fmt.Sprint(*p)
+}