package api import ( "encoding/json" "fmt" "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the // host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the // texts say "an unexpected stop". const ( EventHostCrashRestart = "host_crash_restart" // warning, operator: the box restarted after an unclean stop EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed) EventCrashGuardTripped = "host_crash_guard_tripped" // error, operator: the next crash leaves the box off EventCrashGuardRearmed = "host_crash_guard_rearmed" // info, operator EventKernelOops = "host_kernel_oops" // warning, operator: a kernel oops this boot (no restart) ) type crashSeen struct { Boots []string `json:"boots"` TrippedAt string `json:"tripped_at"` RearmedAt string `json:"rearmed_at"` OopsBoot string `json:"oops_boot"` } // checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is // there (no flood of old history). Pinned by crash_test.go. func (h *Handler) checkCrash(hostID, custID string, body []byte) { sys := sysfacts.Parse(string(body)) cg := sys.Host.CrashGuard if !sys.Present || cg == nil { return } var seen crashSeen raw := h.store.CrashSeen(hostID) first := raw == "" if !first { _ = json.Unmarshal([]byte(raw), &seen) } known := map[string]bool{} for _, b := range seen.Boots { known[b] = true } emit := func(typ, sev, msg string, extra map[string]any) { if first { return } d := map[string]any{"host_id": hostID} for k, v := range extra { d[k] = v } dj, _ := json.Marshal(d) h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ) if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil { h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err) } else if h.dispatcher != nil { go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub") } } for _, b := range cg.UncleanBoots { if known[b] { continue } emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.", hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h}) emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.", map[string]any{"boot_at": b}) seen.Boots = append(seen.Boots, b) } if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt { emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.", hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt}) seen.TrippedAt = cg.TrippedAt } if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt { emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy), map[string]any{"rearmed_at": cg.RearmedAt}) seen.RearmedAt = cg.RearmedAt } if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot { emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID), map[string]any{"boot_id": cg.BootID}) seen.OopsBoot = cg.BootID } if first { seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt } if len(seen.Boots) > 200 { seen.Boots = seen.Boots[len(seen.Boots)-200:] } sj, _ := json.Marshal(seen) if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil { h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err) } } func intOr(p *int, d string) string { if p == nil { return d } return fmt.Sprint(*p) }