felhom.eu/hub/internal/api/crash.go

package api

import (
	"encoding/json"
	"fmt"

	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)

// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the
// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the
// texts say "an unexpected stop".
const (
	EventHostCrashRestart        = "host_crash_restart"         // warning, operator: the box restarted after an unclean stop
	EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed)
	EventCrashGuardTripped       = "host_crash_guard_tripped"   // error, operator: the next crash leaves the box off
	EventCrashGuardRearmed       = "host_crash_guard_rearmed"   // info, operator
	EventKernelOops              = "host_kernel_oops"           // warning, operator: a kernel oops this boot (no restart)
)

type crashSeen struct {
	Boots     []string `json:"boots"`
	TrippedAt string   `json:"tripped_at"`
	RearmedAt string   `json:"rearmed_at"`
	OopsBoot  string   `json:"oops_boot"`
}

// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is
// there (no flood of old history). Pinned by crash_test.go.
func (h *Handler) checkCrash(hostID, custID string, body []byte) {
	sys := sysfacts.Parse(string(body))
	cg := sys.Host.CrashGuard
	if !sys.Present || cg == nil {
		return
	}
	var seen crashSeen
	raw := h.store.CrashSeen(hostID)
	first := raw == ""
	if !first {
		_ = json.Unmarshal([]byte(raw), &seen)
	}
	known := map[string]bool{}
	for _, b := range seen.Boots {
		known[b] = true
	}
	emit := func(typ, sev, msg string, extra map[string]any) {
		if first {
			return
		}
		d := map[string]any{"host_id": hostID}
		for k, v := range extra {
			d[k] = v
		}
		dj, _ := json.Marshal(d)
		h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ)
		if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
			h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
		} else if h.dispatcher != nil {
			go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
		}
	}
	for _, b := range cg.UncleanBoots {
		if known[b] {
			continue
		}
		emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.",
			hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h})
		emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
			map[string]any{"boot_at": b})
		seen.Boots = append(seen.Boots, b)
	}
	if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt {
		emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.",
			hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt})
		seen.TrippedAt = cg.TrippedAt
	}
	if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt {
		emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy),
			map[string]any{"rearmed_at": cg.RearmedAt})
		seen.RearmedAt = cg.RearmedAt
	}
	if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot {
		emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID),
			map[string]any{"boot_id": cg.BootID})
		seen.OopsBoot = cg.BootID
	}
	if first {
		seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt
	}
	if len(seen.Boots) > 200 {
		seen.Boots = seen.Boots[len(seen.Boots)-200:]
	}
	sj, _ := json.Marshal(seen)
	if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil {
		h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err)
	}
}

func intOr(p *int, d string) string {
	if p == nil {
		return d
	}
	return fmt.Sprint(*p)
}