felhom.eu/hub/internal/osupdates/kernel.go

package osupdates

// The kernel lane, hub half (hub v0.143.0; R-836, `09` §3 decision 172, `11` §5.11). The agent stages a kernel, boots
// it ONCE through a flag on the ESP, and makes it the default only after a healthy boot (felhom-agent internal/osupdate
// kernel.go + the wrapper's layer "kernel"). The hub decides WHEN:
//
//   - A box is DUE when ring 0 has a pending kernel (its host report's `proxmox-kernel-X.Y` upgrade), or a ring-1 box
//     runs a kernel a signed os_kernel_step STAGED — and no step for that kernel has ended yet (an ended step is the
//     operator's to judge; the hub never retries it by itself).
//   - The household of a due box gets ONE mail the day before, in its language, between 09:00 and 20:00 Budapest
//     time (KernelNotify): the box restarts tonight; if it is not back by morning, unplug it, wait 10 seconds, plug it
//     back in. Only a mail the mail service ACCEPTED is recorded.
//   - The box's os_update block carries `kernel: {kver, tonight}`; tonight is true only within 24 h of such a mail. No
//     mail → no step: the agent's night leg refuses a kernel without `tonight`.
//   - The operator approves a kernel set ("Approve kernel set") once every ring-0 box booted it healthily as the
//     default after a night step; a ring-1 box takes it only through a signed os_kernel_step.
//
// Pinned by kernel_test.go.

import (
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)

// LayerKernel is the host's kernel (agent v0.152.0).
const LayerKernel = "kernel"

// Kernel-lane events — operator only (the household's one word is the day-before mail).
const (
	EventKernelStep   = "os_kernel_step"   // what became of a kernel step (staged, the default moved, fell back, …)
	EventKernelNotice = "os_kernel_notice" // the household was told — or could not be told (then no step)
)

// The day-before mail's window and limits (decided by CC — operator may reverse): mailed only in the household's
// daytime, valid for the night that follows, never twice within 20 h, at most 3 times for one kernel on one box.
const (
	KernelNoticeFromHour = 9
	KernelNoticeToHour   = 20
	KernelNoticeValid    = 24 * time.Hour
	KernelNoticeGap      = 20 * time.Hour
	KernelNoticeMax      = 3
)

// KernelBlock is a box's kernel instruction (felhom-agent hub.WireKernelStep — field-exact, cross-repo).
type KernelBlock struct {
	Kver       string `json:"kver"`
	Tonight    bool   `json:"tonight"`
	NotifiedAt string `json:"notified_at,omitempty"`
}

var (
	kverRE       = regexp.MustCompile(`^([0-9]+\.[0-9]+)\.[0-9]+-[0-9]+-pve$`)
	kernelMetaRE = regexp.MustCompile(`^proxmox-kernel-[0-9]+\.[0-9]+$`)
)

// kernelEnded are the outcomes after which a kernel is never stepped again by itself (the operator decides).
var kernelEnded = map[string]bool{"applied": true, "fell_back": true, "health_failed": true, "self_reverted": true, "revert_failed": true}

// refusals that will not go away by waiting a night (the box cannot do a one-shot, the set is wrong, no authority,
// not an appliance) — such a refusal ends the step too. A transient one (a lock, the crash guard's window) is retried
// within KernelNoticeMax mails.
var kernelRefusedForGood = map[string]bool{"R20": true, "R23": true, "R3": true, "R12": true}

func budapestLoc() *time.Location {
	if l, err := time.LoadLocation("Europe/Budapest"); err == nil {
		return l
	}
	return time.FixedZone("CET", 3600)
}

// kernelOf is the kernel a kernel-layer report is about: the wrapper's view's "to", else the release id.
func kernelOf(r Report) string {
	var v struct {
		To string `json:"to"`
	}
	if len(r.Kernel) > 0 && json.Unmarshal(r.Kernel, &v) == nil && kverRE.MatchString(v.To) {
		return v.To
	}
	if kverRE.MatchString(r.ReleaseID) {
		return r.ReleaseID
	}
	return ""
}

func refusedCode(raw json.RawMessage) string {
	var c struct {
		Code string `json:"code"`
	}
	_ = json.Unmarshal(raw, &c)
	return c.Code
}

// kernelLane is the box's newest kernel-lane facts (nil: an older agent, or no host report).
func (s *Service) kernelLane(hostID string) *sysfacts.KernelLane {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return nil
	}
	rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID)
	if rj == "" {
		return nil
	}
	return sysfacts.Parse(rj).Host.KernelLane
}

// KernelDue says which kernel the box is due to step to, or why none.
func (s *Service) KernelDue(hostID string) (kver, why string) {
	st := s.Store.GetOSHostSettings(hostID)
	if !st.Enabled {
		return "", "OS updates are switched off"
	}
	rep, _ := s.Store.LatestOSReport(hostID, LayerHost)
	if rep == nil {
		return "", "no host step reported (the kernel lane is for appliances)"
	}
	lane := s.kernelLane(hostID)
	if lane == nil {
		return "", "the box reports no kernel lane (agent older than v0.152.0)"
	}
	if len(lane.SetupProblems) > 0 {
		return "", "the box cannot do a one-shot boot: " + strings.Join(lane.SetupProblems, "; ")
	}
	if st.Ring == 1 {
		if lane.Phase != "staged" || !kverRE.MatchString(lane.To) {
			return "", "ring 1: no kernel staged by a signed os_kernel_step"
		}
		kver = lane.To
	} else {
		var hr Report
		_ = json.Unmarshal([]byte(rep.ReportJSON), &hr)
		for _, p := range hr.Pending {
			if kernelMetaRE.MatchString(p.Name) && p.From != "" && kverRE.MatchString(p.To+"-pve") {
				kver = p.To + "-pve"
			}
		}
		if kver == "" && lane.Phase == "staged" && kverRE.MatchString(lane.To) {
			kver = lane.To
		}
		if kver == "" {
			return "", "no pending kernel"
		}
	}
	if lane.Running == kver {
		return "", "the box already runs " + kver
	}
	reps, _ := s.Store.OSReportsDesc(hostID, LayerKernel, 100)
	for _, r := range reps {
		var kr Report
		if json.Unmarshal([]byte(r.ReportJSON), &kr) != nil || kernelOf(kr) != kver {
			continue
		}
		if kernelEnded[r.Outcome] {
			return "", fmt.Sprintf("the kernel step to %s ended %s at %s — the operator decides", kver, r.Outcome,
				r.ReceivedAt.UTC().Format("2006-01-02 15:04"))
		}
		if r.Outcome == "refused" && kernelRefusedForGood[refusedCode(kr.Refused)] {
			return "", fmt.Sprintf("the kernel step to %s was refused (%s) — the operator decides", kver, refusedCode(kr.Refused))
		}
	}
	return kver, ""
}

// KernelBlockFor is the box's kernel instruction (nil: not due).
func (s *Service) KernelBlockFor(hostID string) *KernelBlock {
	kver, _ := s.KernelDue(hostID)
	if kver == "" {
		return nil
	}
	b := &KernelBlock{Kver: kver}
	ns, _ := s.Store.KernelNotices(hostID, kver)
	if len(ns) > 0 {
		last := ns[len(ns)-1].SentAt
		if s.now().Sub(last) < KernelNoticeValid {
			b.Tonight, b.NotifiedAt = true, last.UTC().Format(time.RFC3339)
		}
	}
	return b
}

// KernelNotify tells each due box's household, the day before (`09` §3 decision 172). Called every minute; it acts only
// between KernelNoticeFromHour and KernelNoticeToHour Budapest time. Returns the boxes it told.
func (s *Service) KernelNotify() []string {
	now := s.now()
	lt := now.In(budapestLoc())
	if lt.Hour() < KernelNoticeFromHour || lt.Hour() >= KernelNoticeToHour {
		return nil
	}
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil
	}
	var told []string
	for _, h := range hosts {
		kver, _ := s.KernelDue(h.HostID)
		if kver == "" {
			continue
		}
		ns, _ := s.Store.KernelNotices(h.HostID, kver)
		if len(ns) > 0 && now.Sub(ns[len(ns)-1].SentAt) < KernelNoticeGap {
			continue
		}
		if len(ns) >= KernelNoticeMax {
			key := "kernel_notice_max:" + h.HostID + ":" + kver
			if s.Store.OSAlarmRaised(key).IsZero() {
				_ = s.Store.SetOSAlarmRaised(key, now)
				s.event(h.CustomerID, EventKernelNotice, "warning", fmt.Sprintf("Kernel %s on %s: the household was told %d times "+
					"and no kernel step ran — no further mail; the operator decides (the System page shows why).", kver, h.HostID, len(ns)),
					map[string]any{"host_id": h.HostID, "kver": kver, "notices": len(ns)})
			}
			continue
		}
		if s.KernelMail == nil {
			continue
		}
		lang, err := s.KernelMail(h.CustomerID, kver)
		if err != nil {
			key := "kernel_notice_failed:" + h.HostID + ":" + kver + ":" + lt.Format("2006-01-02")
			if s.Store.OSAlarmRaised(key).IsZero() {
				_ = s.Store.SetOSAlarmRaised(key, now)
				s.event(h.CustomerID, EventKernelNotice, "warning", fmt.Sprintf("Kernel %s on %s: the household could NOT be told "+
					"(%v) — so no kernel step tonight (no mail, no step).", kver, h.HostID, err),
					map[string]any{"host_id": h.HostID, "kver": kver, "error": err.Error()})
			}
			continue
		}
		if err := s.Store.SaveKernelNotice(store.KernelNotice{HostID: h.HostID, Kver: kver, SentAt: now, Lang: lang}); err != nil {
			s.logf("[ERROR] osupdates: kernel notice for %s sent but NOT recorded (%v) — no step tonight", h.HostID, err)
			continue
		}
		told = append(told, h.HostID)
		s.logf("[INFO] osupdates: kernel %s — the household of %s was told the box restarts tonight (lang=%s)", kver, h.HostID, lang)
		s.event(h.CustomerID, EventKernelNotice, "info", fmt.Sprintf("Kernel %s on %s: the household was told the box "+
			"restarts tonight (mail %d of at most %d).", kver, h.HostID, len(ns)+1, KernelNoticeMax),
			map[string]any{"host_id": h.HostID, "kver": kver, "lang": lang})
		if s.Bump != nil {
			s.Bump(h.HostID)
		}
	}
	sort.Strings(told)
	return told
}

// kernelIngest raises the kernel layer's operator events (called by Ingest for layer "kernel").
func (s *Service) kernelIngest(hostID, customerID string, r Report, details map[string]any) {
	var v struct {
		From, To, Reason string
	}
	_ = json.Unmarshal(r.Kernel, &v)
	to := kernelOf(r)
	details["kver"] = to
	switch r.Outcome {
	case "staged":
		s.event(customerID, EventKernelStep, "info", fmt.Sprintf("Kernel %s staged on %s (trigger %s): installed, never the "+
			"default; the box boots it ONCE at its night reboot.", to, hostID, r.Trigger), details)
	case "applied":
		s.event(customerID, EventKernelStep, "info", fmt.Sprintf("Kernel %s booted healthily on %s and is the default now "+
			"(was %s). %s", to, hostID, v.From, r.HealthReason), details)
		s.event(customerID, EventApplied, "info", "System security fixes installed on the box's kernel (the box restarted at night).", details)
	case "fell_back":
		s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s did NOT come up on %s: the box came back on %s "+
			"by itself. %s is installed but never the default; the operator decides.", to, hostID, v.From, to), details)
	case "health_failed":
		s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s booted on %s but is NOT healthy (%s): the box "+
			"restarts ONCE into %s by itself.", to, hostID, r.HealthReason, v.From), details)
	case "self_reverted":
		s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s on %s: back on %s after the self-revert (%s). "+
			"The operator decides.", to, hostID, v.From, r.HealthReason), details)
	case "revert_failed":
		s.event(customerID, EventKernelStep, "critical", fmt.Sprintf("Kernel %s on %s: the self-revert did NOT bring back %s "+
			"(%s). No second revert — look at the box.", to, hostID, v.From, r.HealthReason), details)
	case "refused", "failed":
		s.event(customerID, EventFailed, "error", fmt.Sprintf("Kernel step on %s %s: %s", hostID, r.Outcome,
			strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
	}
}

// kernelSet is the approved set for a kernel version: the series meta-package and the signed image.
func kernelSet(kver string) []Package {
	m := kverRE.FindStringSubmatch(kver)
	if m == nil {
		return nil
	}
	v := strings.TrimSuffix(kver, "-pve")
	return []Package{{Name: "proxmox-kernel-" + m[1], Version: v, Origin: "Proxmox Debian Repository"},
		{Name: "proxmox-kernel-" + kver + "-signed", Version: v, Origin: "Proxmox Debian Repository"}}
}

// KernelStatus is the kernel set ring 0 booted and whether the operator's button may approve it: every ring-0 box's
// newest ended kernel step is "applied" (a healthy one-shot boot made it the default) for the SAME kernel, after a
// night stage; none fell back or reverted.
func (s *Service) KernelStatus() (Status, error) {
	st := Status{Layer: LayerKernel}
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		st.Waiting = "no ring-0 box"
		return st, err
	}
	kver := ""
	for _, h := range ring0 {
		reps, err := s.Store.OSReportsDesc(h, LayerKernel, 100)
		if err != nil {
			return st, err
		}
		var applied Report
		var appliedAt time.Time
		for _, r := range reps {
			var kr Report
			if json.Unmarshal([]byte(r.ReportJSON), &kr) != nil {
				continue
			}
			if kernelEnded[r.Outcome] {
				if r.Outcome != "applied" || !r.Healthy {
					st.Waiting = fmt.Sprintf("%s: the kernel step to %s ended %s", h, kernelOf(kr), r.Outcome)
					return st, nil
				}
				applied, appliedAt = kr, r.ReceivedAt
				break
			}
		}
		k := kernelOf(applied)
		if k == "" {
			st.Waiting = h + " has not booted a new kernel healthily yet"
			return st, nil
		}
		night := false
		for _, r := range reps {
			var kr Report
			if json.Unmarshal([]byte(r.ReportJSON), &kr) == nil && r.Outcome == "staged" && kr.Trigger == "night" &&
				kernelOf(kr) == k && !r.ReceivedAt.After(appliedAt) {
				night = true
				break
			}
		}
		if !night {
			st.Waiting = fmt.Sprintf("%s booted %s healthily, but not after a night step", h, k)
			return st, nil
		}
		if kver != "" && k != kver {
			st.Waiting = fmt.Sprintf("the ring-0 boxes booted different kernels (%s, %s)", kver, k)
			return st, nil
		}
		kver = k
	}
	set := kernelSet(kver)
	c := map[string]Package{}
	for _, p := range set {
		c[p.Name] = p
	}
	fp, list := fingerprint(LayerKernel, c)
	pj, _ := json.Marshal(list)
	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
	if err != nil {
		return st, err
	}
	st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list)
	if rel, _ := s.Store.LatestOSRelease(LayerKernel); rel != nil && rel.Fingerprint == fp {
		st.Approved, st.Waiting = rel.ID, "already approved"
	}
	return st, nil
}

// ApproveKernel is the operator's "Approve kernel set" (`09` §3 decision 172). A ring-1 box takes it only through a
// signed os_kernel_step — approval alone installs nothing and restarts nothing.
func (s *Service) ApproveKernel() (string, error) {
	st, err := s.KernelStatus()
	if err != nil {
		return "", err
	}
	if st.Waiting != "" || st.Fingerprint == "" {
		return "", fmt.Errorf("osupdates: the kernel set cannot be approved yet: %s", st.Waiting)
	}
	var list []Package
	_ = json.Unmarshal([]byte(s.candidatePackages(st.Fingerprint)), &list)
	if err := s.approve(LayerKernel, st.Fingerprint, list, "operator"); err != nil {
		return "", err
	}
	rel, _ := s.Store.LatestOSRelease(LayerKernel)
	return rel.ID, nil
}

func (s *Service) candidatePackages(fp string) string {
	pj, _ := s.Store.OSCandidatePackages(fp)
	return pj
}

// KernelLine is one box's kernel view for the System page.
type KernelLine struct {
	Due, Why       string // the kernel it is due to step to, or why none
	LastOutcome    string
	LastKernel     string
	LastAt         time.Time
	LastReason     string
	NoticeAt       time.Time // the newest day-before mail (any kernel)
	NoticeKver     string
	Tonight        bool
}

// KernelLineFor reads one box's kernel line.
func (s *Service) KernelLineFor(hostID string) KernelLine {
	var l KernelLine
	l.Due, l.Why = s.KernelDue(hostID)
	if b := s.KernelBlockFor(hostID); b != nil {
		l.Tonight = b.Tonight
	}
	if rep, _ := s.Store.LatestOSReport(hostID, LayerKernel); rep != nil {
		var kr Report
		_ = json.Unmarshal([]byte(rep.ReportJSON), &kr)
		l.LastOutcome, l.LastKernel, l.LastAt, l.LastReason = rep.Outcome, kernelOf(kr), rep.ReceivedAt, kr.HealthReason
		if l.LastReason == "" && len(kr.Refused) > 0 {
			l.LastReason = string(kr.Refused)
		}
	}
	if n, _ := s.Store.LatestKernelNotice(hostID); n != nil {
		l.NoticeAt, l.NoticeKver = n.SentAt, n.Kver
	}
	return l
}