package osupdates // The kernel lane, hub half (hub v0.143.0; R-836, `09` §3 decision 172, `11` §5.11). The agent stages a kernel, boots // it ONCE through a flag on the ESP, and makes it the default only after a healthy boot (felhom-agent internal/osupdate // kernel.go + the wrapper's layer "kernel"). The hub decides WHEN: // // - A box is DUE when ring 0 has a pending kernel (its host report's `proxmox-kernel-X.Y` upgrade), or a ring-1 box // runs a kernel a signed os_kernel_step STAGED — and no step for that kernel has ended yet (an ended step is the // operator's to judge; the hub never retries it by itself). // - The household of a due box gets ONE mail the day before, in its language, between 09:00 and 20:00 Budapest // time (KernelNotify): the box restarts tonight; if it is not back by morning, unplug it, wait 10 seconds, plug it // back in. Only a mail the mail service ACCEPTED is recorded. // - The box's os_update block carries `kernel: {kver, tonight}`; tonight is true only within 24 h of such a mail. No // mail → no step: the agent's night leg refuses a kernel without `tonight`. // - The operator approves a kernel set ("Approve kernel set") once every ring-0 box booted it healthily as the // default after a night step; a ring-1 box takes it only through a signed os_kernel_step. // // Pinned by kernel_test.go. import ( "encoding/json" "fmt" "regexp" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // LayerKernel is the host's kernel (agent v0.152.0). const LayerKernel = "kernel" // Kernel-lane events — operator only (the household's one word is the day-before mail). const ( EventKernelStep = "os_kernel_step" // what became of a kernel step (staged, the default moved, fell back, …) EventKernelNotice = "os_kernel_notice" // the household was told — or could not be told (then no step) ) // The day-before mail's window and limits (decided by CC — operator may reverse): mailed only in the household's // daytime, valid for the night that follows, never twice within 20 h, at most 3 times for one kernel on one box. const ( KernelNoticeFromHour = 9 KernelNoticeToHour = 20 KernelNoticeValid = 24 * time.Hour KernelNoticeGap = 20 * time.Hour KernelNoticeMax = 3 ) // KernelBlock is a box's kernel instruction (felhom-agent hub.WireKernelStep — field-exact, cross-repo). type KernelBlock struct { Kver string `json:"kver"` Tonight bool `json:"tonight"` NotifiedAt string `json:"notified_at,omitempty"` } var ( kverRE = regexp.MustCompile(`^([0-9]+\.[0-9]+)\.[0-9]+-[0-9]+-pve$`) kernelMetaRE = regexp.MustCompile(`^proxmox-kernel-[0-9]+\.[0-9]+$`) ) // kernelEnded are the outcomes after which a kernel is never stepped again by itself (the operator decides). var kernelEnded = map[string]bool{"applied": true, "fell_back": true, "health_failed": true, "self_reverted": true, "revert_failed": true} // refusals that will not go away by waiting a night (the box cannot do a one-shot, the set is wrong, no authority, // not an appliance) — such a refusal ends the step too. A transient one (a lock, the crash guard's window) is retried // within KernelNoticeMax mails. var kernelRefusedForGood = map[string]bool{"R20": true, "R23": true, "R3": true, "R12": true} func budapestLoc() *time.Location { if l, err := time.LoadLocation("Europe/Budapest"); err == nil { return l } return time.FixedZone("CET", 3600) } // kernelOf is the kernel a kernel-layer report is about: the wrapper's view's "to", else the release id. func kernelOf(r Report) string { var v struct { To string `json:"to"` } if len(r.Kernel) > 0 && json.Unmarshal(r.Kernel, &v) == nil && kverRE.MatchString(v.To) { return v.To } if kverRE.MatchString(r.ReleaseID) { return r.ReleaseID } return "" } func refusedCode(raw json.RawMessage) string { var c struct { Code string `json:"code"` } _ = json.Unmarshal(raw, &c) return c.Code } // kernelLane is the box's newest kernel-lane facts (nil: an older agent, or no host report). func (s *Service) kernelLane(hostID string) *sysfacts.KernelLane { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return nil } rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID) if rj == "" { return nil } return sysfacts.Parse(rj).Host.KernelLane } // KernelDue says which kernel the box is due to step to, or why none. func (s *Service) KernelDue(hostID string) (kver, why string) { st := s.Store.GetOSHostSettings(hostID) if !st.Enabled { return "", "OS updates are switched off" } rep, _ := s.Store.LatestOSReport(hostID, LayerHost) if rep == nil { return "", "no host step reported (the kernel lane is for appliances)" } lane := s.kernelLane(hostID) if lane == nil { return "", "the box reports no kernel lane (agent older than v0.152.0)" } if len(lane.SetupProblems) > 0 { return "", "the box cannot do a one-shot boot: " + strings.Join(lane.SetupProblems, "; ") } if st.Ring == 1 { if lane.Phase != "staged" || !kverRE.MatchString(lane.To) { return "", "ring 1: no kernel staged by a signed os_kernel_step" } kver = lane.To } else { var hr Report _ = json.Unmarshal([]byte(rep.ReportJSON), &hr) for _, p := range hr.Pending { if kernelMetaRE.MatchString(p.Name) && p.From != "" && kverRE.MatchString(p.To+"-pve") { kver = p.To + "-pve" } } if kver == "" && lane.Phase == "staged" && kverRE.MatchString(lane.To) { kver = lane.To } if kver == "" { return "", "no pending kernel" } } if lane.Running == kver { return "", "the box already runs " + kver } reps, _ := s.Store.OSReportsDesc(hostID, LayerKernel, 100) for _, r := range reps { var kr Report if json.Unmarshal([]byte(r.ReportJSON), &kr) != nil || kernelOf(kr) != kver { continue } if kernelEnded[r.Outcome] { return "", fmt.Sprintf("the kernel step to %s ended %s at %s — the operator decides", kver, r.Outcome, r.ReceivedAt.UTC().Format("2006-01-02 15:04")) } if r.Outcome == "refused" && kernelRefusedForGood[refusedCode(kr.Refused)] { return "", fmt.Sprintf("the kernel step to %s was refused (%s) — the operator decides", kver, refusedCode(kr.Refused)) } } return kver, "" } // KernelBlockFor is the box's kernel instruction (nil: not due). func (s *Service) KernelBlockFor(hostID string) *KernelBlock { kver, _ := s.KernelDue(hostID) if kver == "" { return nil } b := &KernelBlock{Kver: kver} ns, _ := s.Store.KernelNotices(hostID, kver) if len(ns) > 0 { last := ns[len(ns)-1].SentAt if s.now().Sub(last) < KernelNoticeValid { b.Tonight, b.NotifiedAt = true, last.UTC().Format(time.RFC3339) } } return b } // KernelNotify tells each due box's household, the day before (`09` §3 decision 172). Called every minute; it acts only // between KernelNoticeFromHour and KernelNoticeToHour Budapest time. Returns the boxes it told. func (s *Service) KernelNotify() []string { now := s.now() lt := now.In(budapestLoc()) if lt.Hour() < KernelNoticeFromHour || lt.Hour() >= KernelNoticeToHour { return nil } hosts, err := s.Store.ListHosts() if err != nil { return nil } var told []string for _, h := range hosts { kver, _ := s.KernelDue(h.HostID) if kver == "" { continue } ns, _ := s.Store.KernelNotices(h.HostID, kver) if len(ns) > 0 && now.Sub(ns[len(ns)-1].SentAt) < KernelNoticeGap { continue } if len(ns) >= KernelNoticeMax { key := "kernel_notice_max:" + h.HostID + ":" + kver if s.Store.OSAlarmRaised(key).IsZero() { _ = s.Store.SetOSAlarmRaised(key, now) s.event(h.CustomerID, EventKernelNotice, "warning", fmt.Sprintf("Kernel %s on %s: the household was told %d times "+ "and no kernel step ran — no further mail; the operator decides (the System page shows why).", kver, h.HostID, len(ns)), map[string]any{"host_id": h.HostID, "kver": kver, "notices": len(ns)}) } continue } if s.KernelMail == nil { continue } lang, err := s.KernelMail(h.CustomerID, kver) if err != nil { key := "kernel_notice_failed:" + h.HostID + ":" + kver + ":" + lt.Format("2006-01-02") if s.Store.OSAlarmRaised(key).IsZero() { _ = s.Store.SetOSAlarmRaised(key, now) s.event(h.CustomerID, EventKernelNotice, "warning", fmt.Sprintf("Kernel %s on %s: the household could NOT be told "+ "(%v) — so no kernel step tonight (no mail, no step).", kver, h.HostID, err), map[string]any{"host_id": h.HostID, "kver": kver, "error": err.Error()}) } continue } if err := s.Store.SaveKernelNotice(store.KernelNotice{HostID: h.HostID, Kver: kver, SentAt: now, Lang: lang}); err != nil { s.logf("[ERROR] osupdates: kernel notice for %s sent but NOT recorded (%v) — no step tonight", h.HostID, err) continue } told = append(told, h.HostID) s.logf("[INFO] osupdates: kernel %s — the household of %s was told the box restarts tonight (lang=%s)", kver, h.HostID, lang) s.event(h.CustomerID, EventKernelNotice, "info", fmt.Sprintf("Kernel %s on %s: the household was told the box "+ "restarts tonight (mail %d of at most %d).", kver, h.HostID, len(ns)+1, KernelNoticeMax), map[string]any{"host_id": h.HostID, "kver": kver, "lang": lang}) if s.Bump != nil { s.Bump(h.HostID) } } sort.Strings(told) return told } // kernelIngest raises the kernel layer's operator events (called by Ingest for layer "kernel"). func (s *Service) kernelIngest(hostID, customerID string, r Report, details map[string]any) { var v struct { From, To, Reason string } _ = json.Unmarshal(r.Kernel, &v) to := kernelOf(r) details["kver"] = to switch r.Outcome { case "staged": s.event(customerID, EventKernelStep, "info", fmt.Sprintf("Kernel %s staged on %s (trigger %s): installed, never the "+ "default; the box boots it ONCE at its night reboot.", to, hostID, r.Trigger), details) case "applied": s.event(customerID, EventKernelStep, "info", fmt.Sprintf("Kernel %s booted healthily on %s and is the default now "+ "(was %s). %s", to, hostID, v.From, r.HealthReason), details) s.event(customerID, EventApplied, "info", "System security fixes installed on the box's kernel (the box restarted at night).", details) case "fell_back": s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s did NOT come up on %s: the box came back on %s "+ "by itself. %s is installed but never the default; the operator decides.", to, hostID, v.From, to), details) case "health_failed": s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s booted on %s but is NOT healthy (%s): the box "+ "restarts ONCE into %s by itself.", to, hostID, r.HealthReason, v.From), details) case "self_reverted": s.event(customerID, EventKernelStep, "error", fmt.Sprintf("Kernel %s on %s: back on %s after the self-revert (%s). "+ "The operator decides.", to, hostID, v.From, r.HealthReason), details) case "revert_failed": s.event(customerID, EventKernelStep, "critical", fmt.Sprintf("Kernel %s on %s: the self-revert did NOT bring back %s "+ "(%s). No second revert — look at the box.", to, hostID, v.From, r.HealthReason), details) case "refused", "failed": s.event(customerID, EventFailed, "error", fmt.Sprintf("Kernel step on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) } } // kernelSet is the approved set for a kernel version: the series meta-package and the signed image. func kernelSet(kver string) []Package { m := kverRE.FindStringSubmatch(kver) if m == nil { return nil } v := strings.TrimSuffix(kver, "-pve") return []Package{{Name: "proxmox-kernel-" + m[1], Version: v, Origin: "Proxmox Debian Repository"}, {Name: "proxmox-kernel-" + kver + "-signed", Version: v, Origin: "Proxmox Debian Repository"}} } // KernelStatus is the kernel set ring 0 booted and whether the operator's button may approve it: every ring-0 box's // newest ended kernel step is "applied" (a healthy one-shot boot made it the default) for the SAME kernel, after a // night stage; none fell back or reverted. func (s *Service) KernelStatus() (Status, error) { st := Status{Layer: LayerKernel} ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { st.Waiting = "no ring-0 box" return st, err } kver := "" for _, h := range ring0 { reps, err := s.Store.OSReportsDesc(h, LayerKernel, 100) if err != nil { return st, err } var applied Report var appliedAt time.Time for _, r := range reps { var kr Report if json.Unmarshal([]byte(r.ReportJSON), &kr) != nil { continue } if kernelEnded[r.Outcome] { if r.Outcome != "applied" || !r.Healthy { st.Waiting = fmt.Sprintf("%s: the kernel step to %s ended %s", h, kernelOf(kr), r.Outcome) return st, nil } applied, appliedAt = kr, r.ReceivedAt break } } k := kernelOf(applied) if k == "" { st.Waiting = h + " has not booted a new kernel healthily yet" return st, nil } night := false for _, r := range reps { var kr Report if json.Unmarshal([]byte(r.ReportJSON), &kr) == nil && r.Outcome == "staged" && kr.Trigger == "night" && kernelOf(kr) == k && !r.ReceivedAt.After(appliedAt) { night = true break } } if !night { st.Waiting = fmt.Sprintf("%s booted %s healthily, but not after a night step", h, k) return st, nil } if kver != "" && k != kver { st.Waiting = fmt.Sprintf("the ring-0 boxes booted different kernels (%s, %s)", kver, k) return st, nil } kver = k } set := kernelSet(kver) c := map[string]Package{} for _, p := range set { c[p.Name] = p } fp, list := fingerprint(LayerKernel, c) pj, _ := json.Marshal(list) first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) if err != nil { return st, err } st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list) if rel, _ := s.Store.LatestOSRelease(LayerKernel); rel != nil && rel.Fingerprint == fp { st.Approved, st.Waiting = rel.ID, "already approved" } return st, nil } // ApproveKernel is the operator's "Approve kernel set" (`09` §3 decision 172). A ring-1 box takes it only through a // signed os_kernel_step — approval alone installs nothing and restarts nothing. func (s *Service) ApproveKernel() (string, error) { st, err := s.KernelStatus() if err != nil { return "", err } if st.Waiting != "" || st.Fingerprint == "" { return "", fmt.Errorf("osupdates: the kernel set cannot be approved yet: %s", st.Waiting) } var list []Package _ = json.Unmarshal([]byte(s.candidatePackages(st.Fingerprint)), &list) if err := s.approve(LayerKernel, st.Fingerprint, list, "operator"); err != nil { return "", err } rel, _ := s.Store.LatestOSRelease(LayerKernel) return rel.ID, nil } func (s *Service) candidatePackages(fp string) string { pj, _ := s.Store.OSCandidatePackages(fp) return pj } // KernelLine is one box's kernel view for the System page. type KernelLine struct { Due, Why string // the kernel it is due to step to, or why none LastOutcome string LastKernel string LastAt time.Time LastReason string NoticeAt time.Time // the newest day-before mail (any kernel) NoticeKver string Tonight bool } // KernelLineFor reads one box's kernel line. func (s *Service) KernelLineFor(hostID string) KernelLine { var l KernelLine l.Due, l.Why = s.KernelDue(hostID) if b := s.KernelBlockFor(hostID); b != nil { l.Tonight = b.Tonight } if rep, _ := s.Store.LatestOSReport(hostID, LayerKernel); rep != nil { var kr Report _ = json.Unmarshal([]byte(rep.ReportJSON), &kr) l.LastOutcome, l.LastKernel, l.LastAt, l.LastReason = rep.Outcome, kernelOf(kr), rep.ReceivedAt, kr.HealthReason if l.LastReason == "" && len(kr.Refused) > 0 { l.LastReason = string(kr.Refused) } } if n, _ := s.Store.LatestKernelNotice(hostID); n != nil { l.NoticeAt, l.NoticeKver = n.SentAt, n.Kver } return l }