package osupdate // The kernel lane (R-836, `09` §3 decisions 164 + 172, `11` §5.11). The root half is felhom-os-apply's layer "kernel" // (configs/, its own tests); this file decides WHEN and judges the boot: // // - the night leg (Run → runKernel): after a healthy host step, on a night the hub marks as told (the household was // mailed the day before — no mail, no step): ring 0 STAGES the pending kernel (select pending-kernel, the root-owned // ring-0 mark) and reboots; ring 1 reboots only a step a signed os_kernel_step staged earlier (KernelStepExecutor). // - after a boot (KernelAfterBoot, at daemon start): the wrapper says what became of the step. On the new kernel the // agent JUDGES the boot — the host health rule (`11` §8.2: the Proxmox daemons, the guest running and healthy, the // tunnel) AND the box reaching the hub — for KernelJudgeWait. Healthy → kernel-good (the new kernel becomes the // default). Not healthy by the deadline → ONE self-revert (kernel-revert: a reboot into the old kernel, still the // default). A crash on the new kernel needs nothing from the agent: GRUB already boots the old default. // // The host is rebooted by this file only through the wrapper (kernel-reboot, kernel-revert), and only for a staged step. import ( "context" "encoding/base64" "encoding/json" "fmt" "log/slog" "regexp" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" "gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs" ) // OpKernelStep is the signed op class that STAGES a kernel set on a ring-1 box (it never reboots: the night leg does, // once the household was told). CC may sign it until the first paying customer (R-530 ruling). const OpKernelStep = "os_kernel_step" // DefaultKernelJudgeWait is how long a one-shot boot may take to come back healthy before the agent reverts it ONCE. // Measured 2026-10-07 (`audits/kernel-lane-2026-10-07/B/`): every container healthy 68 s after a reboot on demo-felhom, // 272 s on demo-hp; 20 minutes stays under the hub's 45-minute host_stale (a box that never comes back alarms after it). const DefaultKernelJudgeWait = 20 * time.Minute var kverRE = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$`) // KernelView is the wrapper's kernel object (felhom-os-apply Kernel.view). type KernelView struct { Running string `json:"running"` Default string `json:"default"` Flag *string `json:"flag"` Phase string `json:"phase"` From string `json:"from"` To string `json:"to"` SelfRevertUsed bool `json:"self_revert_used"` Reason string `json:"reason"` VMID int `json:"vmid"` // the customer guest the step was staged for (the health rule's guest) } // kernelReportRing is the ring an after-boot report carries. In the first second after a boot the agent has not // fetched the hub's block yet, and Block() then answers ring 1 — so a ring-0 box's „judging" report said ring 1 // (seen on demo-felhom, 2026-10-08 night, `audits/kernel-night-2026-10-07/readback/`). Order: the fetched block; the // block the daemon saved on disk before the reboot (R-866); ring 1 as before. A label only: the hub's approval reads its // own ring list. Pinned by TestKernelReportRing_BeforeFirstFetch. func (l *Leg) kernelReportRing() int { l.mu.Lock() fetched := l.block l.mu.Unlock() if fetched != nil { return fetched.Ring } if b, _, ok := LoadSavedBlock(l.planDir()); ok && b != nil { return b.Ring } return 1 } func parseKernel(raw json.RawMessage) KernelView { var v KernelView _ = json.Unmarshal(raw, &v) return v } // kernelPlan is one kernel-layer wrapper call. func kernelPlan(mode string, vmid int, extra map[string]any) map[string]any { p := map[string]any{"release_id": "kernel", "layer": LayerKernel, "lane": "slow", "vmid": vmid, "mode": mode, "packages": []Package{}} for k, v := range extra { p[k] = v } return p } // KernelStatus reads the kernel lane's state (read only). func (l *Leg) KernelStatus(ctx context.Context, vmid int) (KernelView, error) { wr, err := l.call(ctx, "kstatus"+l.now().UTC().Format("150405"), kernelPlan("kernel-status", vmid, nil)) if err != nil { return KernelView{}, err } if wr.refused() { return KernelView{}, fmt.Errorf("kernel-status refused: %s", wr.Refused) } return parseKernel(wr.Kernel), nil } // kernelDue reports whether tonight's leg may take a kernel step, and why not. func kernelDue(blk hub.WireOSUpdate, trigger string) (bool, string) { switch { case trigger != "night": return false, "a kernel step runs only in the night leg (never a debug pass)" case blk.Kernel == nil: return false, "the hub names no kernel step for this box" case !blk.Enabled: return false, "OS updates are switched off for this box" case !kverRE.MatchString(blk.Kernel.Kver): return false, "the hub's kernel " + blk.Kernel.Kver + " is not a kernel version" case !blk.Kernel.Tonight: return false, "the household has not been told about tonight (no mail, no step — `09` §3 decision 172)" } return true, "" } // runKernel is the night leg's last step. It returns the stage report (Layer "" when nothing ran). On success the box // is rebooting when it returns. func (l *Leg) runKernel(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report { lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", trigger, "ring", blk.Ring) if ok, why := kernelDue(blk, trigger); !ok { lg.Info("osupdate: kernel step skipped — " + why) return Report{} } want := blk.Kernel.Kver st, err := l.KernelStatus(ctx, vmid) if err != nil { return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply", Outcome: "failed", HealthReason: "kernel status unreadable: " + err.Error()}) } rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply", ReleaseID: want} switch { case st.Phase == "staged" && st.To == want: lg.Info("osupdate: kernel step — a staged kernel waits for tonight", "from", st.From, "to", st.To) rep.Outcome, rep.Healthy = "staged", true case blk.Ring != 0: lg.Info("osupdate: kernel step skipped — ring 1 boots only a kernel a signed os_kernel_step staged", "phase", st.Phase, "staged", st.To, "want", want) return Report{} default: // R-898: EXACTLY the kernel the household was told about — never "whatever is pending tonight" (the sources can // offer a newer one by night; the step then refused, R23, and the night was lost). A version no longer // installable is refused by the wrapper before any change (R7) and the hub tells the household again. wr, cerr := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": "ring0-" + runID, "select": "listed", "packages": KernelSet(want), "expect_kver": want, "run_id": runID, "trigger": trigger, "ring": blk.Ring})) rep.unsent = reportFile(l.planDir(), runID, LayerKernel, "apply") rep.Kernel = rawOrNil(wr.Kernel) switch { case cerr != nil: rep.Outcome, rep.HealthReason = "failed", cerr.Error() return l.finish(ctx, lg, rep) case wr.refused(): rep.Outcome, rep.Refused = "refused", wr.Refused return l.finish(ctx, lg, rep) case wr.failed(): rep.Outcome, rep.Refused = "failed", wr.Failed return l.finish(ctx, lg, rep) case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0: rep.Outcome, rep.Healthy = "nothing", true return l.finish(ctx, lg, rep) } rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds rep.RebootNeeded = true } rep = l.finish(ctx, lg, rep) // the hub hears "staged" BEFORE the box goes down wr, err := l.call(ctx, runID, kernelPlan("kernel-reboot", vmid, nil)) switch { case err != nil: return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "kernel-reboot", ReleaseID: want, Outcome: "failed", HealthReason: "kernel-reboot: " + err.Error()}) case wr.refused() || wr.failed(): return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "kernel-reboot", ReleaseID: want, Outcome: "refused", Refused: firstRaw(wr.Refused, wr.Failed), Kernel: rawOrNil(wr.Kernel)}) } lg.Warn("osupdate: kernel step — the box restarts now for its one-shot boot", "to", want) return rep } func firstRaw(a, b json.RawMessage) json.RawMessage { if r := rawOrNil(a); r != nil { return r } return rawOrNil(b) } // KernelJudge is what KernelAfterBoot needs besides the leg: the hub reachability probe is the "judging" report itself. type KernelJudge struct { Wait time.Duration // default DefaultKernelJudgeWait Poll time.Duration // default 30 s } // KernelAfterBoot runs once at daemon start: what became of a kernel step across the boot. On the new kernel it judges // the boot (blocking up to the wait — run it in a goroutine). vmid 0 = the guest the step recorded (it may not run yet). func (l *Leg) KernelAfterBoot(ctx context.Context, vmid int, j KernelJudge) Report { runID := "boot-" + l.now().UTC().Format("20060102T150405Z") lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid) wr, err := l.call(ctx, runID, kernelPlan("kernel-boot", vmid, nil)) if err != nil { lg.Warn("osupdate: kernel after-boot check failed", "err", err) return Report{} } if wr.refused() { lg.Info("osupdate: kernel after-boot check refused (an older wrapper, or a BYO host)", "refused", string(wr.Refused)) return Report{} } v := parseKernel(wr.Kernel) if vmid <= 0 { vmid = v.VMID // after a boot the guest may not run yet — the step's own record names it } rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.kernelReportRing(), VMID: vmid, Mode: "kernel-boot", ReleaseID: v.To, Kernel: rawOrNil(wr.Kernel)} switch wr.KernelEvent { case "fell_back": rep.Outcome, rep.HealthReason = "fell_back", v.Reason lg.Warn("osupdate: kernel step FELL BACK — the new kernel did not come up; the box runs the old one", "from", v.From, "to", v.To) return l.finish(ctx, lg, rep) case "self_reverted": rep.Outcome, rep.HealthReason = "self_reverted", v.Reason lg.Warn("osupdate: kernel step SELF-REVERTED — back on the old kernel", "from", v.From, "to", v.To, "reason", v.Reason) return l.finish(ctx, lg, rep) case "revert_failed": rep.Outcome, rep.HealthReason = "revert_failed", v.Reason lg.Error("osupdate: kernel self-revert came back on the NEW kernel — no second revert; the operator decides", "to", v.To) return l.finish(ctx, lg, rep) case "judging": return l.judgeKernel(ctx, runID, vmid, v, wr.HealthBefore, j, lg) } return Report{} } // KernelVerdict is THE one-shot boot rule (R-836; pinned by TestKernelVerdict): the host health rule (`11` §8.2 — // the Proxmox daemons and the agent active, the customer guest running and its own rule passing, the tunnel running) // AND the box reached the hub since this boot. func KernelVerdict(before, after *Health, tunnel string, hubReached bool) (bool, string) { if ok, why := HostHealthVerdict(before, after, tunnel); !ok { return false, why } if !hubReached { return false, "the box has not reached the hub since the boot" } return true, "" } func (l *Leg) judgeKernel(ctx context.Context, runID string, vmid int, v KernelView, before *Health, j KernelJudge, lg *slog.Logger) Report { wait, poll := j.Wait, j.Poll if wait <= 0 { wait = DefaultKernelJudgeWait } if poll <= 0 { poll = 30 * time.Second } lg.Info("osupdate: kernel step — judging the one-shot boot", "from", v.From, "to", v.To, "wait", wait.String()) start := l.now() deadline := start.Add(wait) hubReached := false var why string for { if !hubReached && l.Hub != nil { // the hub's reachability IS this report reaching it (and the operator sees the box is back on the new kernel) body, _ := json.Marshal(Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.kernelReportRing(), VMID: vmid, Mode: "kernel-boot", ReleaseID: v.To, Outcome: "judging", Kernel: mustRaw(v)}) rctx, cancel := context.WithTimeout(ctx, 30*time.Second) if err := l.Hub.PostOSReport(rctx, body); err == nil { hubReached = true lg.Info("osupdate: kernel step — the box reached the hub on the new kernel", "after", l.now().Sub(start).Round(time.Second).String()) } cancel() } var h *Health hr, err := l.call(ctx, runID, kernelPlan("health", vmid, nil)) switch { case err != nil: why = "no health reading: " + err.Error() case hr.refused(): why = "no health reading: " + string(hr.Refused) default: h = hr.Health } if h != nil { t := hub.TunnelUnknown if l.Tunnel != nil { t, _ = l.Tunnel.Status(ctx) } var ok bool ok, why = KernelVerdict(before, h, t, hubReached) if ok { return l.kernelGood(ctx, runID, vmid, v, start, lg) } } if !l.now().Before(deadline) || ctx.Err() != nil { break } l.sleep(ctx, poll) } if ctx.Err() != nil { lg.Warn("osupdate: kernel judging stopped (the agent is stopping) — the next start judges again", "reason", why) return Report{} } // not healthy by the deadline: tell the hub (best effort), then ONE self-revert into the old kernel rep := l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.kernelReportRing(), VMID: vmid, Mode: "kernel-revert", ReleaseID: v.To, Outcome: "health_failed", HealthReason: why + " — reverting to " + v.From, Kernel: mustRaw(v)}) lg.Error("osupdate: kernel step — the one-shot boot is NOT healthy; restarting ONCE into the old kernel", "reason", why, "waited", wait.String(), "from", v.From, "to", v.To) wr, err := l.call(ctx, runID, kernelPlan("kernel-revert", vmid, map[string]any{"reason": truncate(why, 280)})) if err != nil || wr.refused() || wr.failed() { lg.Error("osupdate: kernel self-revert did not start — the box stays on the new kernel; the operator decides", "err", err, "refused", string(firstRaw(wr.Refused, wr.Failed))) return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.kernelReportRing(), VMID: vmid, Mode: "kernel-revert", ReleaseID: v.To, Outcome: "revert_failed", Refused: firstRaw(wr.Refused, wr.Failed), HealthReason: "the self-revert did not start"}) } return rep } func (l *Leg) kernelGood(ctx context.Context, runID string, vmid int, v KernelView, start time.Time, lg *slog.Logger) Report { wr, err := l.call(ctx, runID, kernelPlan("kernel-good", vmid, nil)) rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.kernelReportRing(), VMID: vmid, Mode: "kernel-good", ReleaseID: v.To} switch { case err != nil: rep.Outcome, rep.HealthReason = "failed", "kernel-good: "+err.Error() case wr.refused() || wr.failed(): rep.Outcome, rep.Refused, rep.HealthReason = "failed", firstRaw(wr.Refused, wr.Failed), "kernel-good did not move the default" default: rep.Outcome, rep.Healthy = "applied", true rep.HealthReason = fmt.Sprintf("healthy %s after the agent started; the new kernel is the default", l.now().Sub(start).Round(time.Second)) } rep.Kernel = rawOrNil(wr.Kernel) lg.Info("osupdate: kernel step — "+rep.Outcome, "to", v.To, "reason", rep.HealthReason) return l.finish(ctx, lg, rep) } func mustRaw(v any) json.RawMessage { b, _ := json.Marshal(v) return b } func truncate(s string, n int) string { if len(s) <= n { return s } return s[:n] } // KernelSet is the package set that installs exactly kver (R-898; the hub's kernelSet, field-exact): the series // meta-package and the signed image, both at the kernel's own version. Proxmox keeps old kernel versions in its archive. // nil for a string that is not a kernel version. func KernelSet(kver string) []Package { m := kverSeriesRE.FindStringSubmatch(kver) if m == nil { return nil } v := kver[:len(kver)-len("-pve")] return []Package{{Name: "proxmox-kernel-" + m[1], Version: v, Origin: PVEOrigin}, {Name: "proxmox-kernel-" + kver + "-signed", Version: v, Origin: PVEOrigin}} } var kverSeriesRE = regexp.MustCompile(`^([0-9]+\.[0-9]+)\.[0-9]+-[0-9]+-pve$`) // KernelStepParams are a signed os_kernel_step's params: the exact kernel set (the wrapper compares it with the plan). type KernelStepParams struct { ReleaseID string `json:"release_id"` Packages []Package `json:"packages"` Kver string `json:"kver"` VMID int `json:"vmid,omitempty"` } // KernelStepExecutor STAGES a verified os_kernel_step (signedjobs.Executor) under the heavy-op gate. It never reboots: // the night leg reboots a staged kernel on a night the household was told about. type KernelStepExecutor struct { Leg *Leg Guest func(ctx context.Context) (int, error) Gate func(ctx context.Context) (release func(), err error) } // Execute implements signedjobs.Executor. func (e KernelStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error { if op != OpKernelStep { return signedjobs.ErrNoExecutor } so, ok := signedjobs.SignedOpFrom(ctx) if !ok { return fmt.Errorf("os_kernel_step: no signed envelope in the context — the wrapper could not verify it") } var p KernelStepParams if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 || !kverRE.MatchString(p.Kver) { return fmt.Errorf("os_kernel_step: params must name the kernel set and its kver: %v", err) } vmid := p.VMID if vmid == 0 { if e.Guest == nil { return fmt.Errorf("os_kernel_step: no vmid and no guest finder") } v, err := e.Guest(ctx) if err != nil { return fmt.Errorf("os_kernel_step: find the customer guest: %w", err) } vmid = v } if e.Gate != nil { release, err := e.Gate(ctx) if err != nil { return fmt.Errorf("os_kernel_step: heavy-op gate busy (a backup or restore-test runs): %w", err) } defer release() } rep := e.Leg.StageKernelSigned(ctx, vmid, p, so.Blob, string(so.Sig)) if rep.Outcome == "staged" || rep.Outcome == "nothing" { return nil } return fmt.Errorf("os_kernel_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused)) } // StageKernelSigned stages a signed kernel set (ring 1): install + flag, no reboot. func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParams, blob []byte, sig string) Report { unlock := l.lockPass(true) defer unlock() l.sendUnsentLocked(ctx) // R-868 runID := l.now().UTC().Format("20060102T150405Z") rid := p.ReleaseID if rid == "" { rid = "signed-" + runID } lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", "signed", "release", rid) wr, err := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": rid, "select": "listed", "packages": p.Packages, "expect_kver": p.Kver, "run_id": runID, "trigger": "signed", "ring": l.Block().Ring, "signed": map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}})) rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid, Mode: "apply", ReleaseID: rid, Kernel: rawOrNil(wr.Kernel), unsent: reportFile(l.planDir(), runID, LayerKernel, "apply")} switch { case err != nil: rep.Outcome, rep.HealthReason = "failed", err.Error() case wr.refused(): rep.Outcome, rep.Refused = "refused", wr.Refused case wr.failed(): rep.Outcome, rep.Refused = "failed", wr.Failed case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0: rep.Outcome, rep.Healthy = "nothing", true default: rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds rep.RebootNeeded = true } return l.finish(ctx, lg, rep) }