felhom-agent/internal/osupdate/kernel.go

package osupdate

// The kernel lane (R-836, `09` §3 decisions 164 + 172, `11` §5.11). The root half is felhom-os-apply's layer "kernel"
// (configs/, its own tests); this file decides WHEN and judges the boot:
//
//   - the night leg (Run → runKernel): after a healthy host step, on a night the hub marks as told (the household was
//     mailed the day before — no mail, no step): ring 0 STAGES the pending kernel (select pending-kernel, the root-owned
//     ring-0 mark) and reboots; ring 1 reboots only a step a signed os_kernel_step staged earlier (KernelStepExecutor).
//   - after a boot (KernelAfterBoot, at daemon start): the wrapper says what became of the step. On the new kernel the
//     agent JUDGES the boot — the host health rule (`11` §8.2: the Proxmox daemons, the guest running and healthy, the
//     tunnel) AND the box reaching the hub — for KernelJudgeWait. Healthy → kernel-good (the new kernel becomes the
//     default). Not healthy by the deadline → ONE self-revert (kernel-revert: a reboot into the old kernel, still the
//     default). A crash on the new kernel needs nothing from the agent: GRUB already boots the old default.
//
// The host is rebooted by this file only through the wrapper (kernel-reboot, kernel-revert), and only for a staged step.

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log/slog"
	"regexp"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)

// OpKernelStep is the signed op class that STAGES a kernel set on a ring-1 box (it never reboots: the night leg does,
// once the household was told). CC may sign it until the first paying customer (R-530 ruling).
const OpKernelStep = "os_kernel_step"

// DefaultKernelJudgeWait is how long a one-shot boot may take to come back healthy before the agent reverts it ONCE.
// Measured 2026-10-07 (`audits/kernel-lane-2026-10-07/B/`): every container healthy 68 s after a reboot on demo-felhom,
// 272 s on demo-hp; 20 minutes stays under the hub's 30-minute host_stale (a box that never comes back alarms after it).
const DefaultKernelJudgeWait = 20 * time.Minute

var kverRE = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$`)

// KernelView is the wrapper's kernel object (felhom-os-apply Kernel.view).
type KernelView struct {
	Running        string  `json:"running"`
	Default        string  `json:"default"`
	Flag           *string `json:"flag"`
	Phase          string  `json:"phase"`
	From           string  `json:"from"`
	To             string  `json:"to"`
	SelfRevertUsed bool    `json:"self_revert_used"`
	Reason         string  `json:"reason"`
	VMID           int     `json:"vmid"` // the customer guest the step was staged for (the health rule's guest)
}

func parseKernel(raw json.RawMessage) KernelView {
	var v KernelView
	_ = json.Unmarshal(raw, &v)
	return v
}

// kernelPlan is one kernel-layer wrapper call.
func kernelPlan(mode string, vmid int, extra map[string]any) map[string]any {
	p := map[string]any{"release_id": "kernel", "layer": LayerKernel, "lane": "slow", "vmid": vmid, "mode": mode,
		"packages": []Package{}}
	for k, v := range extra {
		p[k] = v
	}
	return p
}

// KernelStatus reads the kernel lane's state (read only).
func (l *Leg) KernelStatus(ctx context.Context, vmid int) (KernelView, error) {
	wr, err := l.call(ctx, "kstatus"+l.now().UTC().Format("150405"), kernelPlan("kernel-status", vmid, nil))
	if err != nil {
		return KernelView{}, err
	}
	if wr.refused() {
		return KernelView{}, fmt.Errorf("kernel-status refused: %s", wr.Refused)
	}
	return parseKernel(wr.Kernel), nil
}

// kernelDue reports whether tonight's leg may take a kernel step, and why not.
func kernelDue(blk hub.WireOSUpdate, trigger string) (bool, string) {
	switch {
	case trigger != "night":
		return false, "a kernel step runs only in the night leg (never a debug pass)"
	case blk.Kernel == nil:
		return false, "the hub names no kernel step for this box"
	case !blk.Enabled:
		return false, "OS updates are switched off for this box"
	case !kverRE.MatchString(blk.Kernel.Kver):
		return false, "the hub's kernel " + blk.Kernel.Kver + " is not a kernel version"
	case !blk.Kernel.Tonight:
		return false, "the household has not been told about tonight (no mail, no step — `09` §3 decision 172)"
	}
	return true, ""
}

// runKernel is the night leg's last step. It returns the stage report (Layer "" when nothing ran). On success the box
// is rebooting when it returns.
func (l *Leg) runKernel(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
	lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", trigger, "ring", blk.Ring)
	if ok, why := kernelDue(blk, trigger); !ok {
		lg.Info("osupdate: kernel step skipped — " + why)
		return Report{}
	}
	want := blk.Kernel.Kver
	st, err := l.KernelStatus(ctx, vmid)
	if err != nil {
		return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
			Mode: "apply", Outcome: "failed", HealthReason: "kernel status unreadable: " + err.Error()})
	}
	rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply", ReleaseID: want}
	switch {
	case st.Phase == "staged" && st.To == want:
		lg.Info("osupdate: kernel step — a staged kernel waits for tonight", "from", st.From, "to", st.To)
		rep.Outcome, rep.Healthy = "staged", true
	case blk.Ring != 0:
		lg.Info("osupdate: kernel step skipped — ring 1 boots only a kernel a signed os_kernel_step staged", "phase", st.Phase, "staged", st.To, "want", want)
		return Report{}
	default:
		wr, cerr := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": "ring0-" + runID,
			"select": "pending-kernel", "expect_kver": want, "run_id": runID, "trigger": trigger, "ring": blk.Ring}))
		rep.unsent = reportFile(l.planDir(), runID, LayerKernel, "apply")
		rep.Kernel = rawOrNil(wr.Kernel)
		switch {
		case cerr != nil:
			rep.Outcome, rep.HealthReason = "failed", cerr.Error()
			return l.finish(ctx, lg, rep)
		case wr.refused():
			rep.Outcome, rep.Refused = "refused", wr.Refused
			return l.finish(ctx, lg, rep)
		case wr.failed():
			rep.Outcome, rep.Refused = "failed", wr.Failed
			return l.finish(ctx, lg, rep)
		case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
			rep.Outcome, rep.Healthy = "nothing", true
			return l.finish(ctx, lg, rep)
		}
		rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
		rep.RebootNeeded = true
	}
	rep = l.finish(ctx, lg, rep) // the hub hears "staged" BEFORE the box goes down
	wr, err := l.call(ctx, runID, kernelPlan("kernel-reboot", vmid, nil))
	switch {
	case err != nil:
		return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
			Mode: "kernel-reboot", ReleaseID: want, Outcome: "failed", HealthReason: "kernel-reboot: " + err.Error()})
	case wr.refused() || wr.failed():
		return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
			Mode: "kernel-reboot", ReleaseID: want, Outcome: "refused", Refused: firstRaw(wr.Refused, wr.Failed),
			Kernel: rawOrNil(wr.Kernel)})
	}
	lg.Warn("osupdate: kernel step — the box restarts now for its one-shot boot", "to", want)
	return rep
}

func firstRaw(a, b json.RawMessage) json.RawMessage {
	if r := rawOrNil(a); r != nil {
		return r
	}
	return rawOrNil(b)
}

// KernelJudge is what KernelAfterBoot needs besides the leg: the hub reachability probe is the "judging" report itself.
type KernelJudge struct {
	Wait time.Duration // default DefaultKernelJudgeWait
	Poll time.Duration // default 30 s
}

// KernelAfterBoot runs once at daemon start: what became of a kernel step across the boot. On the new kernel it judges
// the boot (blocking up to the wait — run it in a goroutine). vmid 0 = the guest the step recorded (it may not run yet).
func (l *Leg) KernelAfterBoot(ctx context.Context, vmid int, j KernelJudge) Report {
	runID := "boot-" + l.now().UTC().Format("20060102T150405Z")
	lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid)
	wr, err := l.call(ctx, runID, kernelPlan("kernel-boot", vmid, nil))
	if err != nil {
		lg.Warn("osupdate: kernel after-boot check failed", "err", err)
		return Report{}
	}
	if wr.refused() {
		lg.Info("osupdate: kernel after-boot check refused (an older wrapper, or a BYO host)", "refused", string(wr.Refused))
		return Report{}
	}
	v := parseKernel(wr.Kernel)
	if vmid <= 0 {
		vmid = v.VMID // after a boot the guest may not run yet — the step's own record names it
	}
	rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-boot",
		ReleaseID: v.To, Kernel: rawOrNil(wr.Kernel)}
	switch wr.KernelEvent {
	case "fell_back":
		rep.Outcome, rep.HealthReason = "fell_back", v.Reason
		lg.Warn("osupdate: kernel step FELL BACK — the new kernel did not come up; the box runs the old one", "from", v.From, "to", v.To)
		return l.finish(ctx, lg, rep)
	case "self_reverted":
		rep.Outcome, rep.HealthReason = "self_reverted", v.Reason
		lg.Warn("osupdate: kernel step SELF-REVERTED — back on the old kernel", "from", v.From, "to", v.To, "reason", v.Reason)
		return l.finish(ctx, lg, rep)
	case "revert_failed":
		rep.Outcome, rep.HealthReason = "revert_failed", v.Reason
		lg.Error("osupdate: kernel self-revert came back on the NEW kernel — no second revert; the operator decides", "to", v.To)
		return l.finish(ctx, lg, rep)
	case "judging":
		return l.judgeKernel(ctx, runID, vmid, v, wr.HealthBefore, j, lg)
	}
	return Report{}
}

// KernelVerdict is THE one-shot boot rule (R-836; pinned by TestKernelVerdict): the host health rule (`11` §8.2 —
// the Proxmox daemons and the agent active, the customer guest running and its own rule passing, the tunnel running)
// AND the box reached the hub since this boot.
func KernelVerdict(before, after *Health, tunnel string, hubReached bool) (bool, string) {
	if ok, why := HostHealthVerdict(before, after, tunnel); !ok {
		return false, why
	}
	if !hubReached {
		return false, "the box has not reached the hub since the boot"
	}
	return true, ""
}

func (l *Leg) judgeKernel(ctx context.Context, runID string, vmid int, v KernelView, before *Health, j KernelJudge, lg *slog.Logger) Report {
	wait, poll := j.Wait, j.Poll
	if wait <= 0 {
		wait = DefaultKernelJudgeWait
	}
	if poll <= 0 {
		poll = 30 * time.Second
	}
	lg.Info("osupdate: kernel step — judging the one-shot boot", "from", v.From, "to", v.To, "wait", wait.String())
	start := l.now()
	deadline := start.Add(wait)
	hubReached := false
	var why string
	for {
		if !hubReached && l.Hub != nil {
			// the hub's reachability IS this report reaching it (and the operator sees the box is back on the new kernel)
			body, _ := json.Marshal(Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
				Mode: "kernel-boot", ReleaseID: v.To, Outcome: "judging", Kernel: mustRaw(v)})
			rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
			if err := l.Hub.PostOSReport(rctx, body); err == nil {
				hubReached = true
				lg.Info("osupdate: kernel step — the box reached the hub on the new kernel", "after", l.now().Sub(start).Round(time.Second).String())
			}
			cancel()
		}
		var h *Health
		hr, err := l.call(ctx, runID, kernelPlan("health", vmid, nil))
		switch {
		case err != nil:
			why = "no health reading: " + err.Error()
		case hr.refused():
			why = "no health reading: " + string(hr.Refused)
		default:
			h = hr.Health
		}
		if h != nil {
			t := hub.TunnelUnknown
			if l.Tunnel != nil {
				t, _ = l.Tunnel.Status(ctx)
			}
			var ok bool
			ok, why = KernelVerdict(before, h, t, hubReached)
			if ok {
				return l.kernelGood(ctx, runID, vmid, v, start, lg)
			}
		}
		if !l.now().Before(deadline) || ctx.Err() != nil {
			break
		}
		l.sleep(ctx, poll)
	}
	if ctx.Err() != nil {
		lg.Warn("osupdate: kernel judging stopped (the agent is stopping) — the next start judges again", "reason", why)
		return Report{}
	}
	// not healthy by the deadline: tell the hub (best effort), then ONE self-revert into the old kernel
	rep := l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
		Mode: "kernel-revert", ReleaseID: v.To, Outcome: "health_failed", HealthReason: why + " — reverting to " + v.From,
		Kernel: mustRaw(v)})
	lg.Error("osupdate: kernel step — the one-shot boot is NOT healthy; restarting ONCE into the old kernel", "reason", why,
		"waited", wait.String(), "from", v.From, "to", v.To)
	wr, err := l.call(ctx, runID, kernelPlan("kernel-revert", vmid, map[string]any{"reason": truncate(why, 280)}))
	if err != nil || wr.refused() || wr.failed() {
		lg.Error("osupdate: kernel self-revert did not start — the box stays on the new kernel; the operator decides",
			"err", err, "refused", string(firstRaw(wr.Refused, wr.Failed)))
		return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
			Mode: "kernel-revert", ReleaseID: v.To, Outcome: "revert_failed", Refused: firstRaw(wr.Refused, wr.Failed),
			HealthReason: "the self-revert did not start"})
	}
	return rep
}

func (l *Leg) kernelGood(ctx context.Context, runID string, vmid int, v KernelView, start time.Time, lg *slog.Logger) Report {
	wr, err := l.call(ctx, runID, kernelPlan("kernel-good", vmid, nil))
	rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-good",
		ReleaseID: v.To}
	switch {
	case err != nil:
		rep.Outcome, rep.HealthReason = "failed", "kernel-good: "+err.Error()
	case wr.refused() || wr.failed():
		rep.Outcome, rep.Refused, rep.HealthReason = "failed", firstRaw(wr.Refused, wr.Failed), "kernel-good did not move the default"
	default:
		rep.Outcome, rep.Healthy = "applied", true
		rep.HealthReason = fmt.Sprintf("healthy %s after the agent started; the new kernel is the default", l.now().Sub(start).Round(time.Second))
	}
	rep.Kernel = rawOrNil(wr.Kernel)
	lg.Info("osupdate: kernel step — "+rep.Outcome, "to", v.To, "reason", rep.HealthReason)
	return l.finish(ctx, lg, rep)
}

func mustRaw(v any) json.RawMessage {
	b, _ := json.Marshal(v)
	return b
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n]
}

// KernelStepParams are a signed os_kernel_step's params: the exact kernel set (the wrapper compares it with the plan).
type KernelStepParams struct {
	ReleaseID string    `json:"release_id"`
	Packages  []Package `json:"packages"`
	Kver      string    `json:"kver"`
	VMID      int       `json:"vmid,omitempty"`
}

// KernelStepExecutor STAGES a verified os_kernel_step (signedjobs.Executor) under the heavy-op gate. It never reboots:
// the night leg reboots a staged kernel on a night the household was told about.
type KernelStepExecutor struct {
	Leg   *Leg
	Guest func(ctx context.Context) (int, error)
	Gate  func(ctx context.Context) (release func(), err error)
}

// Execute implements signedjobs.Executor.
func (e KernelStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
	if op != OpKernelStep {
		return signedjobs.ErrNoExecutor
	}
	so, ok := signedjobs.SignedOpFrom(ctx)
	if !ok {
		return fmt.Errorf("os_kernel_step: no signed envelope in the context — the wrapper could not verify it")
	}
	var p KernelStepParams
	if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 || !kverRE.MatchString(p.Kver) {
		return fmt.Errorf("os_kernel_step: params must name the kernel set and its kver: %v", err)
	}
	vmid := p.VMID
	if vmid == 0 {
		if e.Guest == nil {
			return fmt.Errorf("os_kernel_step: no vmid and no guest finder")
		}
		v, err := e.Guest(ctx)
		if err != nil {
			return fmt.Errorf("os_kernel_step: find the customer guest: %w", err)
		}
		vmid = v
	}
	if e.Gate != nil {
		release, err := e.Gate(ctx)
		if err != nil {
			return fmt.Errorf("os_kernel_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
		}
		defer release()
	}
	rep := e.Leg.StageKernelSigned(ctx, vmid, p, so.Blob, string(so.Sig))
	if rep.Outcome == "staged" || rep.Outcome == "nothing" {
		return nil
	}
	return fmt.Errorf("os_kernel_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
}

// StageKernelSigned stages a signed kernel set (ring 1): install + flag, no reboot.
func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParams, blob []byte, sig string) Report {
	unlock := l.lockPass(true)
	defer unlock()
	l.sendUnsentLocked(ctx) // R-868
	runID := l.now().UTC().Format("20060102T150405Z")
	rid := p.ReleaseID
	if rid == "" {
		rid = "signed-" + runID
	}
	lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", "signed", "release", rid)
	wr, err := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": rid, "select": "listed",
		"packages": p.Packages, "expect_kver": p.Kver, "run_id": runID, "trigger": "signed", "ring": l.Block().Ring,
		"signed": map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}}))
	rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid, Mode: "apply",
		ReleaseID: rid, Kernel: rawOrNil(wr.Kernel), unsent: reportFile(l.planDir(), runID, LayerKernel, "apply")}
	switch {
	case err != nil:
		rep.Outcome, rep.HealthReason = "failed", err.Error()
	case wr.refused():
		rep.Outcome, rep.Refused = "refused", wr.Refused
	case wr.failed():
		rep.Outcome, rep.Refused = "failed", wr.Failed
	case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
		rep.Outcome, rep.Healthy = "nothing", true
	default:
		rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
		rep.RebootNeeded = true
	}
	return l.finish(ctx, lg, rep)
}